#!/usr/bin/env python3 """N2-A-001 step 4: build cohort.csv, results.json, results.md from raw pulls. Applies the pre-registered cohort rule (>=5,000 stars within 60 days of creation, created 2023-01-01..2025-06-30), the frozen AI rubric, and computes RESULT blocks A-D for episodes/N2-A-001/script.md. All deviations listed in data/coldstart/README.md. """ import csv, json, math, pathlib, random, re, statistics from datetime import date, datetime, timedelta ROOT = pathlib.Path(__file__).resolve().parents[2] RAW = ROOT / "data" / "coldstart" / "raw" OUT = ROOT / "data" / "coldstart" EP = ROOT / "episodes" / "N2-A-001" RUN_DATE = date(2026, 8, 27) DORMANT_CUTOFF = RUN_DATE - timedelta(days=183) # pushedAt > 6 months stale # ---------- frozen AI rubric (research.md section 4) ---------- KEYWORDS = ["ai", "llm", "gpt", "agents", "machine-learning", "deep-learning", "chatgpt", "langchain", "rag", "diffusion", "transformer"] AMBIGUOUS = {"ai", "rag", "agents"} # token-boundary match only SUBSTRING = [k for k in KEYWORDS if k not in AMBIGUOUS] def tokenize(text): return set(re.split(r"[^a-z0-9]+", text.lower())) def ai_label(name, description, topics): """Returns (label, evidence, borderline_note).""" topics_l = {t.lower() for t in (topics or [])} topic_hits = sorted(topics_l & set(KEYWORDS)) text = f"{name} {description or ''}".lower() toks = tokenize(text) sub_hits = [k for k in SUBSTRING if k in text or k.replace("-", " ") in text] tok_hits = sorted((AMBIGUOUS & toks)) hits = topic_hits + sub_hits + tok_hits label = bool(hits) borderline = None if label and not topic_hits: borderline = f"keyword-only match: {sorted(set(sub_hits + tok_hits))}" if not label: # ambiguous keyword present as substring but not as token (e.g. 'rag' in 'drag') near = [k for k in AMBIGUOUS if k in text] if near: borderline = f"NOT labeled; ambiguous substring only: {near}" return label, sorted(set(hits)), borderline # ---------- stats helpers (stdlib only) ---------- def pearson(xs, ys): n = len(xs) if n < 3: return float("nan") mx, my = statistics.fmean(xs), statistics.fmean(ys) sx = math.sqrt(sum((x - mx) ** 2 for x in xs)) sy = math.sqrt(sum((y - my) ** 2 for y in ys)) if sx == 0 or sy == 0: return float("nan") return sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / (sx * sy) def ranks(v): order = sorted(range(len(v)), key=lambda i: v[i]) r = [0.0] * len(v) i = 0 while i < len(order): j = i while j + 1 < len(order) and v[order[j + 1]] == v[order[i]]: j += 1 avg = (i + j) / 2 + 1 for k in range(i, j + 1): r[order[k]] = avg i = j + 1 return r def spearman(xs, ys): return pearson(ranks(xs), ranks(ys)) def r2_log(xs, ys): lx = [math.log1p(x) for x in xs] ly = [math.log1p(y) for y in ys] r = pearson(lx, ly) return r * r if not math.isnan(r) else float("nan") def boot_median_ci(vals, n=10000, seed=42): rng = random.Random(seed) meds = sorted(statistics.median(rng.choices(vals, k=len(vals))) for _ in range(n)) return meds[int(0.025 * n)], meds[int(0.975 * n)] # ---------- load ---------- def load(): entries = [json.loads(l) for l in (RAW / "entries.jsonl").read_text().splitlines()] outcomes = {} for l in (RAW / "outcomes.jsonl").read_text().splitlines(): j = json.loads(l) outcomes[j["repo"]] = j series = {} for line in (RAW / "daily_series.tsv").read_text().splitlines()[1:]: repo, day, w, p = line.split("\t") series.setdefault(repo, {})[day] = (int(w), int(p)) search = set() for line in (RAW / "search_crosscheck.tsv").read_text().splitlines()[1:]: search.add(line.split("\t")[0]) return entries, outcomes, series, search def merged_series(entry, series): m = {} for a in entry["aliases"]: for d, (w, p) in series.get(a, {}).items(): w0, p0 = m.get(d, (0, 0)) m[d] = (w0 + w, p0 + p) return dict(sorted(m.items())) def wsum(m, start, days): end = start + timedelta(days=days) return sum(w for d, (w, _) in m.items() if start <= date.fromisoformat(d) < end) def main(): entries, outcomes, series, search = load() review = json.loads((OUT / "ai_label_manual_review.json").read_text()) \ if (OUT / "ai_label_manual_review.json").exists() else {"overrides": {}} rows, borderlines, excluded = [], [], [] for e in entries: m = merged_series(e, series) created = date.fromisoformat(e["created"]) stars_30d = wsum(m, created, 30) o = outcomes.get(e["repo"], {}) label, evidence, bnote = ai_label(e["repo"], e.get("description"), e.get("topics")) if bnote: borderlines.append({"repo": e["repo"], "note": bnote, "evidence": evidence, "topics": e.get("topics", []), "description": (e.get("description") or "")[:120]}) deleted = e["status"] == "gone" archived = bool(o.get("isArchived") if "isArchived" in o else e.get("archived_rest")) pushed = o.get("pushedAt") or e.get("pushed_at_rest") pushed_d = date.fromisoformat(pushed[:10]) if pushed else None dormant_now = (pushed_d is not None and pushed_d < DORMANT_CUTOFF) if deleted: status_now = "deleted" elif archived: status_now = "archived" elif dormant_now: status_now = "dormant" else: status_now = "alive" def win(lb, key): w = o.get(lb) or {} return w.get(key) peak_h = max([x for x in (win("m1", "human_commits_est"), win("m2", "human_commits_est"), win("m3", "human_commits_est")) if x is not None], default=None) peak_t = max([x for x in (win("m1", "total_commits"), win("m2", "total_commits"), win("m3", "total_commits")) if x is not None], default=None) peak_a = max([x for x in (win("m1", "human_authors"), win("m2", "human_authors"), win("m3", "human_authors")) if x is not None], default=None) m12_h, m12_t, m12_a = (win("m12", "human_commits_est"), win("m12", "total_commits"), win("m12", "human_authors")) arch_at = o.get("archivedAt") cross = e.get("crossed_day") # historical status at T+12 if deleted: status_t12 = "deleted" elif arch_at and cross and arch_at[:10] <= (date.fromisoformat(cross) + timedelta(days=360)).isoformat(): status_t12 = "archived" elif m12_h == 0: status_t12 = "dormant" elif m12_h is None: status_t12 = "unknown" else: status_t12 = "active" in_cohort = (e["stars_60d_creation"] >= 5000 and date(2023, 1, 1) <= created <= date(2025, 6, 30)) row = { "repo": e["repo"], "created": e["created"], "crossed_day": cross, "stars_60d": e["stars_60d_creation"], "stars_today": e.get("stars_today"), "ai_label": int(label), "ai_label_reviewed": int(review["overrides"].get(e["repo"], {}) .get("label", label)), "status_t12": status_t12, "commits_peak_month": peak_h, "commits_month12": m12_h, "contributors": peak_a, "releases": o.get("releases_total"), "archived": int(archived), "deleted": int(deleted), # --- extras beyond the pre-registered column list --- "status_now": status_now, "archived_at": arch_at, "pushed_at": pushed, "latest_release": o.get("latest_release"), "contributors_month12": m12_a, "commits_peak_month_total": peak_t, "commits_month12_total": m12_t, "stars_30d": stars_30d, "stars_7d_launch": e["stars_7d_launch"], "stars_60d_launch": e["stars_60d_launch"], "first_seen": e["first_seen"], "creation_source": e["creation_source"], "aliases": "|".join(e["aliases"]), "ai_evidence": ";".join(evidence), "in_search_today": int(any(a in search for a in e["aliases"]) or e["repo"] in search), "fork": int(bool(e.get("fork"))), "in_cohort_strict": int(in_cohort), "gql_error": int(bool(o.get("error"))), } if in_cohort: rows.append(row) else: reason = ("created outside window" if not (date(2023, 1, 1) <= created <= date(2025, 6, 30)) else "under 5000 stars in 60d of creation (born-private late launch)") excluded.append({**row, "exclusion_reason": reason}) rows.sort(key=lambda r: -(r["stars_60d"] or 0)) cols = list(rows[0].keys()) with open(OUT / "cohort.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=cols) w.writeheader(); w.writerows(rows) with open(OUT / "excluded_candidates.csv", "w", newline="") as f: w = csv.DictWriter(f, fieldnames=cols + ["exclusion_reason"]) w.writeheader(); w.writerows(excluded) (OUT / "ai_label_borderline_calls.json").write_text(json.dumps(borderlines, indent=2)) # ================= RESULT A ================= n = len(rows) ai_n = sum(r["ai_label"] for r in rows) gone_n = sum(r["deleted"] for r in rows) hyper = [r for r in rows if r["stars_30d"] >= 10000] missing_from_search = [r["repo"] for r in rows if not r["in_search_today"]] ai_rev_n = sum(r["ai_label_reviewed"] for r in rows) A = { "cohort_n": n, "ai_n": ai_n, "ai_share_pct": round(100 * ai_n / n, 1), "ai_n_reviewed": ai_rev_n, "ai_share_reviewed_pct": round(100 * ai_rev_n / n, 1), "ai_review_note": ("ai_share_pct is the FROZEN pre-registered rubric; " "ai_share_reviewed_pct layers the documented manual audit " "(data/coldstart/ai_label_manual_review.json) on top"), "vanished_n": gone_n, "vanished_pct": round(100 * gone_n / n, 1), "hyperviral_10k_30d_n": len(hyper), "hyperviral_ai_share_pct": round(100 * sum(r["ai_label"] for r in hyper) / max(1, len(hyper)), 1), "not_in_search_today_n": len(missing_from_search), "not_in_search_today": missing_from_search, "not_in_search_breakdown": { "deleted": [r["repo"] for r in rows if not r["in_search_today"] and r["deleted"]], "fork_invisible_to_search": [r["repo"] for r in rows if not r["in_search_today"] and not r["deleted"] and r["fork"]], "decayed_below_5k_today": [r["repo"] for r in rows if not r["in_search_today"] and not r["deleted"] and not r["fork"] and (r["stars_today"] or 0) < 5000], "other": [r["repo"] for r in rows if not r["in_search_today"] and not r["deleted"] and not r["fork"] and (r["stars_today"] or 0) >= 5000], }, "search_universe_created_window_stars5k_today": len({ l.split("\t")[0] for l in (RAW / "search_crosscheck.tsv").read_text().splitlines()[1:]}), } # ================= RESULT B ================= ratios, ratios_ai, ratios_nonai = [], [], [] for r in rows: if r["commits_peak_month"] and r["commits_peak_month"] > 0 \ and r["commits_month12"] is not None: v = 100 * r["commits_month12"] / r["commits_peak_month"] ratios.append(v) (ratios_ai if r["ai_label_reviewed"] else ratios_nonai).append(v) med = statistics.median(ratios) lo, hi = boot_median_ci(ratios) dead_now = sum(1 for r in rows if r["status_now"] in ("deleted", "archived", "dormant")) status_counts = {} for r in rows: status_counts[r["status_now"]] = status_counts.get(r["status_now"], 0) + 1 t12_counts = {} for r in rows: t12_counts[r["status_t12"]] = t12_counts.get(r["status_t12"], 0) + 1 B = { "median_t12_human_commits_pct_of_peak": round(med, 1), "median_ci95": [round(lo, 1), round(hi, 1)], "n_with_commit_data": len(ratios), "pct_archived_dead_dormant_now": round(100 * dead_now / n, 1), "pct_alive_now": round(100 * (n - dead_now) / n, 1), "status_now_counts": status_counts, "status_t12_counts": t12_counts, "pct_zero_human_commits_month12": round( 100 * sum(1 for r in rows if r["commits_month12"] == 0) / max(1, sum(1 for r in rows if r["commits_month12"] is not None)), 1), "dormant_definition": "pushedAt older than 6 months at run date (2026-08-27)", "median_ai_subset_pct": round(statistics.median(ratios_ai), 1) if ratios_ai else None, "median_nonai_subset_pct": (round(statistics.median(ratios_nonai), 1) if ratios_nonai else None), "subset_basis": "ai_label_reviewed", } # ================= RESULT C ================= def paired(ykey, subset=None): xs, ys = [], [] for r in rows: if subset and not subset(r): continue x, y = r["stars_7d_launch"], r[ykey] if x is not None and y is not None: xs.append(x); ys.append(y) return xs, ys C = {"velocity_measure": "stars in first 7 days of public life (launch week)", "transform": "R^2 = squared Pearson r on log1p-transformed values", "r2": {}, "spearman": {}, "n": {}} for label, key in [("commits_month12", "commits_month12"), ("contributors_month12", "contributors_month12"), ("releases_total", "releases"), ("stars_today", "stars_today")]: xs, ys = paired(key) C["r2"][label] = round(r2_log(xs, ys), 3) C["spearman"][label] = round(spearman(xs, ys), 3) C["n"][label] = len(xs) xs, ys = [], [] for r in rows: xs.append(r["stars_7d_launch"]); ys.append(1 if r["status_now"] == "alive" else 0) rpb = pearson([math.log1p(x) for x in xs], ys) C["r2"]["alive_now_binary"] = round(rpb * rpb, 3) C["n"]["alive_now_binary"] = len(xs) # ================= RESULT D ================= vel_ret, commit_ret = [], [] for e in entries: r = next((x for x in rows if x["repo"] == e["repo"]), None) if not r: continue m = merged_series(e, series) cross = date.fromisoformat(r["crossed_day"]) m1 = wsum(m, cross, 30) m3 = wsum(m, cross + timedelta(days=60), 30) if m1 > 0: vel_ret.append(100 * m3 / m1) if r["commits_peak_month"] and r["commits_month12"] is not None: m3c = None o = outcomes.get(r["repo"], {}) if o.get("m3"): m3c = o["m3"].get("human_commits_est") if m3c is not None and r["commits_peak_month"] > 0: commit_ret.append(100 * m3c / r["commits_peak_month"]) D = { "median_star_velocity_month3_vs_month1_pct": round(statistics.median(vel_ret), 1), "star_velocity_ci95": [round(x, 1) for x in boot_median_ci(vel_ret)], "n_star_velocity": len(vel_ret), "median_human_commit_month3_vs_peak_pct": round(statistics.median(commit_ret), 1), "n_commit": len(commit_ret), "icse_framing": ("ICSE 2026 (arXiv:2412.13459): purchased fake stars' promotional " "effect dies within ~2 months. Comparison: organic viral repos' " "star inflow in month 3 after crossing vs month 1, and human " "commit activity in month 3 vs peak month."), } results = { "run_date": RUN_DATE.isoformat(), "route": "clickhouse_playground (route a) + search API cross-check (route b)", "A": A, "B": B, "C": C, "D": D, } (EP / "results.json").write_text(json.dumps(results, indent=2)) print(json.dumps(results, indent=2)[:3000]) print(f"\ncohort rows: {n}, excluded candidates: {len(excluded)}, " f"borderline AI calls: {len(borderlines)}") if __name__ == "__main__": main()