#!/usr/bin/env python3
"""N2-A-001 step 3: per-repo metadata + T+12 outcomes via GitHub REST + GraphQL.

Phase A (REST): resolve renames (redirects), fetch created_at, stars today, topics,
         description, archived, pushed_at. 404/451 = attrition, recorded not dropped.
Phase B: merge alias series, compute creation, crossing day (cumulative WatchEvents
         >= 5000), and the pre-registered 60-day star windows.
Phase C (GraphQL): commit history totalCount + author nodes for months 1,2,3 after
         crossing (peak proxy = max of first 3) and month 12 (T+12, 30-day months);
         releases totalCount + latestRelease; isArchived/archivedAt/pushedAt.
         Bot filter (frozen): author.user.login or author.name matching .*\\[bot\\]$,
         author.user.__typename == 'Bot', or author.email containing '[bot]' /
         'actions@github.com'.
Phase D: the stargazers-restriction test on a foreign repo (research.md §8) —
         exact REST + GraphQL responses recorded to raw/stargazers_restriction_test.json.

Outputs (data/coldstart/raw/): repo_meta.jsonl, outcomes.jsonl,
         stargazers_restriction_test.json
"""
import json, pathlib, re, sys, time, urllib.error, urllib.request
from datetime import date, datetime, timedelta

ROOT = pathlib.Path(__file__).resolve().parents[2]
RAW = ROOT / "data" / "coldstart" / "raw"

def token():
    for line in (ROOT / ".env").read_text().splitlines():
        if line.startswith("GITHUB_TOKEN="):
            return line.split("=", 1)[1].strip().strip('"').strip("'")
    raise SystemExit("GITHUB_TOKEN not found in .env")

TOKEN = token()
HDRS = {"Authorization": f"Bearer {TOKEN}", "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "coldstart-n2a001"}
BOT_RE = re.compile(r".*\[bot\]$")

def rest(path):
    req = urllib.request.Request("https://api.github.com" + path, headers=HDRS)
    for attempt in range(5):
        try:
            with urllib.request.urlopen(req, timeout=60) as r:
                return r.getcode(), json.load(r)
        except urllib.error.HTTPError as e:
            if e.code in (404, 451, 403, 410):
                if e.code == 403 and "rate limit" in (e.read() or b"").decode().lower():
                    time.sleep(60); continue
                return e.code, None
            if attempt == 4:
                raise
            time.sleep(8 * (attempt + 1))
        except Exception:
            if attempt == 4:
                raise
            time.sleep(8 * (attempt + 1))

def graphql(query, variables):
    body = json.dumps({"query": query, "variables": variables}).encode()
    req = urllib.request.Request("https://api.github.com/graphql", data=body, headers=HDRS)
    for attempt in range(5):
        try:
            with urllib.request.urlopen(req, timeout=120) as r:
                return json.load(r)
        except urllib.error.HTTPError as e:
            body_txt = ""
            try:
                body_txt = e.read().decode()
            except Exception:
                pass
            if e.code in (502, 503) or (e.code == 403 and "rate" in body_txt.lower()):
                time.sleep(30 * (attempt + 1)); continue
            if attempt == 4:
                return {"errors": [{"http_status": e.code, "body": body_txt[:500]}]}
            time.sleep(10 * (attempt + 1))
        except Exception as ex:
            if attempt == 4:
                return {"errors": [{"exception": str(ex)}]}
            time.sleep(10 * (attempt + 1))

# ---------- Phase A ----------
def phase_a(repos):
    out_path = RAW / "repo_meta.jsonl"
    done = {}
    if out_path.exists():
        for line in out_path.read_text().splitlines():
            j = json.loads(line)
            done[j["query_name"]] = j
    with open(out_path, "a") as f:
        for i, r in enumerate(repos):
            if r in done:
                continue
            code, j = rest(f"/repos/{r}")
            rec = {"query_name": r, "http_status": code}
            if j:
                rec.update({
                    "resolved_name": j["full_name"],
                    "created_at": j["created_at"],
                    "stars_today": j["stargazers_count"],
                    "archived": j["archived"],
                    "disabled": j.get("disabled", False),
                    "pushed_at": j["pushed_at"],
                    "fork": j["fork"],
                    "topics": j.get("topics", []),
                    "description": j.get("description") or "",
                })
            f.write(json.dumps(rec) + "\n"); f.flush()
            done[r] = rec
            if i % 25 == 0:
                print(f"  meta {i}/{len(repos)}")
            time.sleep(0.15)
    return done

# ---------- Phase B ----------
def load_series():
    series = {}
    for line in (RAW / "daily_series.tsv").read_text().splitlines()[1:]:
        repo, day, w, p = line.split("\t")
        series.setdefault(repo, {})[day] = (int(w), int(p))
    return series

def merge_series(names, series):
    m = {}
    for n in names:
        for d, (w, p) in series.get(n, {}).items():
            w0, p0 = m.get(d, (0, 0))
            m[d] = (w0 + w, p0 + p)
    return dict(sorted(m.items()))

def crossing_day(merged, threshold=5000):
    cum = 0
    for d, (w, _) in merged.items():
        cum += w
        if cum >= threshold:
            return d
    return None

def window_sum(merged, start, days):
    end = start + timedelta(days=days)
    return sum(w for d, (w, _) in merged.items() if start <= date.fromisoformat(d) < end)

# ---------- Phase C ----------
GQL = """
query($owner:String!,$name:String!,$m1s:GitTimestamp!,$m1u:GitTimestamp!,
      $m2s:GitTimestamp!,$m2u:GitTimestamp!,$m3s:GitTimestamp!,$m3u:GitTimestamp!,
      $m12s:GitTimestamp!,$m12u:GitTimestamp!){
  rateLimit{cost remaining}
  repository(owner:$owner,name:$name){
    nameWithOwner isArchived archivedAt pushedAt stargazerCount
    releases{totalCount} latestRelease{publishedAt}
    defaultBranchRef{ target{ ... on Commit {
      m1: history(since:$m1s, until:$m1u, first:100){...C}
      m2: history(since:$m2s, until:$m2u, first:100){...C}
      m3: history(since:$m3s, until:$m3u, first:100){...C}
      m12: history(since:$m12s, until:$m12u, first:100){...C}
    }}}
  }
}
fragment C on CommitHistoryConnection{
  totalCount pageInfo{hasNextPage endCursor}
  nodes{ author{ name email user{ login __typename } } }
}
"""
GQL_PAGE = """
query($owner:String!,$name:String!,$since:GitTimestamp!,$until:GitTimestamp!,$after:String){
  repository(owner:$owner,name:$name){ defaultBranchRef{ target{ ... on Commit {
    h: history(since:$since, until:$until, first:100, after:$after){
      totalCount pageInfo{hasNextPage endCursor}
      nodes{ author{ name email user{ login __typename } } }
    }}}}}
}
"""
MAX_PAGES = 12  # 1,200 commits per window; truncation recorded

def is_bot(a):
    if not a:
        return False
    u = a.get("user") or {}
    login = u.get("login") or ""
    name = a.get("name") or ""
    email = a.get("email") or ""
    return (u.get("__typename") == "Bot" or BOT_RE.match(login) is not None
            or BOT_RE.match(name) is not None or "[bot]" in email
            or email.startswith("actions@github.com"))

def author_key(a):
    u = (a or {}).get("user") or {}
    return u.get("login") or f"{(a or {}).get('name','?')}<{(a or {}).get('email','?')}>"

def summarize_window(owner, name, since, until, conn):
    total = conn["totalCount"]
    nodes = list(conn["nodes"])
    pages = 1
    pi = conn["pageInfo"]
    while pi["hasNextPage"] and pages < MAX_PAGES:
        j = graphql(GQL_PAGE, {"owner": owner, "name": name, "since": since,
                               "until": until, "after": pi["endCursor"]})
        try:
            h = j["data"]["repository"]["defaultBranchRef"]["target"]["h"]
        except (KeyError, TypeError):
            break
        nodes += h["nodes"]; pi = h["pageInfo"]; pages += 1
        time.sleep(0.3)
    bots = sum(1 for n in nodes if is_bot(n.get("author")))
    humans = len(nodes) - bots
    scale = (total / len(nodes)) if nodes else 1.0
    human_authors = {author_key(n["author"]) for n in nodes
                     if n.get("author") and not is_bot(n["author"])}
    return {"total_commits": total, "sampled": len(nodes),
            "truncated": bool(pi["hasNextPage"]),
            "bot_commits_sampled": bots,
            "human_commits_est": round(humans * scale),
            "human_authors": len(human_authors)}

def iso(d):
    return d.strftime("%Y-%m-%dT00:00:00Z")

def phase_c(entries):
    out_path = RAW / "outcomes.jsonl"
    done = set()
    if out_path.exists():
        for line in out_path.read_text().splitlines():
            done.add(json.loads(line)["repo"])
    with open(out_path, "a") as f:
        for i, e in enumerate(entries):
            repo = e["repo"]
            if repo in done or e["status"] != "alive" or not e.get("crossed_day"):
                continue
            owner, name = repo.split("/", 1)
            cross = date.fromisoformat(e["crossed_day"])
            wins = {}
            for label, k in (("m1", 0), ("m2", 1), ("m3", 2), ("m12", 11)):
                s = cross + timedelta(days=30 * k)
                wins[label] = (iso(s), iso(s + timedelta(days=30)))
            v = {"owner": owner, "name": name}
            for lb, (s, u) in wins.items():
                v[lb + "s"] = s; v[lb + "u"] = u
            j = graphql(GQL, v)
            rec = {"repo": repo, "windows": wins}
            if j.get("errors") or not (j.get("data") or {}).get("repository"):
                rec["error"] = j.get("errors")
                f.write(json.dumps(rec) + "\n"); f.flush()
                continue
            r = j["data"]["repository"]
            rec.update({
                "isArchived": r["isArchived"], "archivedAt": r["archivedAt"],
                "pushedAt": r["pushedAt"], "stargazerCount": r["stargazerCount"],
                "releases_total": r["releases"]["totalCount"],
                "latest_release": (r.get("latestRelease") or {}).get("publishedAt"),
            })
            tgt = (r.get("defaultBranchRef") or {}).get("target") or {}
            for lb in ("m1", "m2", "m3", "m12"):
                conn = tgt.get(lb)
                rec[lb] = (summarize_window(owner, name, wins[lb][0], wins[lb][1], conn)
                           if conn else None)
            rl = j["data"].get("rateLimit") or {}
            f.write(json.dumps(rec) + "\n"); f.flush()
            if i % 10 == 0:
                print(f"  outcomes {i}/{len(entries)} (rate remaining {rl.get('remaining')})")
            if (rl.get("remaining") or 9999) < 200:
                print("  low GraphQL budget, sleeping 15 min"); time.sleep(900)
            time.sleep(0.4)

# ---------- Phase D ----------
def phase_d():
    out = {}
    q = """query{ repository(owner:"twitter", name:"the-algorithm"){
      stargazers(first:2, orderBy:{field:STARRED_AT, direction:ASC}){
        totalCount edges{ starredAt node{ login } } } } }"""
    out["graphql_stargazers_starredAt"] = graphql(q, {})
    try:
        code, j = rest("/repos/twitter/the-algorithm/stargazers?per_page=2")
        out["rest_stargazers"] = {"http_status": code, "body_sample": j if isinstance(j, list) else j}
    except urllib.error.HTTPError as e:
        out["rest_stargazers"] = {"http_status": e.code, "body": e.read().decode()[:1000]}
    except Exception as e:
        out["rest_stargazers"] = {"exception": str(e)}
    req = urllib.request.Request(
        "https://api.github.com/repos/twitter/the-algorithm/stargazers?per_page=2",
        headers={**HDRS, "Accept": "application/vnd.github.star+json"})
    try:
        with urllib.request.urlopen(req, timeout=60) as r:
            out["rest_stargazers_starplus"] = {"http_status": r.getcode(),
                                               "body_sample": json.load(r)[:2]}
    except urllib.error.HTTPError as e:
        out["rest_stargazers_starplus"] = {"http_status": e.code, "body": e.read().decode()[:1000]}
    (RAW / "stargazers_restriction_test.json").write_text(json.dumps(out, indent=2))
    print("stargazers restriction test recorded")

# ---------- main ----------
def main():
    broad = {}
    for line in (RAW / "cohort_broad.tsv").read_text().splitlines()[1:]:
        repo, fs, s60, tot = line.split("\t")
        broad[repo] = {"first_seen": fs, "stars_60d_launch_raw": int(s60),
                       "stars_total_archive": int(tot)}
    createvents = {}
    for line in (RAW / "createvent_creations.tsv").read_text().splitlines()[1:]:
        repo, ts = line.split("\t")
        createvents[repo] = ts

    print("== Phase A: REST metadata / rename resolution ==")
    meta = phase_a(list(broad))

    print("== Phase B: merge aliases, creation + crossing ==")
    series = load_series()
    groups = {}
    for qname, rec in meta.items():
        key = rec.get("resolved_name") or qname
        groups.setdefault(key, []).append(qname)
    entries = []
    for resolved, aliases in sorted(groups.items()):
        merged = merge_series(aliases, series)
        recs = [meta[a] for a in aliases]
        live = next((r for r in recs if r.get("resolved_name")), None)
        first_seen = min(broad[a]["first_seen"] for a in aliases)
        if live:
            created = live["created_at"][:10]
            creation_source = "api"
        elif any(a in createvents for a in aliases):
            created = min(createvents[a] for a in aliases if a in createvents)[:10]
            creation_source = "create_event"
        else:
            created = first_seen[:10]
            creation_source = "first_seen"
        cd = date.fromisoformat(created)
        entry = {
            "repo": resolved, "aliases": aliases,
            "status": "alive" if live else "gone",
            "http_status": [meta[a]["http_status"] for a in aliases],
            "created": created, "creation_source": creation_source,
            "first_seen": first_seen,
            "stars_60d_creation": window_sum(merged, cd, 60),
            "stars_7d_launch": window_sum(merged, date.fromisoformat(first_seen[:10]), 7),
            "stars_60d_launch": window_sum(merged, date.fromisoformat(first_seen[:10]), 60),
            "stars_archive_total": sum(w for w, _ in merged.values()),
            "crossed_day": crossing_day(merged),
            "stars_today": (live or {}).get("stars_today"),
            "archived_rest": (live or {}).get("archived"),
            "pushed_at_rest": (live or {}).get("pushed_at"),
            "topics": (live or {}).get("topics", []),
            "description": (live or {}).get("description", ""),
            "fork": (live or {}).get("fork", False),
        }
        entries.append(entry)
    (RAW / "entries.jsonl").write_text("\n".join(json.dumps(e) for e in entries) + "\n")
    print(f"  {len(entries)} resolved entries ({sum(1 for e in entries if e['status']=='gone')} gone)")

    print("== Phase D: stargazers restriction test ==")
    phase_d()

    print("== Phase C: GraphQL outcomes ==")
    phase_c(entries)
    print("done.")

if __name__ == "__main__":
    main()