#!/usr/bin/env python3
"""
Bazaar index scanner.

Purpose (Penny, run 20+):
  1. E2 measurement: is https://www.pennyinpublic.com/api/local-schema listed?
  2. E3 lead list: find public x402 routes with concrete, fixable spec defects.
  3. E3 proof content: aggregate stats for "I audited every x402 endpoint".

Discovery index (public, no auth):
  https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources?limit=100&offset=N

Usage: python3 scan.py [out.json]
"""
import json, sys, time, urllib.request
from collections import Counter
from urllib.parse import urlparse

BASE = "https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources"
OUT = sys.argv[1] if len(sys.argv) > 1 else "bazaar-scan.json"


def fetch(offset, limit=100, tries=4):
    url = f"{BASE}?limit={limit}&offset={offset}"
    for t in range(tries):
        try:
            with urllib.request.urlopen(url, timeout=45) as r:
                return json.load(r)
        except Exception as e:
            if t == tries - 1:
                raise
            time.sleep(1.5 * (t + 1))


def _pct(vals):
    if not vals:
        return {}
    v = sorted(vals)
    def q(p):
        return v[min(len(v) - 1, int(round(p / 100 * (len(v) - 1))))]
    return {"p50": q(50), "p75": q(75), "p90": q(90), "p99": q(99), "max": v[-1]}


def main():
    first = fetch(0)
    total = first["pagination"]["total"]
    items = list(first["items"])
    off = len(items)
    while off < total:
        page = fetch(off)
        got = page["items"]
        if not got:
            break
        items.extend(got)
        off += len(got)
        if off % 2000 < 100:
            print(f"  ...{off}/{total}", file=sys.stderr)
    print(f"fetched {len(items)} of {total}", file=sys.stderr)

    defects = Counter()
    leads = []
    calls = []
    penny = []
    hosts = Counter()
    versions = Counter()

    for it in items:
        res = it.get("resource") or ""
        host = urlparse(res).netloc.lower()
        hosts[host] += 1
        versions[it.get("x402Version")] += 1
        q = it.get("quality") or {}
        calls.append((res, q.get("l30DaysTotalCalls") or 0,
                      q.get("l30DaysUniquePayers") or 0, q.get("lastCalledAt")))
        if "pennyinpublic" in res:
            penny.append(it)

        d = []
        bz = ((it.get("extensions") or {}).get("bazaar") or {})
        info = bz.get("info") or {}
        desc = (it.get("description") or "").strip()
        accepts = it.get("accepts") or []

        if not bz:
            d.append("no-bazaar-extension")
        if not info.get("input"):
            d.append("no-input-schema")
        if not info.get("output"):
            d.append("no-output-schema")
        if not desc:
            d.append("no-description")
        elif len(desc) > 500:
            d.append("description-over-500")
        if it.get("x402Version") == 1:
            d.append("x402-v1")
        if not accepts:
            d.append("no-accepts")
        else:
            for a in accepts:
                if not a.get("payTo") and not a.get("recipient"):
                    d.append("accepts-missing-payTo")
                    break
        if res and not res.startswith("https://"):
            d.append("not-https")

        for x in d:
            defects[x] += 1
        if d:
            leads.append({"resource": res, "host": host, "defects": d,
                          "description": desc[:160],
                          "lastUpdated": it.get("lastUpdated"),
                          "quality": it.get("quality")})

    paid = [c for c in calls if c[1] > 0]
    paid.sort(key=lambda c: -c[1])
    total_calls = sum(c[1] for c in calls)
    ranked = [c[0] for c in paid]
    penny_rank = next((i + 1 for i, r in enumerate(ranked) if "pennyinpublic" in r), None)

    out = {
        "total_reported": total,
        "total_fetched": len(items),
        "penny_listed": len(penny) > 0,
        "penny_entries": penny,
        "defect_counts": dict(defects.most_common()),
        "clean_count": len(items) - len(leads),
        "defective_count": len(leads),
        "top_hosts": dict(hosts.most_common(25)),
        "distinct_hosts": len(hosts),
        "x402_versions": {str(k): v for k, v in versions.items()},
        "endpoints_with_a_paid_call_l30d": len(paid),
        "endpoints_with_zero_calls_l30d": len(calls) - len(paid),
        "total_calls_l30d": total_calls,
        "calls_top10_share": round(sum(c[1] for c in paid[:10]) / total_calls, 4) if total_calls else None,
        "calls_top100_share": round(sum(c[1] for c in paid[:100]) / total_calls, 4) if total_calls else None,
        "penny_rank_among_paid": penny_rank,
        "call_percentiles_all": _pct([c[1] for c in calls]),
        "call_percentiles_paid": _pct([c[1] for c in paid]),
        "single_call_endpoints": sum(1 for c in calls if c[1] == 1),
        "top_paid": [{"resource": c[0], "calls": c[1], "payers": c[2]} for c in paid[:20]],
        "leads": leads,
    }
    with open(OUT, "w") as f:
        json.dump(out, f, indent=1)
    print(json.dumps({k: v for k, v in out.items()
                      if k not in ("leads", "penny_entries", "top_hosts", "top_paid")}, indent=1))


if __name__ == "__main__":
    main()