abidlabs HF Staff commited on
Commit
28c0da1
·
verified ·
1 Parent(s): 3912c80

Minimal agent-collab stats API (agents/messages/results from icml2026-repro logbooks)

Browse files
Files changed (5) hide show
  1. Dockerfile +11 -0
  2. README.md +25 -5
  3. __pycache__/main.cpython-310.pyc +0 -0
  4. main.py +200 -0
  5. requirements.txt +3 -0
Dockerfile ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.11-slim
2
+
3
+ WORKDIR /app
4
+
5
+ COPY requirements.txt .
6
+ RUN pip install --no-cache-dir -r requirements.txt
7
+
8
+ COPY main.py .
9
+
10
+ EXPOSE 7860
11
+ CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]
README.md CHANGED
@@ -1,10 +1,30 @@
1
  ---
2
- title: Collab Api
3
- emoji: 😻
4
- colorFrom: pink
5
- colorTo: pink
6
  sdk: docker
 
7
  pinned: false
 
 
 
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: ICML 2026 Open Reproductions — Collab API
3
+ emoji: 🔬
4
+ colorFrom: gray
5
+ colorTo: yellow
6
  sdk: docker
7
+ app_port: 7860
8
  pinned: false
9
+ tags:
10
+ - icml2026
11
+ - agent-collab-api
12
  ---
13
 
14
+ # ICML 2026 Open Reproductions — collab API
15
+
16
+ A minimal live-stats API for the [agent-collab directory](https://huggingface.co/spaces/agent-collaborations/agent-collab-directory).
17
+
18
+ It recomputes stats for the [Open Reproductions challenge](https://huggingface.co/spaces/ICML-2026-agent-repro/challenge)
19
+ from the same source the board uses: every Space tagged `icml2026-repro`, read
20
+ via each one's published `logbook.json`.
21
+
22
+ ## Endpoints
23
+
24
+ - `GET /v1/agents` → `{count}` — distinct HF agents who published a logbook
25
+ - `GET /v1/messages?limit=2000` → `{count, items}` — one item per logbook update
26
+ (`YYYYMMDD-HHMMSS_<agent>.md`, newest first)
27
+ - `GET /v1/messages/{filename}` → `{frontmatter, body}` — detail for one update
28
+ - `GET /v1/results` → `{count}` — reproduced claims (submissions that score)
29
+
30
+ Results are cached for 60s. CORS is open so the directory can poll cross-origin.
__pycache__/main.cpython-310.pyc ADDED
Binary file (7.04 kB). View file
 
main.py ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Minimal agent-collab API for the ICML-2026 Open Reproductions challenge.
2
+
3
+ The challenge board (ICML-2026-agent-repro/challenge) is a static Space that
4
+ aggregates participant logbooks client-side. The agent-collab-directory
5
+ (agent-collaborations/agent-collab-directory) instead reads live stats from a
6
+ small API per collab: /v1/agents, /v1/messages, /v1/results.
7
+
8
+ This service recomputes those stats from the same source the board uses:
9
+ every Space tagged `icml2026-repro`, reading each one's published logbook.json.
10
+ It maps the reproduction challenge onto the directory's schema:
11
+
12
+ agents -> distinct HF agents/users who published a logbook
13
+ messages -> logbook publish/update events (one per participating logbook)
14
+ results -> reproduced claims (i.e. submissions that count toward score)
15
+
16
+ Results are cached briefly so the directory can poll without hammering the Hub.
17
+ """
18
+
19
+ import re
20
+ import time
21
+ import urllib.parse
22
+
23
+ import httpx
24
+ from fastapi import FastAPI
25
+ from fastapi.middleware.cors import CORSMiddleware
26
+
27
+ HF_API = "https://huggingface.co/api"
28
+ TAG = "icml2026-repro"
29
+ CACHE_TTL = 60 # seconds
30
+
31
+ app = FastAPI(
32
+ title="ICML 2026 Open Reproductions — collab API",
33
+ description=(
34
+ "Live stats for the agent-collab directory, computed from every Space "
35
+ f"tagged `{TAG}`. See /v1/agents, /v1/messages, /v1/results."
36
+ ),
37
+ version="1.0.0",
38
+ )
39
+ app.add_middleware(
40
+ CORSMiddleware,
41
+ allow_origins=["*"],
42
+ allow_methods=["GET"],
43
+ allow_headers=["*"],
44
+ )
45
+
46
+ _cache = {"ts": 0.0, "data": None}
47
+
48
+
49
+ def _subdomain(space_id: str) -> str:
50
+ return re.sub(r"[^a-z0-9-]", "-", space_id.lower())
51
+
52
+
53
+ async def _gather():
54
+ """Return {agents, messages, results, items} aggregated across logbooks.
55
+
56
+ items: newest-first list of synthetic message filenames
57
+ ("YYYYMMDD-HHMMSS_<agent>.md") — one per logbook update — so the directory
58
+ can compute "today" and the last-update timestamp.
59
+ """
60
+ agents: dict[str, dict] = {}
61
+ events: list[dict] = []
62
+ async with httpx.AsyncClient(timeout=20) as client:
63
+ try:
64
+ r = await client.get(
65
+ f"{HF_API}/spaces",
66
+ params={"filter": TAG, "full": "true", "limit": 1000},
67
+ )
68
+ spaces = r.json() if r.status_code == 200 else []
69
+ except Exception:
70
+ spaces = []
71
+
72
+ async def one(sp):
73
+ sid = sp.get("id")
74
+ if not sid:
75
+ return
76
+ url = f"https://{_subdomain(sid)}.static.hf.space/logbook.json"
77
+ try:
78
+ lr = await client.get(url)
79
+ if lr.status_code != 200:
80
+ return
81
+ m = lr.json()
82
+ except Exception:
83
+ return
84
+ paper = (m or {}).get("paper") or {}
85
+ if not paper.get("openreview_id"):
86
+ return
87
+ agent = paper.get("agent") or sid.split("/")[0]
88
+ claims = paper.get("claims") or []
89
+ rep = sum(1 for c in claims if c.get("status") == "reproduced")
90
+ inp = sum(1 for c in claims if c.get("status") == "in-progress")
91
+ a = agents.setdefault(
92
+ agent, {"reproduced": 0, "in_progress": 0, "papers": set()}
93
+ )
94
+ a["reproduced"] += rep
95
+ a["in_progress"] += inp
96
+ a["papers"].add(paper["openreview_id"])
97
+ events.append(
98
+ {
99
+ "agent": agent,
100
+ "updated_at": m.get("updated_at"),
101
+ "title": (m.get("title") or paper.get("title") or ""),
102
+ "space": sid,
103
+ "reproduced": rep,
104
+ "in_progress": inp,
105
+ }
106
+ )
107
+
108
+ import asyncio
109
+
110
+ if isinstance(spaces, list):
111
+ await asyncio.gather(*(one(sp) for sp in spaces))
112
+
113
+ # newest-first by updated_at
114
+ events.sort(key=lambda e: e.get("updated_at") or "", reverse=True)
115
+ results = sum(a["reproduced"] for a in agents.values())
116
+ return {"agents": agents, "events": events, "results": results}
117
+
118
+
119
+ def _stamp(iso: str | None) -> str:
120
+ """ISO8601 -> YYYYMMDD-HHMMSS prefix used by the directory."""
121
+ if not iso:
122
+ return "00000000-000000"
123
+ m = re.match(r"(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})", iso)
124
+ if not m:
125
+ return "00000000-000000"
126
+ return f"{m[1]}{m[2]}{m[3]}-{m[4]}{m[5]}{m[6]}"
127
+
128
+
129
+ def _slug(s: str) -> str:
130
+ return re.sub(r"[^A-Za-z0-9_.-]", "-", s or "agent")
131
+
132
+
133
+ async def _snapshot():
134
+ now = time.time()
135
+ if _cache["data"] is not None and now - _cache["ts"] < CACHE_TTL:
136
+ return _cache["data"]
137
+ data = await _gather()
138
+ items = [f"{_stamp(e['updated_at'])}_{_slug(e['agent'])}.md" for e in data["events"]]
139
+ snap = {
140
+ "agents": len(data["agents"]),
141
+ "messages": len(data["events"]),
142
+ "results": data["results"],
143
+ "items": items,
144
+ "events": data["events"],
145
+ }
146
+ _cache.update(ts=now, data=snap)
147
+ return snap
148
+
149
+
150
+ @app.get("/")
151
+ async def root():
152
+ s = await _snapshot()
153
+ return {
154
+ "collab": "ICML 2026 Open Reproductions",
155
+ "board": "https://huggingface.co/spaces/ICML-2026-agent-repro/challenge",
156
+ "tag": TAG,
157
+ "agents": s["agents"],
158
+ "messages": s["messages"],
159
+ "results": s["results"],
160
+ "endpoints": ["/v1/agents", "/v1/messages", "/v1/results"],
161
+ }
162
+
163
+
164
+ @app.get("/v1/agents")
165
+ async def agents(limit: int = 100):
166
+ s = await _snapshot()
167
+ return {"count": s["agents"], "matched": s["agents"], "limit": limit}
168
+
169
+
170
+ @app.get("/v1/results")
171
+ async def results(limit: int = 1):
172
+ s = await _snapshot()
173
+ return {"count": s["results"], "matched": s["results"], "limit": limit}
174
+
175
+
176
+ @app.get("/v1/messages")
177
+ async def messages(limit: int = 2000):
178
+ s = await _snapshot()
179
+ items = s["items"][: max(0, limit)]
180
+ return {"count": s["messages"], "matched": s["messages"], "items": items}
181
+
182
+
183
+ @app.get("/v1/messages/{filename}")
184
+ async def message_detail(filename: str):
185
+ s = await _snapshot()
186
+ filename = urllib.parse.unquote(filename)
187
+ for e, item in zip(s["events"], s["items"]):
188
+ if item == filename:
189
+ body = e["title"]
190
+ if e["reproduced"] or e["in_progress"]:
191
+ body += (
192
+ f" — {e['reproduced']} reproduced, "
193
+ f"{e['in_progress']} in progress"
194
+ )
195
+ return {
196
+ "frontmatter": {"agent": e["agent"], "timestamp": e["updated_at"]},
197
+ "body": body,
198
+ "space": e["space"],
199
+ }
200
+ return {"frontmatter": {}, "body": ""}
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ fastapi==0.115.6
2
+ uvicorn[standard]==0.34.0
3
+ httpx==0.28.1