add live dataset growth + treemap; switch docs to roboto

Author: atharvas

package.json

@@ -1,69 +1,70 @@
 {
-	"name": "formulacode-www",
-	"version": "5.15.1",
-	"scripts": {
-		"deleteImgs": "node tasks/deleteNonAnimalImgs.js",
-		"analysis": "node tasks/wine-analysis.js",
-		"imgDownload": "node tasks/imgDownload.js",
-		"limitWine": "node tasks/limit-wine.js",
-		"identifyWine": "node tasks/identify-wine.js",
-		"assessWineTxt": "node tasks/assess-wine-txt.js",
-		"combineCSV": "node tasks/combine-gpt-csv.js",
-		"gdoc": "node tasks/fetch-google.js",
-		"style": "node tasks/style-dictionary.js",
-		"dev": "vite dev",
-		"build": "vite build",
-		"preview": "vite preview",
-		"lint": "prettier --check --plugin-search-dir=. .",
-		"format": "prettier --write --plugin-search-dir=. ."
-	},
-	"devDependencies": {
-		"@rollup/plugin-dsv": "^3.0.4",
-		"@sveltejs/adapter-static": "^3.0.1",
-		"@sveltejs/kit": "^2.3.5",
-		"@sveltejs/vite-plugin-svelte": "3.0.1",
-		"archieml": "^0.5.0",
-		"autoprefixer": "^10.4.17",
-		"baseline-browser-mapping": "^2.9.19",
-		"d3": "^7.8.5",
-		"layercake": "^8.0.2",
-		"lodash.debounce": "^4.0.8",
-		"lucide-svelte": "^0.294.0",
-		"postcss": "^8.4.33",
-		"prettier": "^3.2.4",
-		"prettier-plugin-svelte": "^3.1.2",
-		"rollup": "^4.9.5",
-		"rollup-plugin-svg": "^2.0.0",
-		"style-dictionary": "^3.9.2",
-		"svelte": "^4.2.9",
-		"svelte-copy": "^1.4.1",
-		"svelte-preprocess": "^5.1.3",
-		"vite": "^5.0.11",
-		"vite-plugin-svgstring": "^1.0.0"
-	},
-	"type": "module",
-	"engines": {
-		"node": ">= 18.12.0"
-	},
-	"browserslist": "> 0.5%, last 4 versions, not ie <= 11, not ie_mob <= 11",
-	"dependencies": {
-		"@leveluptuts/svelte-fit": "^1.0.3",
-		"asciinema-player": "^3.12.1",
-		"axios": "^1.6.7",
-		"d3-regression": "^1.3.10",
-		"fs": "^0.0.1-security",
-		"image-download": "^2.1.0",
-		"image-downloader": "^4.3.0",
-		"mathjax-full": "^3.2.2",
-		"openai": "^4.28.4",
-		"svelte-carousel": "^1.0.25",
-		"svelte-headless-table": "^0.18.3",
-		"svelte-highlight": "^7.9.0",
-		"svelte-lazy-image": "^0.5.1",
-		"svelte-multiselect": "^10.0.0",
-		"svelte-typeahead": "^5.0.0",
-		"wx-grid-locales": "^2.2.0",
-		"wx-svelte-grid": "^1.3.3",
-		"yootils": "^0.3.1"
-	}
+  "name": "formulacode-www",
+  "version": "5.15.1",
+  "scripts": {
+    "deleteImgs": "node tasks/deleteNonAnimalImgs.js",
+    "analysis": "node tasks/wine-analysis.js",
+    "imgDownload": "node tasks/imgDownload.js",
+    "limitWine": "node tasks/limit-wine.js",
+    "identifyWine": "node tasks/identify-wine.js",
+    "assessWineTxt": "node tasks/assess-wine-txt.js",
+    "combineCSV": "node tasks/combine-gpt-csv.js",
+    "gdoc": "node tasks/fetch-google.js",
+    "style": "node tasks/style-dictionary.js",
+    "dashboard": "node scripts/build-dashboard.mjs",
+    "dev": "vite dev",
+    "build": "vite build",
+    "preview": "vite preview",
+    "lint": "prettier --check --plugin-search-dir=. .",
+    "format": "prettier --write --plugin-search-dir=. ."
+  },
+  "devDependencies": {
+    "@rollup/plugin-dsv": "^3.0.4",
+    "@sveltejs/adapter-static": "^3.0.1",
+    "@sveltejs/kit": "^2.3.5",
+    "@sveltejs/vite-plugin-svelte": "3.0.1",
+    "archieml": "^0.5.0",
+    "autoprefixer": "^10.4.17",
+    "baseline-browser-mapping": "^2.9.19",
+    "d3": "^7.8.5",
+    "layercake": "^8.0.2",
+    "lodash.debounce": "^4.0.8",
+    "lucide-svelte": "^0.294.0",
+    "postcss": "^8.4.33",
+    "prettier": "^3.2.4",
+    "prettier-plugin-svelte": "^3.1.2",
+    "rollup": "^4.9.5",
+    "rollup-plugin-svg": "^2.0.0",
+    "style-dictionary": "^3.9.2",
+    "svelte": "^4.2.9",
+    "svelte-copy": "^1.4.1",
+    "svelte-preprocess": "^5.1.3",
+    "vite": "^5.0.11",
+    "vite-plugin-svgstring": "^1.0.0"
+  },
+  "type": "module",
+  "engines": {
+    "node": ">= 18.12.0"
+  },
+  "browserslist": "> 0.5%, last 4 versions, not ie <= 11, not ie_mob <= 11",
+  "dependencies": {
+    "@leveluptuts/svelte-fit": "^1.0.3",
+    "asciinema-player": "^3.12.1",
+    "axios": "^1.6.7",
+    "d3-regression": "^1.3.10",
+    "fs": "^0.0.1-security",
+    "image-download": "^2.1.0",
+    "image-downloader": "^4.3.0",
+    "mathjax-full": "^3.2.2",
+    "openai": "^4.28.4",
+    "svelte-carousel": "^1.0.25",
+    "svelte-headless-table": "^0.18.3",
+    "svelte-highlight": "^7.9.0",
+    "svelte-lazy-image": "^0.5.1",
+    "svelte-multiselect": "^10.0.0",
+    "svelte-typeahead": "^5.0.0",
+    "wx-grid-locales": "^2.2.0",
+    "wx-svelte-grid": "^1.3.3",
+    "yootils": "^0.3.1"
+  }
 }

scripts/build-dashboard.mjs

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+/*
+ * Builds src/data/dashboard.json from api.formulacode.org/rest/v1.
+ *
+ * Replicates the Grafana "DataSmith Pipeline Overview" panels (Dataset
+ * Growth + Problems by Repository + Repository Distribution) using only
+ * the publicly readable Supabase tables — no auth required.
+ *
+ * Run manually with `npm run dashboard`. The output is committed so the
+ * landing page stays self-contained in case the API is down at deploy.
+ */
+
+import { writeFile, mkdir } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const API = "https://api.formulacode.org/rest/v1";
+const OUT = resolve(
+	dirname(fileURLToPath(import.meta.url)),
+	"../src/data/dashboard.json"
+);
+
+async function getCount(path) {
+	const url = `${API}${path}${path.includes("?") ? "&" : "?"}select=*&limit=1`;
+	const r = await fetch(url, {
+		headers: { Range: "0-0", "Range-Unit": "items", Prefer: "count=exact" }
+	});
+	if (!r.ok) throw new Error(`count ${r.status} ${url}`);
+	const cr = r.headers.get("content-range");
+	const total = cr?.split("/")[1];
+	return total === "*" ? null : Number(total);
+}
+
+async function fetchAll(path) {
+	const out = [];
+	const pageSize = 1000;
+	let offset = 0;
+	while (true) {
+		const sep = path.includes("?") ? "&" : "?";
+		const url = `${API}${path}${sep}limit=${pageSize}&offset=${offset}`;
+		const r = await fetch(url);
+		if (!r.ok) throw new Error(`fetch ${r.status} ${url}`);
+		const rows = await r.json();
+		out.push(...rows);
+		if (rows.length < pageSize) break;
+		offset += pageSize;
+	}
+	return out;
+}
+
+function groupBy(rows, keyFn) {
+	const m = new Map();
+	for (const r of rows) {
+		const k = keyFn(r);
+		if (!m.has(k)) m.set(k, []);
+		m.get(k).push(r);
+	}
+	return m;
+}
+
+async function main() {
+	const t0 = Date.now();
+	console.log(`fetching from ${API} ...`);
+
+	// Counts (cheap — HEAD with count=exact).
+	const [total_prs, total_problems, total_repos, perf_prs] = await Promise.all([
+		getCount("/pull_requests"),
+		getCount("/candidate_containers"),
+		getCount("/repositories"),
+		getCount("/pull_requests?is_performance_commit=eq.true")
+	]);
+	console.log(
+		`  totals: prs=${total_prs} problems=${total_problems} repos=${total_repos} perf=${perf_prs}`
+	);
+
+	// Full rows for joining.
+	const [containers, repositories] = await Promise.all([
+		fetchAll("/candidate_containers?select=owner,repo,issue_number"),
+		fetchAll("/repositories?select=owner,repo,stars")
+	]);
+	console.log(
+		`  fetched: ${containers.length} containers, ${repositories.length} repos`
+	);
+
+	// For the monthly histogram we need merged_at on each candidate's PR.
+	// candidate_containers has no FK to pull_requests in the schema cache,
+	// so do the join ourselves: batch issue_number lookups per (owner, repo).
+	const byRepo = groupBy(containers, (c) => `${c.owner}/${c.repo}`);
+	const merged = [];
+	let queries = 0;
+	for (const [, group] of byRepo) {
+		const { owner, repo } = group[0];
+		const issues = group.map((g) => g.issue_number);
+		for (let i = 0; i < issues.length; i += 200) {
+			const chunk = issues.slice(i, i + 200);
+			const url =
+				`${API}/pull_requests?owner=eq.${encodeURIComponent(owner)}` +
+				`&repo=eq.${encodeURIComponent(repo)}` +
+				`&issue_number=in.(${chunk.join(",")})` +
+				`&select=issue_number,merged_at`;
+			const r = await fetch(url);
+			if (!r.ok) throw new Error(`pr lookup ${r.status} ${url}`);
+			const rows = await r.json();
+			for (const row of rows) merged.push({ owner, repo, ...row });
+			queries += 1;
+		}
+	}
+	console.log(`  joined ${merged.length} PRs in ${queries} queries`);
+
+	// Aggregations.
+	const monthly = new Map();
+	for (const m of merged) {
+		if (!m.merged_at) continue;
+		const month = m.merged_at.slice(0, 7); // "YYYY-MM"
+		monthly.set(month, (monthly.get(month) || 0) + 1);
+	}
+
+	const byRepoCount = new Map();
+	for (const c of containers) {
+		const k = `${c.owner}/${c.repo}`;
+		byRepoCount.set(k, (byRepoCount.get(k) || 0) + 1);
+	}
+
+	// Output shape — matches what the Svelte components consume.
+	const out = {
+		generated_at: new Date().toISOString(),
+		source: API,
+		totals: {
+			pull_requests: total_prs,
+			problems: total_problems,
+			repositories: total_repos,
+			performance_prs: perf_prs,
+			pr_to_problem_rate: total_prs ? total_problems / total_prs : 0
+		},
+		monthly: [...monthly.entries()]
+			.sort(([a], [b]) => a.localeCompare(b))
+			.map(([month, count]) => ({ month, count })),
+		by_repository: [...byRepoCount.entries()]
+			.sort((a, b) => b[1] - a[1])
+			.map(([repository, count]) => ({ repository, count })),
+		repository_stars: repositories
+			.filter((r) => r.stars != null)
+			.sort((a, b) => b.stars - a.stars)
+			.map((r) => ({ repository: `${r.owner}/${r.repo}`, stars: r.stars }))
+	};
+
+	await mkdir(dirname(OUT), { recursive: true });
+	await writeFile(OUT, JSON.stringify(out, null, 2) + "\n");
+	const dt = ((Date.now() - t0) / 1000).toFixed(1);
+	console.log(`wrote ${OUT} in ${dt}s`);
+}
+
+main().catch((err) => {
+	console.error(err);
+	process.exit(1);
+});

src/components/pages/DocsPage.svelte

@@ -1,7 +1,6 @@
 <script>
 	const repos = [
 		{
-			path: "formula-code/fc-eval",
 			name: "fc-eval",
 			description:
 				"Run frontier LLM agents against the FormulaCode benchmark. Spins up reproducible Docker environments, replays the unit-test suite, and computes per-workload speedup, advantage, and stratified scores. Bring your own Terminus or OpenHands agent — `fceval run -d formulacode -a <your-agent>` is all it takes.",
@@ -11,7 +10,6 @@
 			imageAlt: "fc-eval repository preview"
 		},
 		{
-			path: "formula-code/datasmith",
 			name: "datasmith",
 			description:
 				"The four-stage pipeline that mines FormulaCode's tasks from real GitHub repositories: scraping high-quality performance PRs, attribute-filtering them with LLM judges, synthesising reproducible build environments, and running the statistical-significance tests that admit a candidate into the benchmark.",
@@ -71,9 +69,6 @@
 						href={r.docsUrl}
 						aria-label="Open {r.name} documentation"
 					>
-						<div class="card-header">
-							<span class="repo-path">{r.path}</span>
-						</div>
 						<div class="card-image">
 							<img src={r.image} alt={r.imageAlt} loading="lazy" />
 						</div>
@@ -229,21 +224,6 @@
 		color: inherit;
 	}
 
-	.card-header {
-		display: flex;
-		justify-content: space-between;
-		align-items: baseline;
-		gap: var(--space-sm);
-		padding: var(--space-md) var(--space-lg) 0;
-		flex-wrap: wrap;
-	}
-
-	.repo-path {
-		font-family: var(--mono);
-		font-size: 0.78rem;
-		color: var(--text-muted);
-	}
-
 	.card-image {
 		padding: var(--space-md) var(--space-lg);
 	}

src/components/pages/OverviewPage.svelte

@@ -2,6 +2,8 @@
 	import { getContext } from "svelte";
 	import OverviewHeader from "$components/layout/OverviewHeader.svelte";
 	import CompactLeaderboard from "$components/sections/CompactLeaderboard.svelte";
+	import DatasetGrowth from "$components/sections/DatasetGrowth.svelte";
+	import ProblemsByRepository from "$components/sections/ProblemsByRepository.svelte";
 	import ResultsHighlights from "$components/sections/ResultsHighlights.svelte";
 	import SubmitCta from "$components/sections/SubmitCta.svelte";
 
@@ -83,6 +85,9 @@
 		</section>
 	</main>
 
+	<DatasetGrowth />
+	<ProblemsByRepository />
+
 	<ResultsHighlights stratified={leaderboardData.stratified} />
 
 	<CompactLeaderboard