From 533a8fb0a2cb591c49a1ce628aeb949bd5e6f22a Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 09:18:40 +0000 Subject: [PATCH 01/11] Add WebVoyager adapter for the Kernel env + cua agent Generates WebVoyager's 643 live-web tasks (15 sites) as Harbor task dirs that run on the Kernel environment via the shared cua_harbor agent. Each record becomes instruction.md + environment/kernel.json (start_url + stealth + 1280x1024) + a per-task ground_truth.json; the dataset is vendored and pinned to upstream commit 0915445 for hermetic generation. The verifier ports WebVoyager's single multimodal judge (SYSTEM_PROMPT verbatim from upstream auto_eval.py) to the Anthropic Messages API: it reads /logs/agent/answer.txt + the last-k /logs/agent/shots/shot-.png the agent spilled and writes a 0/1 reward (SUCCESS/NOT SUCCESS, ambiguous fails closed). Site names with spaces are slugified so [task].name matches ORG_NAME_PATTERN, and reference answers with stray control chars are escaped for valid TOML. Generated task dirs and caches are gitignored. Mocked unit tests + ruff green. --- benchmarks/adapters/webvoyager/.gitignore | 9 + benchmarks/adapters/webvoyager/README.md | 66 + benchmarks/adapters/webvoyager/SMOKE.md | 94 + .../adapters/webvoyager/adapter_metadata.json | 22 + benchmarks/adapters/webvoyager/pyproject.toml | 22 + .../adapters/webvoyager/run_webvoyager.yaml | 23 + .../webvoyager/src/webvoyager/__init__.py | 5 + .../webvoyager/src/webvoyager/adapter.py | 226 ++ .../src/webvoyager/data/WebVoyager_data.jsonl | 643 ++++ .../src/webvoyager/data/reference_answer.json | 3292 +++++++++++++++++ .../webvoyager/src/webvoyager/main.py | 76 + .../task-template/environment/kernel.json | 5 + .../webvoyager/task-template/instruction.md | 9 + .../task-template/solution/solve.sh | 8 + .../src/webvoyager/task-template/task.toml | 32 + .../webvoyager/task-template/tests/test.sh | 5 + .../task-template/tests/webjudge.py | 128 + .../adapters/webvoyager/tests/test_adapter.py | 151 + .../webvoyager/tests/test_webjudge.py | 126 + benchmarks/adapters/webvoyager/uv.lock | 108 + 20 files changed, 5050 insertions(+) create mode 100644 benchmarks/adapters/webvoyager/.gitignore create mode 100644 benchmarks/adapters/webvoyager/README.md create mode 100644 benchmarks/adapters/webvoyager/SMOKE.md create mode 100644 benchmarks/adapters/webvoyager/adapter_metadata.json create mode 100644 benchmarks/adapters/webvoyager/pyproject.toml create mode 100644 benchmarks/adapters/webvoyager/run_webvoyager.yaml create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/__init__.py create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/adapter.py create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/data/WebVoyager_data.jsonl create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/data/reference_answer.json create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/main.py create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/task-template/environment/kernel.json create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/task-template/instruction.md create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/task-template/solution/solve.sh create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh create mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py create mode 100644 benchmarks/adapters/webvoyager/tests/test_adapter.py create mode 100644 benchmarks/adapters/webvoyager/tests/test_webjudge.py create mode 100644 benchmarks/adapters/webvoyager/uv.lock diff --git a/benchmarks/adapters/webvoyager/.gitignore b/benchmarks/adapters/webvoyager/.gitignore new file mode 100644 index 0000000..8a7d7a5 --- /dev/null +++ b/benchmarks/adapters/webvoyager/.gitignore @@ -0,0 +1,9 @@ +.tasks/ +jobs/ +trials/ +__pycache__/ +*.egg-info/ +.venv/ +.cache/ +.pytest_cache/ +.ruff_cache/ diff --git a/benchmarks/adapters/webvoyager/README.md b/benchmarks/adapters/webvoyager/README.md new file mode 100644 index 0000000..1efd5c9 --- /dev/null +++ b/benchmarks/adapters/webvoyager/README.md @@ -0,0 +1,66 @@ +# WebVoyager adapter + +Generates [WebVoyager](https://github.com/MinorJerry/WebVoyager) (643 live-web tasks, +15 sites) as Harbor task dirs that run on the Kernel environment with the cua agent +(`cua_harbor:CuaHarborAgent`). + +## What it produces + +One task dir per WebVoyager record: + +``` +webvoyager---/ +├── instruction.md # the ques + a "state the answer in your last line" directive +├── environment/kernel.json # { start_url: , stealth: true, viewport: 1280x1024 } +├── tests/ +│ ├── test.sh # pip install anthropic; python webjudge.py +│ ├── webjudge.py # WebVoyager single-call multimodal judge (Anthropic port) +│ └── ground_truth.json # { task, web_name, start_url, reference_answer, reference_type } +├── solution/solve.sh # oracle plumbing: writes the reference answer to answer.txt +└── task.toml # name, timeouts, [verifier.env] judge config +``` + +The dataset is vendored under `src/webvoyager/data/` (pinned to upstream commit +`0915445`, see `adapter_metadata.json`) so generation is hermetic. `--refresh` +re-fetches it from upstream. + +## Generate + +```bash +cd benchmarks +uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks # all 643 +uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --limit 20 # first 20 +uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --task-ids Allrecipes--0 Amazon--3 +``` + +`.tasks/` is gitignored. + +## Run (Kernel env + cua) + +```bash +cd benchmarks && uv sync && (cd node && npm install && npm run build) +uv run harbor run \ + -p adapters/webvoyager/.tasks \ + -e kernel --environment-kwarg pool_size=8 \ + --agent-import-path cua_harbor:CuaHarborAgent \ + -m anthropic/claude-sonnet-4-6 \ + --ae ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -n 6 +``` + +## Verifier + +WebVoyager's grader is a single multimodal call (task + last-k screenshots + answer -> +`SUCCESS`/`NOT SUCCESS`), ported to the Anthropic Messages API in `webjudge.py` +(SYSTEM_PROMPT is verbatim from upstream `evaluation/auto_eval.py`). It reads the agent +answer from `/logs/agent/answer.txt` and the spilled screenshots from +`/logs/agent/shots/shot-.png` (both written by the shared cua agent / Node entrypoint), +and writes a single `0|1` reward to `/logs/verifier/reward.txt` plus a `grading_details.json`. + +`[verifier.env]` knobs (per task, overridable from the host): + +| var | default | meaning | +|---|---|---| +| `ANTHROPIC_API_KEY` | `${ANTHROPIC_API_KEY}` | resolved from the host at exec time | +| `JUDGE_MODEL` | `claude-sonnet-4-5` | vision judge model (`WEBVOYAGER_JUDGE_MODEL` to override) | +| `MAX_IMAGES` | `3` | last-k screenshots sent to the judge (`WEBVOYAGER_MAX_IMAGES`; set `1` to match the paper) | diff --git a/benchmarks/adapters/webvoyager/SMOKE.md b/benchmarks/adapters/webvoyager/SMOKE.md new file mode 100644 index 0000000..b6d3146 --- /dev/null +++ b/benchmarks/adapters/webvoyager/SMOKE.md @@ -0,0 +1,94 @@ +# WebVoyager adapter — smoke / validation notes + +## Live smoke status: NOT RUN (deferred) + +The live 20-task Kernel smoke was intentionally **not** run for this change (the task +scoped to build + lint + mocked unit tests only; the parent runs the live smoke after +review). The pipeline is wired and ready. To run it: + +```bash +cd benchmarks && uv sync && (cd node && npm install && npm run build) +uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --limit 20 +uv run harbor run \ + -p adapters/webvoyager/.tasks \ + -e kernel --environment-kwarg pool_size=8 \ + --agent-import-path cua_harbor:CuaHarborAgent \ + -m anthropic/claude-sonnet-4-6 \ + --ae ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -n 6 +``` + +Validate 1–2 tasks green end to end (browser provisions, agent drives, `answer.txt` + +`shots/*.png` land, judge scores) before spending the full 20. If pool create/acquire +403s (quota/plan), drop `--environment-kwarg pool_size=8` to fall back to per-task create. + +## Build-time validation that WAS done + +Generation is the deterministic, network-free half of this adapter and was exercised in +full (no live browser/judge needed): + +- **Generated all 643 task dirs** from the vendored dataset (`uv run python -m + webvoyager.main --output-dir .tasks`): 643 dirs, 643 unique registry names. +- **Every `task.toml` loads through harbor's real `TaskConfig.model_validate_toml`** + (`name`, `[verifier.env]`, timeouts all parse). +- **Every `task.toml` `[task].name` matches `ORG_NAME_PATTERN`** and contains no `..`. +- **Every `environment/kernel.json` parses** with an `http(s)` `start_url` and the pinned + `viewport 1280x1024` / `stealth:true`. +- **Every `tests/ground_truth.json` parses** and carries a non-empty `task`. +- **No `{placeholder}` leaks** into any generated `instruction.md` / `task.toml` / + `kernel.json` / `solve.sh` / `ground_truth.json`. +- **Generated `test.sh` / `solve.sh` pass `bash -n`** (incl. the form-feed edge case below). +- **Mocked unit tests green**: `uv run pytest` (23 tests) covers dataset load, id + slugification, reference indexing, template substitution, TOML/control-char escaping, + the judge's numeric shot sort + last-k selection + `SUCCESS`/`NOT SUCCESS` verdict parse + (Anthropic client stubbed). `uv run ruff check` clean. + +## Bugs found and fixed during validation + +1. **Multi-word site names broke registry validation.** `ORG_NAME_PATTERN` + (`harbor/src/harbor/constants.py`) is + `^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*$` — it **forbids spaces**. + Upstream ids like `Google Flights--0` / `BBC News--3` / `Wolfram Alpha--45` would have + produced names with spaces and failed registration for ~250 tasks across 5 sites + (Google Flights, Google Map, BBC News, Wolfram Alpha, Cambridge Dictionary). The design + doc's claim that "double-dash and single-dash are fine" missed the space. Fixed by + slugifying the site portion (`normalize_id`: lowercase + spaces→`-`) in both the dir + name and the `task.toml` name. The `--` suffix is fine (no `..`). + +2. **A reference answer contained a raw form-feed.** `Wolfram Alpha--39`'s + `reference_answer` has a literal `\x0c` (a broken LaTeX `\frac` where `\f` became a + form-feed). TOML basic strings forbid raw control chars, so the naive + `replace('"').replace('\\')` escape produced an invalid `task.toml`. Fixed `_toml_escape` + to emit `\uXXXX` for every control char and collapse newlines to spaces. + +## Deviations from the design doc (`map-webvoyager.md`) + +- **Judge provider: Anthropic, not OpenAI.** `OPENAI_API_KEY` is absent this session and + the shared core standardizes on Anthropic for the live-web judges. `tests/webjudge.py` + ports WebVoyager's single multimodal call to the Anthropic Messages API (SYSTEM_PROMPT is + verbatim from upstream `evaluation/auto_eval.py`; last-k screenshots + verdict parse + unchanged). `[verifier.env]` carries `ANTHROPIC_API_KEY` + `JUDGE_MODEL` + (`claude-sonnet-4-5`) instead of `OPENAI_API_KEY` + `gpt-4o`. +- **Reuse the shared `cua_harbor` agent + Node entrypoint** (not the `simpleqa`/`-a cua` + layout in the doc). The agent and the screenshot/answer spill already exist on the + shared-core branch; the adapter only emits task dirs + the judge. +- **Answer/screenshot paths follow the shared core**: `/logs/agent/answer.txt` and + `/logs/agent/shots/shot-.png`. The shots are **not** zero-padded, so the judge sorts + numerically (not lexically) to take the last k. +- **No `captureScreenshot` re-screenshot fallback in the verifier.** Keeping the verifier + to a single `pip install anthropic` (the doc's own stated rationale, §2a) is simpler than + pulling the Kernel SDK into the grade path. The shared-core sink reliably spills a shot + per step; if both answer and shots are empty the judge fails closed to 0. +- **Dataset vendored + pinned** under `src/webvoyager/data/` (commit `0915445`, checksums in + `adapter_metadata.json`) for hermetic generation; `--refresh` re-fetches from upstream. + +## Expected live failure taxonomy (from the design doc, to watch for) + +- **env-vs-task ambiguity**: a captcha/login wall scores `NOT SUCCESS`, indistinguishable + from a real miss in a 0/1 reward. `grading_details.json` records the raw verdict for + post-hoc triage. +- **site drift / anti-bot**: Amazon, Booking, Google properties are the block surface; + `stealth:true` is on for every task. An operator may add a residential `proxy_id` / + `profile` at the env level. +- **judge disagreement**: inherent to an LLM judge; `temperature=0` + pinned `JUDGE_MODEL`. +- **adapter bug**: none expected post-fix; generation validated on all 643. diff --git a/benchmarks/adapters/webvoyager/adapter_metadata.json b/benchmarks/adapters/webvoyager/adapter_metadata.json new file mode 100644 index 0000000..58cfcc5 --- /dev/null +++ b/benchmarks/adapters/webvoyager/adapter_metadata.json @@ -0,0 +1,22 @@ +{ + "name": "webvoyager", + "source": "https://github.com/MinorJerry/WebVoyager", + "paper": "https://arxiv.org/abs/2401.13919", + "task_count": 643, + "sites": 15, + "dataset": { + "file": "data/WebVoyager_data.jsonl", + "reference_file": "data/reference_answer.json", + "upstream_commit": "091544539eba485dbd74ef3742011ddeede37336", + "sha256": { + "WebVoyager_data.jsonl": "69b19fd86c23f1a500244a3724e039aa7ca6a1223d03e11eb10e308d4f11c488", + "reference_answer.json": "5473818408d8a19960c4f820c1e5721d218b87b547499d7281c2a0fc4267b0c4" + } + }, + "verifier": { + "type": "multimodal-llm-judge", + "port_of": "WebVoyager evaluation/auto_eval.py (single-call GPT-4V judge)", + "provider": "anthropic", + "default_model": "claude-sonnet-4-5" + } +} diff --git a/benchmarks/adapters/webvoyager/pyproject.toml b/benchmarks/adapters/webvoyager/pyproject.toml new file mode 100644 index 0000000..7d6014b --- /dev/null +++ b/benchmarks/adapters/webvoyager/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "webvoyager-adapter" +version = "0.1.0" +description = "WebVoyager -> Harbor task generator for the Kernel env + cua agent" +requires-python = ">=3.12" +dependencies = [] + +[dependency-groups] +dev = ["pytest>=8", "ruff>=0.6"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/webvoyager"] + +[tool.ruff] +target-version = "py312" + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/benchmarks/adapters/webvoyager/run_webvoyager.yaml b/benchmarks/adapters/webvoyager/run_webvoyager.yaml new file mode 100644 index 0000000..9bdb5bd --- /dev/null +++ b/benchmarks/adapters/webvoyager/run_webvoyager.yaml @@ -0,0 +1,23 @@ +# Reference config for a WebVoyager run on the Kernel env with the cua agent. +# +# cd benchmarks +# uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --limit 20 +# uv run harbor run \ +# -p adapters/webvoyager/.tasks \ +# -e kernel --environment-kwarg pool_size=8 \ +# --agent-import-path cua_harbor:CuaHarborAgent \ +# -m anthropic/claude-sonnet-4-6 \ +# --ae ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ +# -n 6 +# +# The judge model is pinned via [verifier.env] in each task.toml +# (JUDGE_MODEL=claude-sonnet-4-5, override with WEBVOYAGER_JUDGE_MODEL); the verifier +# reads ANTHROPIC_API_KEY from the host. MAX_IMAGES (last-k screenshots sent to the +# judge) defaults to 3; override with WEBVOYAGER_MAX_IMAGES=1 to match the paper. +tasks_path: adapters/webvoyager/.tasks +environment: kernel +environment_kwargs: + pool_size: 8 +agent_import_path: cua_harbor:CuaHarborAgent +model: anthropic/claude-sonnet-4-6 +n_concurrent: 6 diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/__init__.py b/benchmarks/adapters/webvoyager/src/webvoyager/__init__.py new file mode 100644 index 0000000..4d5b051 --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/__init__.py @@ -0,0 +1,5 @@ +"""WebVoyager -> Harbor task generator for the Kernel env + cua agent.""" + +from webvoyager.adapter import WebVoyagerAdapter + +__all__ = ["WebVoyagerAdapter"] diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py b/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py new file mode 100644 index 0000000..548104f --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py @@ -0,0 +1,226 @@ +"""WebVoyager adapter: dataset records -> Harbor task dirs for the Kernel env. + +Each upstream record (``web_name``/``id``/``ques``/``web``) becomes one task dir: +``instruction.md`` (the ``ques`` plus an answer directive), ``environment/kernel.json`` +(``start_url`` = ``web`` + stealth + viewport), ``tests/`` (the ported WebVoyager +multimodal judge + a per-task ``ground_truth.json``), ``solution/solve.sh`` (writes the +reference answer for an oracle plumbing run), and ``task.toml``. + +The dataset is vendored under ``data/`` so generation is hermetic and pinned to the +upstream commit recorded in ``adapter_metadata.json``; ``--refresh`` re-fetches it. +""" + +from __future__ import annotations + +import json +import logging +import re +import shutil +import urllib.request +from pathlib import Path + +PACKAGE_DIR = Path(__file__).parent +TEMPLATE_DIR = PACKAGE_DIR / "task-template" +DATA_DIR = PACKAGE_DIR / "data" +DATASET_FILE = DATA_DIR / "WebVoyager_data.jsonl" +REFERENCE_FILE = DATA_DIR / "reference_answer.json" + +RAW_BASE = "https://raw.githubusercontent.com/MinorJerry/WebVoyager/main/data" +DATASET_URL = f"{RAW_BASE}/WebVoyager_data.jsonl" +REFERENCE_URL = f"{RAW_BASE}/reference_answer.json" + +# Upstream `id` is "--" (e.g. "Allrecipes--0"); n is the index used to key +# reference_answer.json (which lists answers per web_name with integer ids). +_ID_SUFFIX = re.compile(r"--(\d+)$") + +logger = logging.getLogger(__name__) + + +class WebVoyagerTask: + """One WebVoyager record plus its (optional) human reference answer.""" + + def __init__(self, record: dict, reference: dict | None): + self.web_name: str = record["web_name"] + self.source_id: str = record["id"] + self.ques: str = record["ques"] + self.start_url: str = record["web"] + self.reference_ans: str = (reference or {}).get("ans", "") + self.reference_type: str = (reference or {}).get("type", "") + + @property + def web_name_slug(self) -> str: + return self.web_name.lower().replace(" ", "-") + + +def _load_jsonl(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def _index_reference(reference: dict) -> dict[tuple[str, int], dict]: + """Map ``(web_name, n)`` -> ``{type, ans, ...}`` from reference_answer.json.""" + index: dict[tuple[str, int], dict] = {} + for web_name, payload in reference.items(): + for answer in payload.get("answers", []): + index[(web_name, int(answer["id"]))] = answer + return index + + +class WebVoyagerAdapter: + """Converts the vendored WebVoyager dataset into Harbor task directories.""" + + NAME = "webvoyager" + + @staticmethod + def normalize_id(source_id: str) -> str: + """``Google Flights--7`` -> ``google-flights--7``. + + Lowercase and replace spaces with ``-`` so the result is a valid Harbor + registry name segment (``ORG_NAME_PATTERN`` forbids spaces; multi-word sites + like Google Flights / BBC News would otherwise fail registration). + """ + return source_id.lower().replace(" ", "-") + + @classmethod + def make_local_task_id(cls, source_id: str) -> str: + """``Google Flights--7`` -> ``webvoyager-google-flights--7`` (dir name).""" + return f"webvoyager-{cls.normalize_id(source_id)}" + + def __init__( + self, + output_dir: Path, + limit: int | None = None, + overwrite: bool = False, + task_ids: list[str] | None = None, + refresh: bool = False, + **kwargs: object, + ): + self.output_dir = Path(output_dir) + self.limit = limit + self.overwrite = overwrite + self.task_ids = task_ids + + if refresh: + self._refresh_dataset() + + records = _load_jsonl(DATASET_FILE) + reference = json.loads(REFERENCE_FILE.read_text()) + ref_index = _index_reference(reference) + + self.tasks: list[WebVoyagerTask] = [] + for record in records: + suffix = _ID_SUFFIX.search(record["id"]) + key = (record["web_name"], int(suffix.group(1))) if suffix else None + self.tasks.append(WebVoyagerTask(record, ref_index.get(key))) + + def _refresh_dataset(self) -> None: + DATA_DIR.mkdir(parents=True, exist_ok=True) + for url, dest in ((DATASET_URL, DATASET_FILE), (REFERENCE_URL, REFERENCE_FILE)): + logger.info(f"Refreshing {dest.name} from {url}") + with urllib.request.urlopen(url) as resp: # noqa: S310 (pinned upstream host) + dest.write_bytes(resp.read()) + + def _select(self) -> list[WebVoyagerTask]: + tasks = self.tasks + if self.task_ids is not None: + requested = set(self.task_ids) + tasks = [ + task + for task in tasks + if task.source_id in requested + or self.make_local_task_id(task.source_id) in requested + ] + if self.limit is not None: + tasks = tasks[: self.limit] + return tasks + + def _prepare_task(self, task: WebVoyagerTask, task_dir: Path) -> None: + task_dir.mkdir(parents=True, exist_ok=True) + local_id = self.make_local_task_id(task.source_id) + + instruction = (TEMPLATE_DIR / "instruction.md").read_text() + instruction = instruction.replace("{start_url}", task.start_url) + instruction = instruction.replace("{web_name}", task.web_name) + instruction = instruction.replace("{ques}", task.ques) + (task_dir / "instruction.md").write_text(instruction) + + env_dir = task_dir / "environment" + env_dir.mkdir(exist_ok=True) + kernel_json = (TEMPLATE_DIR / "environment/kernel.json").read_text() + kernel_json = kernel_json.replace("{start_url}", task.start_url) + (env_dir / "kernel.json").write_text(kernel_json) + + tests_dir = task_dir / "tests" + tests_dir.mkdir(exist_ok=True) + shutil.copy2(TEMPLATE_DIR / "tests/test.sh", tests_dir / "test.sh") + shutil.copy2(TEMPLATE_DIR / "tests/webjudge.py", tests_dir / "webjudge.py") + (tests_dir / "ground_truth.json").write_text( + json.dumps( + { + "task": task.ques, + "web_name": task.web_name, + "start_url": task.start_url, + "reference_answer": task.reference_ans, + "reference_type": task.reference_type, + }, + indent=2, + ) + ) + + solution_dir = task_dir / "solution" + solution_dir.mkdir(exist_ok=True) + solution = (TEMPLATE_DIR / "solution/solve.sh").read_text() + solution = solution.replace("{reference_answer}", _shell_single_quote(task.reference_ans)) + (solution_dir / "solve.sh").write_text(solution) + + task_toml = (TEMPLATE_DIR / "task.toml").read_text() + task_toml = task_toml.replace("{task_id}", self.normalize_id(task.source_id)) + task_toml = task_toml.replace("{web_name_slug}", task.web_name_slug) + task_toml = task_toml.replace("{web_name}", task.web_name) + task_toml = task_toml.replace("{start_url}", task.start_url) + task_toml = task_toml.replace("{reference_answer}", _toml_escape(task.reference_ans)) + task_toml = task_toml.replace("{reference_type}", task.reference_type) + (task_dir / "task.toml").write_text(task_toml) + + logger.debug(f"Wrote task {local_id}") + + def run(self) -> None: + selected = self._select() + self.output_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"Generating {len(selected)} WebVoyager tasks -> {self.output_dir}") + + for i, task in enumerate(selected): + task_dir = self.output_dir / self.make_local_task_id(task.source_id) + if task_dir.exists(): + if not self.overwrite: + continue + shutil.rmtree(task_dir) + self._prepare_task(task, task_dir) + if (i + 1) % 100 == 0 or i == 0: + logger.info(f"Progress: {i + 1}/{len(selected)}") + + +def _shell_single_quote(value: str) -> str: + """Make ``value`` safe inside a single-quoted shell string.""" + return value.replace("'", "'\\''") + + +def _toml_escape(value: str) -> str: + """Escape a value for a single-line TOML basic (double-quoted) string. + + Backslash and quote are escaped; every control char (TOML basic strings forbid + raw control chars, including the stray form-feed in one reference answer) becomes a + ``\\uXXXX`` escape, with newlines collapsed to spaces so the value stays single-line. + """ + out = [] + for ch in value: + if ch == "\\": + out.append("\\\\") + elif ch == '"': + out.append('\\"') + elif ch in "\n\r": + out.append(" ") + elif ch < "\x20" or ch == "\x7f": + out.append(f"\\u{ord(ch):04x}") + else: + out.append(ch) + return "".join(out) diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/data/WebVoyager_data.jsonl b/benchmarks/adapters/webvoyager/src/webvoyager/data/WebVoyager_data.jsonl new file mode 100644 index 0000000..7ba2ebd --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/data/WebVoyager_data.jsonl @@ -0,0 +1,643 @@ +{"web_name": "Allrecipes", "id": "Allrecipes--0", "ques": "Provide a recipe for vegetarian lasagna with more than 100 reviews and a rating of at least 4.5 stars suitable for 6 people.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--1", "ques": "Find a recipe for a vegetarian lasagna that has at least a four-star rating and uses zucchini.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--2", "ques": "Find a recipe for a vegetarian lasagna under 600 calories per serving that has a prep time of less than 1 hour.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--3", "ques": "Locate a recipe for vegan chocolate chip cookies with over 60 reviews and a rating of at least 4.5 stars on Allrecipes.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--4", "ques": "Find a recipe for Baked Salmon that takes less than 30 minutes to prepare and has at least a 4 star rating based on user reviews.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--5", "ques": "Search for a popular Pasta Sauce with more than 1000 reviews and a rating above 4 stars. Create a shopping list of ingredients for this recipe.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--6", "ques": "Search for a vegetarian lasagna recipe that has at least a four-star rating and over 500 reviews.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--7", "ques": "Find a popular recipe for a chocolate chip cookie and list the ingredients and preparation steps.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--8", "ques": "Search for a recipe for Beef Wellington on Allrecipes that has at least 200 reviews and an average rating of 4.5 stars or higher. List the main ingredients required for the dish.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--9", "ques": "Find a high-rated recipe for vegetarian lasagna, list the key ingredients required, and include the total preparation and cook time stated on the recipe.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--10", "ques": "Find The Most Popular Recipes of the 1960s, noting the recipe name, preparation time and total time of the second recipe in this collection.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--11", "ques": "Discover a suitable chocolate cupcake recipe on Allrecipes that has a preparation time of under 1 hour and at least 100 user reviews.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--12", "ques": "Search for a popular cookie recipe on Allrecipes with more than 1000 reviews and a rating of 4.5 stars or better. Provide the list of ingredients needed.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--13", "ques": "Find a recipe with over 100 reviews for Fried Fish on Allrecipes, list the Full Nutrition Label and tell me the amount of Iron per Serving.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--14", "ques": "Search for a recipe that includes \"chicken breast\" and \"quinoa\" with preparation time under 30 minutes on Allrecipes.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--15", "ques": "Choose a dessert recipe on Allrecipes with a prep time of less than 30 minutes, has chocolate as an ingredient, and has a user rating of 4 stars or higher. Provide the name of the recipe, ingredients list, and step-by-step instructions.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--16", "ques": "Find a five-star rated chocolate chip cookie recipe that takes less than 1 hour to make on Allrecipes. Note how many reviews the recipe has and the main ingredients required.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--17", "ques": "Find the Easy Vegetarian Spinach Lasagna recipe on Allrecipes and tell me what the latest review says.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--18", "ques": "Find a recipe for a vegetarian lasagna that has over 300 reviews and an average rating of 4.5 or higher on Allrecipes.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--19", "ques": "Find a vegan lasagna recipe on Allrecipes that requires 10 ingredients or less and has feedback of more than 200 reviews. Provide a brief overview of the ingredient list and the total prep and cook time.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--20", "ques": "Find a recipe for a cauliflower pizza crust that has a preparation time of under 30 minutes and a rating of at least 4 stars on Allrecipes. Include the number of calories per serving.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--21", "ques": "Locate a high-rated recipe for gluten-free brownies on Allrecipes with at least 50 reviews. List the main ingredients and the total time required for preparation and cooking.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--22", "ques": "Find a recipe for a healthy avocado salad on Allrecipes that has a preparation time of less than 20 minutes and more than 30 user reviews. Include the nutritional information per serving.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--23", "ques": "Search Allrecipes for a baked lemon chicken recipe that has a prep time under 45 minutes, with at least a 4.5-star rating based on user reviews, and over 200 reviews. List the primary ingredients required.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--24", "ques": "Locate a recipe for an eggplant Parmesan on Allrecipes with a rating of at least 4.5 stars and over 50 reviews. Include the preparation time and the number of servings provided by the recipe.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--25", "ques": "Find a popular quinoa salad recipe on Allrecipes with more than 500 reviews and a rating above 4 stars. Create a shopping list of ingredients for this recipe and include the total cooking and preparation time.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--26", "ques": "Search for a high-protein vegetarian chili recipe on Allrecipes that has at least 50 reviews and a rating of 4 stars or higher. Provide the ingredient list, cooking time, and a brief description of the cooking steps.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--27", "ques": "Locate a chicken curry recipe on Allrecipes that has been reviewed more than 30 times and has a rating of at least 4 stars. Provide a summary of the recipe including ingredients, preparation time, and cooking instructions.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--28", "ques": "On Allrecipes, find a vegan brownie recipe that has at least 40 reviews and a rating of 4.5 or higher. Include the list of ingredients, total prep and cook time, and a brief overview of the preparation steps.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--29", "ques": "Search for a Mediterranean-style grilled fish recipe on Allrecipes that includes ingredients like olives, has at least a 4-star rating, and more than 25 reviews. Detail the ingredients, cooking method, and total time required for preparation and cooking.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--30", "ques": "Find a recipe for a vegan smoothie bowl on Allrecipes that includes bananas and leaves, has more than 20 reviews, and a rating of at least 4 stars. Provide a list of ingredients, preparation time, and a summary of the recipe steps.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--31", "ques": "Search for a seafood paella recipe on Allrecipes with a minimum of 4.5 stars rating and at least 50 reviews. The recipe should include shrimp and mussels. Provide the ingredients, total time, and an overview of the preparation steps.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--32", "ques": "Find a high-rated beef stew recipe on Allrecipes that requires a slow cooker and has at least 30 reviews. Detail the cooking time and the first five ingredients listed in the recipe.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--33", "ques": "Find a recipe for a low-carb breakfast on Allrecipes with at least 25 reviews. Show the Nutrition Facts and the total carbohydrate content per serving.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--34", "ques": "Locate a baked salmon recipe on Allrecipes that has at least 50 reviews and a rating of 4.5 stars or higher. Note the primary seasoning or herb used and the estimated cooking time.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--35", "ques": "Search for an Italian-style meatball recipe on Allrecipes that has more than 100 reviews. Detail the type of meat used and the overall cooking time required.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--36", "ques": "Locate a recipe for an American apple pie on Allrecipes with a rating of at least 4 stars and more than 50 reviews. Note the maximum temperature mentioned in the Directions.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--37", "ques": "Search for a Greek salad recipe on Allrecipes that has a prep time of under 25 minutes and more than 15 reviews. Include the primary cheese used and the type of dressing recommended.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--38", "ques": "Find a French ratatouille recipe on Allrecipes with a 4-star rating or higher and at least 15 reviews. Note the variety of vegetables included and the overall cooking time.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--39", "ques": "Locate a recipe for sushi rolls on Allrecipes with a minimum of 20 reviews. Show the Nutrition Facts and the main ingredients. Tell me how to store these rolls.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--40", "ques": "Browse the about us section of Allrecipes for a brief introduction to The Allrecipes Allstars.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--41", "ques": "List 3 recommended dinner recipes in the Allrecipes Dinners section.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--42", "ques": "Find a recipe for banana bread with more than 200 reviews and a rating of at least 4.0 stars on Allrecipes.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--43", "ques": "Find a recipe for a vegan pumpkin pie on Allrecipes with a minimum four-star rating and a total cook time exceeding 1 hour.", "web": "https://www.allrecipes.com/"} +{"web_name": "Allrecipes", "id": "Allrecipes--44", "ques": "List at least 6 holiday recipes sections mentioned in the Occasions section of Allrecipes.", "web": "https://www.allrecipes.com/"} +{"web_name": "Amazon", "id": "Amazon--0", "ques": "Search an Xbox Wireless controller with green color and rated above 4 stars.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--1", "ques": "Search for women's golf polos in m size, priced between 50 to 75 dollars, and save the lowest priced among results.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--2", "ques": "Find a gaming desktop with Windows 11 Home, and the disk size should be 1TB.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--3", "ques": "Find climbing gears and sort the results by price high to low. Answer the first 3 results after sorting.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--4", "ques": "Find the used Nintendo Switch Lite on Amazon then filter by 'Used - Good', tell me the cheapest one that is 'Used - Good'.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--5", "ques": "Find a Blue iPhone 12 Pro 128gb and add to cart.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--6", "ques": "Browse black strollers within $100 to $200 on Amazon. Then find one Among these black strollers with over 20,000 reviews and a rating greater than 4 star.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--7", "ques": "Browse the women's hiking boots on Amazon and filter the results to show only those that are waterproof and have a rating of at least 4 stars and size 6.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--8", "ques": "Find the cheapest Samsung-made Android tablet with screen between 10-10.9 inches on Amazon. Only answer the cheapest one.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--9", "ques": "Find a dog bed on Amazon that is washable and has a length of at least 30 inches.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--10", "ques": "Find the cost of a 2-year protection for PS4 on Amazon.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--11", "ques": "Find a stainless steel kitchen sink with double bowls on Amazon. Sort the results and find the cheapest one with FREE delivery.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--12", "ques": "Check reviews for a Ride On Car with 100+ reviews & 4+ stars rating on Amazon. Give me the top review about this Ride On Car.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--13", "ques": "Browse best selling black hoodies in mens size Big and Tall that is between $25 and $50 on Amazon.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--14", "ques": "Find the new surge protector on Amazon with 6 to 8 outlets under 25 dollars with customer reviews above 4+ stars.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--15", "ques": "Find a pair of mens running shoes in black, size 7, 4+ stars and under $50 and add them to my cart on Amazon.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--16", "ques": "Find the Return Policy for Mens Rhinestone Skull Graphic Shirt on Amazon. Color: Black, Size: XX-Large. If Free return is avaliable, tell me how to return this item.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--17", "ques": "Show me the list of baby products that are on sale and under 10 dollars on Amazon. Provide at least 2 on sale products", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--18", "ques": "Open Amazon's home page and tell me what the deal is that is going on at the moment, list the names of at least 2 items that are on offer and tell me what percent off they are.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--19", "ques": "Look for an English language book on roman empire history in the Amazon Kindle store. Sort by newests arrivals and look for a title that will be released within a month.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--20", "ques": "Search for a wireless ergonomic keyboard with backlighting and a rating of at least 4 stars. The price should be between $40 to $60. Save the product with the 500+ customer reviews.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--21", "ques": "Find a stainless steel, 12-cup programmable coffee maker on Amazon. The price range should be between $100 to $200. Report the one with the 4+ customer rating.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--22", "ques": "Search for a set of non-stick, oven-safe cookware on Amazon. The set should include at least 10 pieces and be priced under $150.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--23", "ques": "Look for a men's waterproof digital sports watch with a heart rate monitor on Amazon. It should be priced between $50 to $100.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--24", "ques": "Browse for a compact air fryer on Amazon with a capacity of 2 to 3 quarts. It should have a digital display, auto shutoff and be priced under $100.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--25", "ques": "Search for a queen-sized, hypoallergenic mattress topper on Amazon. It should have a memory foam material and be priced between $50 to $100.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--26", "ques": "Find a portable Bluetooth speaker on Amazon with a water-resistant design, under $50. It should have a minimum battery life of 10 hours.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--27", "ques": "Look for a USB-C hub on Amazon compatible with MacBook Pro, featuring at least 4 ports, including HDMI and SD card reader. The price should be under $50. Select the one after sorting by Best Sellers.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--28", "ques": "Search for a yoga mat on Amazon that is at least 6mm thick, non-slip, and eco-friendly. The price should be under $50.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--29", "ques": "Find a set of solar-powered garden lights on Amazon with a minimum pack of 10 lights. They should be LED and priced under $50.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--30", "ques": "Locate the highest-rated fiction book released in 2024 on Amazon, with a minimum of 50 customer reviews.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--31", "ques": "Find a compact digital camera on Amazon with a zoom capability of at least 10x, rated 4 stars or higher, and priced between $100 to $300.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--32", "ques": "Search for an electric kettle on Amazon with a capacity of at least 1.5 liters, made of stainless steel, and with a customer rating of 4 stars or above.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--33", "ques": "Search for a portable air conditioner on Amazon suitable for a room size of 300 sq ft, with energy efficiency rating, and compare the prices of the top three search results.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--34", "ques": "Find a beginner's acrylic paint set on Amazon, with at least 24 colors, suitable for canvas painting, and priced under $40.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--35", "ques": "Find a men's leather wallet on Amazon with RFID blocking, at least 6 card slots, and priced below $50. Check if it's available for FREE delivery.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--36", "ques": "Search for a children's science experiment kit on Amazon suitable for ages 8-13, with at least a 4-star rating and priced under $30.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--37", "ques": "Locate a queen-sized bedspread on Amazon with a floral pattern, and check if it's available in blue color.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--38", "ques": "Find a bird feeder on Amazon suitable for small birds, with an anti-squirrel mechanism, and check if it's available with free shipping.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--39", "ques": "Locate a travel guide book on Amazon for Japan, published in 2024, with at least 20 customer reviews.", "web": "https://www.amazon.com/"} +{"web_name": "Amazon", "id": "Amazon--40", "ques": "Locate a women's yoga mat in purple, with a thickness of at least 5mm, rated 4+ stars, and priced under $30 on Amazon. Check how many colors are available in total, and what is the return and delivery policy.", "web": "https://www.amazon.com/"} +{"web_name": "Apple", "id": "Apple--0", "ques": "Compare the prices of the latest models of MacBook Air available on Apple's website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--1", "ques": "Research the new features of the iOS 17 on Apple support and check its compatibility with the iPhone 12.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--2", "ques": "Compare the prices and chips for the iPhone 14 Pro and iPhone 15 Pro models directly from Apple's website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--3", "ques": "Find the latest model of the iPhone and compare the price and screen size between the pro and pro max.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--4", "ques": "How much does it cost to buy a Macbook pro, 16-inch, Apple M3 Max chip with 16-core CPU, 40-core GPU, 64GB unified memory, 1TB SSD.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--5", "ques": "Check the release date and price for the latest version of the iPhone.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--6", "ques": "Find AirPods on Apple and how many types are currently available.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--7", "ques": "When and where the Apple Vision Pro will be released.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--8", "ques": "Identify and list the specifications of the latest iPad model released by Apple, including its storage options, processor type, and display features.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--9", "ques": "Check the Apple Store for the availability of the latest iPhone model and schedule an in-store pickup at the nearest Apple Store for January 10, 2024.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--10", "ques": "Find information on the latest (as of today's date) MacBook model, including its key features such as processor type, memory size, and storage capacity.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--11", "ques": "Get information about the latest iPad model released by Apple, including its release date, base storage capacity, and starting price available on Apple's official website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--12", "ques": "What Apple Repair ways are mentioned on apple website, answer 2 of them.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--13", "ques": "How many colors does the latest MacBook Air come in?", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--14", "ques": "Identify the upgrade options available for the cheapest base model of the MacBook Pro 14-inch with M3 chip, and calculate the total price difference from the base model to the maximum upgrade (no Pre-Installed Software) offered by Apple.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--15", "ques": "On Apple's website, how many different types of keyboards are available when customizing your 14-inch MacBook Pro?", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--16", "ques": "Find on Apple website how many types of AirPods (3rd generation) are available and what is the price difference.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--17", "ques": "Search Apple for the accessory Smart Folio for iPad and check the closest pickup availability next to zip code 90038.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--18", "ques": "Check if there are trade-in offers for the latest model of iPhone.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--19", "ques": "On Apple's website, what is the slogan for the Mac and what is the slogan for the Macbook pro.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--20", "ques": "Check the price for an Apple iPhone 14 Plus with 256GB storage in Purple color.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--21", "ques": "Identify the available storage options for the latest iPad Pro on the Apple website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--22", "ques": "Find out the trade-in value for an iPhone 13 Pro Max in good condition on the Apple website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--23", "ques": "Determine the price difference between the latest series of Apple Watch and Apple Watch SE on the Apple website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--24", "ques": "Find out the starting price for the most recent model of the iMac on the Apple website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--25", "ques": "On the Apple website, look up the processor for the latest model of the Apple TV.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--26", "ques": "Find the maximum video recording resolution supported by the latest iPad mini on the Apple website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--27", "ques": "On Apple's website, check if the HomePod mini in store is available in multiple colors and list them.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--28", "ques": "On the Apple website, find out if the Mac Mini can be configured with a GPU larger than 16-core.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--29", "ques": "On Apple's website, check the estimated battery life of the latest MacBook Air during web browsing in Tech Specs.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--30", "ques": "Check the storage options and prices for the latest iPad Pro models on Apple's website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--31", "ques": "On Apple's website, what is the slogan for the latest Apple Watch Series.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--32", "ques": "Investigate the trade-in value for an iPhone 11 Pro Max on Apple's website.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--33", "ques": "Look for the color options available for the newest iMac.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--34", "ques": "Identify the size and weight for the Apple TV 4K and list the Siri Remote features introduced.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--35", "ques": "How many types of Apple Pencil are currently available on the Apple's website? Which one supports Wireless pairing and charging.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--36", "ques": "Browse Apple Music on the entertainment section of the Apple's website, and see which singers' names are included in the pictures on this page.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--37", "ques": "Compare the color options of iPhone 13 Pro, iPhone 14 Pro and iPhone 15 Pro.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--38", "ques": "Explore accessories for Apple Vision Pro, list at least three accessories.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--39", "ques": "Find solutions on Apple's website if you forgot your Apple ID password.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--40", "ques": "Find information on Apple website, and tell me the device weight of Apple Vision Pro and list 5 Built-in Apps it supports.", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--41", "ques": "How much does it cost to buy an ipad mini with 64GB storage and Wi-Fi + Cellular connectivity? (no engraving, no apple pencil, no smart folio, no apple trade-in).", "web": "https://www.apple.com/"} +{"web_name": "Apple", "id": "Apple--42", "ques": "Find updates for Apple Watch Series 7,8,9 on Apple's website.", "web": "https://www.apple.com/"} +{"web_name": "ArXiv", "id": "ArXiv--0", "ques": "Search for the latest preprints about 'quantum computing'.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--1", "ques": "Search for the latest research papers on quantum computing submitted to ArXiv within the last two days.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--2", "ques": "Look up the most recent papers related to 'cs.CL', select one and show its abstract.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--3", "ques": "Locate the most recent research paper about 'Algebraic Topology' under Mathematics published on ArXiv. Provide the title of the paper, the name of the authors, and the abstract.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--4", "ques": "Find the most recent research papers in Astrophysics of Galaxies. How many papers have been announced in the last day?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--5", "ques": "Search papers about \"quantum computing\" which has been submitted to the Quantum Physics category on ArXiv. How many results in total. What if search in all archives?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--6", "ques": "How many figures and tables are in the paper \"On the Sentence Embeddings from Pre-trained Language Models\"?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--7", "ques": "Find the most recent paper submitted on machine learning in the Computer Science category posted on ArXiv.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--8", "ques": "What is the latest news on ArXiv?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--9", "ques": "Find the latest research paper about neural networks published on ArXiv which has been submitted within the last week.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--10", "ques": "Visit ArXiv Help on how to withdraw an article if the submission is not yet announced.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--11", "ques": "For Non-English submissions, do I need to provide a multi-language abstract, if need, answer the separator between the multiple abstracts.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--12", "ques": "Find store in arXiv Help, tell me how many styles of arXiv Logo Shirt are available?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--13", "ques": "How many articles on ArXiv with 'SimCSE' in the title?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--14", "ques": "On ArXiv, how many articles have 'SimCSE' in the article and are originally announced in October 2023?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--15", "ques": "Searching Chinese Benchmark on ArXiv, how many papers announced in December 2023 mention being accepted for AAAI 2024?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--16", "ques": "Locate the latest research about gravitational waves that were uploaded to ArXiv this week and provide a brief summary of one article's main findings.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--17", "ques": "Find the paper 'GPT-4 Technical Report', when was v3 submitted?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--18", "ques": "Download the paper 'Dense Passage Retrieval for Open-Domain Question Answering'. How many formulas are in the article and which one is the loss function?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--19", "ques": "Which university maintains and manages ArXiv. Accessing the university's website from ArXiv, how many underegraduate students are currently at the university.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--20", "ques": "Find the latest paper on 'machine learning in the Statistics section of ArXiv and provide its abstract.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--21", "ques": "Search for papers on 'neural networks for image processing' in the Computer Science category on ArXiv and report how many were submitted in the last week.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--22", "ques": "Locate the ArXiv Help section and find instructions on how to subscribe to daily listing emails for new submissions in a specific category.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--23", "ques": "Determine how many articles with the keyword 'autonomous vehicles' were published in the 'Electrical Engineering and Systems Science' section of ArXiv yesterday.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--24", "ques": "Identify the most recent paper related to 'graph neural networks' on ArXiv and determine the affiliation of the first author.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--25", "ques": "Browse the ArXiv store and let me know how many different types of merchandise are available.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--26", "ques": "Search for papers related to 'climate change modeling' on ArXiv and find out how many have been published in the Earth and Planetary Astrophysics (astro-ph.EP) category in the last week.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--27", "ques": "On ArXiv, what categories does Economics include, and what are their abbreviations?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--28", "ques": "Search 'Poly encoder' by title on ArXiv and check whether the articles in the search results provide HTML access.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--29", "ques": "On ArXiv, search for papers with 'Neural Network Optimization' in the title published in 2023, and provide the number of such papers.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--30", "ques": "Look up the submission guidelines on ArXiv for submitting a paper and tell me the formats for figures.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--31", "ques": "Search ArXiv for papers with 'Graph Neural Networks' in the abstract that were submitted between Jan 1, 2024, and Jan 3, 2024, and determine how many of these papers have more than five authors.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--32", "ques": "Locate the latest paper on ArXiv within the 'Nonlinear Sciences - Chaotic Dynamics' category, summarize the abstract and note the submission date.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--33", "ques": "Query ArXiv for the latest research article in the category of Systems and Control under Computer Science. Summarize the main objective or hypothesis presented in the paper and provide the names of the authors.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--34", "ques": "Search for the most recent paper related to non-commutative geometry submitted by an author with the first name John. Provide the title and the abstract.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--35", "ques": "Retrieve the latest research paper in Quantum Physics from ArXiv and provide the title, author(s), and date of submission.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--36", "ques": "Search 'CVPR 2023' and 'CVPR2023' through journal ref on ArXiv to see how many results there are respectively.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--37", "ques": "Find the names of people in ArXiv's Leadership Team.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--38", "ques": "Find the ArXiv Blog on the ArXiv website and summarize the content of its latest article.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--39", "ques": "Search the title 'GPT-4 Technical Report' and access this paper through HTML format. Read the paper on this page and tell me what is 'one of the main goals of developing such models' mentioned in the Introduction.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--40", "ques": "How many articles are there on each of the three most recent announce days in the Solar and Stellar Astrophysics section of ArXiv. Choose one at random and answer its title and when the first version was uploaded?", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--41", "ques": "Find the button to share arxiv non-profit store and follow the QR code to share the shop. Then add arXiv Forever short sleeve (XL) to your cart.", "web": "https://arxiv.org/"} +{"web_name": "ArXiv", "id": "ArXiv--42", "ques": "Find an article published between 1 January 2000 and 1 January 2005 that requires Support Vector Machines in the title and its Journey ref is ACL Workshop.", "web": "https://arxiv.org/"} +{"web_name": "BBC News", "id": "BBC News--0", "ques": "Find a report on the BBC News website about recent developments in renewable energy technologies in the UK.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--1", "ques": "Read the latest health-related news article published on BBC News and summarize the key points discussed.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--2", "ques": "Read the latest article regarding the environmental impacts of deforestation published within the last two days.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--3", "ques": "Check the leaderboard for Golf's DP World Tour in the SPORT section, what was the name of the most recent tournament, and how many teams have a Total of -10 strokes.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--4", "ques": "Find the latest article regarding the economic implications of climate change in Europe as reported by BBC News and summarize the central points.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--5", "ques": "Find the article \"What is climate change? A really simple guide\" and use it to answer what human activities are causing climate change.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--6", "ques": "Find the top story from BBC News in the technology section for today.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--7", "ques": "Find a AI-related story under Technology of Business. What is in the first picture in the story?", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--8", "ques": "Get a brief overview of the economic implications of the UK's latest trade deal posted on BBC News and the date when the article was published.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--9", "ques": "Find out which musician made the headlines in Music News.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--10", "ques": "Identify the main headlines covering the UK's plan to tackle climate change on BBC News.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--11", "ques": "Find out how many teams are in the Scottish Premiership of the Football Tournament and when did the Hibernian team's most recent match start?", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--12", "ques": "Find a picture in the travel section that contains food, tell me what the food is called and what region it comes from.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--13", "ques": "Search for recent news related to Trump and summarize the main points.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--14", "ques": "Find a news article on BBC News about the impact of the recent tech industry layoffs on the global economy. Summarize the key points and the name of the author, and provide the date of publication.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--15", "ques": "What does the current headline in Natural Wonders tell about.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--16", "ques": "Identify the most recent development or update in Brexit negotiations as reported on BBC News and report the key points and any stated impacts on European economies.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--17", "ques": "How many War related sections are currently in BBC News.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--18", "ques": "Visit BBC News Audio, What are the best PodCasts for 2023? List 2 of them.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--19", "ques": "Visit the Athletics calendar for the date of the next earliest game.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--20", "ques": "Find the latest article in the Green Living section on BBC News and provide a summary of its main points.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--21", "ques": "Identify the top headline in the World News section on BBC News and describe the region it is related to.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--22", "ques": "Determine the current top business story on BBC News and give a brief overview of its economic implications.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--23", "ques": "Identify the latest health-related news on BBC News and summarize the main findings or recommendations.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--24", "ques": "Search the latest article about space exploration on BBC News and summarize its key points.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--25", "ques": "Find the most recent sports analysis article on BBC News related to the English Premier League and summarize its key insights.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--26", "ques": "Locate the latest report on BBC News about the impact of recent natural disasters in Asia and summarize the key points and areas affected.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--27", "ques": "Find the most recent article on BBC News about archaeological discoveries and summarize the main findings and their significance.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--28", "ques": "Find the Market Data section on BBC News and tell me which company the data comes from.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--29", "ques": "Visit BBC News Audio and find out which podcast episode is currently featured as the \"New Releases\".", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--30", "ques": "In the Culture section, identify the latest film release reviewed and provide a brief summary of the review.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--31", "ques": "Check the Sports section for the result of the most recent Manchester United football match.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--32", "ques": "Find the artificial intelligence section, what is the top headline at this time, and which companies are involved?", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--33", "ques": "In the World News section, find the latest war situations of Middle East and provide a brief summary.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--34", "ques": "Find The SpeciaList section in Travel and browse the page to see which cities are mentioned.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--35", "ques": "In the Asia section, browse and identify the most recent report about technological advancements and summarize its content.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--36", "ques": "Look up recent articles in the Africa news section in World, summarize what topics most of these news are about", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--37", "ques": "Identify the latest book review featured in the Culture section and provide the title and author of the book.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--38", "ques": "Find news related to the storm in Weather section and indicate where and when the severe weather occurred.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--39", "ques": "Check the Horse Racing results in Sport section, browse all the games that took place yesterday and see which one had the highest number of runners.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--40", "ques": "Read and summarise a recent story on BBC News about people being injured or killed in wars.", "web": "https://www.bbc.com/news/"} +{"web_name": "BBC News", "id": "BBC News--41", "ques": "Find Golf in BBC News, check the Leaderboard at this point in Women's Majors and count which country has the most players in the top 20? Which player has the best score amongst the Australian players and in what place.", "web": "https://www.bbc.com/news/"} +{"web_name": "Booking", "id": "Booking--0", "ques": "Find a Mexico hotel with deals for December 25-26.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--1", "ques": "Find the cheapest available hotel room for a three night stay from 1st Jan in Jakarta. The room is for 2 adults, just answer the cheapest hotel room and the price.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--2", "ques": "Find a hotel in Ohio From December 20th to December 23th for 3 adults and 2 rooms.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--3", "ques": "Find a hotel with 4 star and above rating in Los Angeles for 3 days from Dec 18th.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--4", "ques": "Search for the cheapest Hotel near Kashi Vishwanath Temple that offer breakfast from Dec 25th - Dec 26th.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--5", "ques": "Search a hotel with free WiFi and air conditioning in Bali from Jan 1 to Jan 4, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--6", "ques": "Book one room which provides breakfast, and airport shuttle from Jan 22 to 25 in Los Angeles.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--7", "ques": "Find a hotel room on January 3-6 that is closest to National University of Singapore and costs less than $500", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--8", "ques": "Get the hotel with highest review score and free cancelation in Chennai for 20/12/2023 - 21/12/2023.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--9", "ques": "Find hotels for 2 adults in London with a price less than 250 dollars for four days starting from December 25. You must browse the page and offer at least 3 options.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--10", "ques": "Find a well-reviewed hotel in Paris with available bookings suitable for a couple (2 adults) on Valentine's Day week, February 14-21, 2024, that offers free cancellation options.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--11", "ques": "Reserve a hotel in downtown Chicago with a rating of 9 or higher for a stay from March 20-27, 2024, which offers free cancellation and includes a fitness center.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--12", "ques": "Find a hotel in Paris with a customer review score of 8 or higher, free Wi-Fi, and available for a 5-night stay starting on January 5th, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--13", "ques": "Find and book a hotel in Paris with suitable accommodations for a family of four (two adults and two children) offering free cancellation for the dates of February 14-21, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--14", "ques": "Book a highly-rated hotel with a swimming pool and free WiFi near the Louvre Museum in Paris for the weekend of March 3-5, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--15", "ques": "Find the highest-rated luxury hotel in Rome available for booking from January 10, 2024, to January 20, 2024, for 2 adults. Include the cost, amenities offered, and customer rating.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--16", "ques": "Look for a hotel in Paris with a user rating of 9 or higher and available for a 5-night stay starting January 15, 2024. The hotel should also offer free Wi-Fi and breakfast included in the price. Provide the name, location, and price per night.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--17", "ques": "Find a hotel in Paris with a fitness center and a rating of 8 or higher available for a 5-night stay starting from February 14, 2024, and sort the results by best reviewed.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--18", "ques": "Search a hotel in London with a user rating of 8 or higher for a stay between February 14th, 2024, and February 21st, 2024, suitable for a couple. Provide the name and a short description of the hotel.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--19", "ques": "Look for a hotel with customer ratings above an 8.0 in Paris, France for a weekend stay from March 18, 2024, to March 20, 2024, and list top three suggestions based on user reviews.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--20", "ques": "Locate a hotel in Rome with a good rating (7 or above) that offers free cancellation and breakfast included, for a three-night stay from February 28 to March 2, 2024, for two adults.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--21", "ques": "Find a hotel in Sydney with a rating of 8 or higher, providing free Wi-Fi and parking, available for a four-night stay starting on March 10, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--22", "ques": "Search for a hotel in Amsterdam with a customer review score of 9 or higher, offering bicycle rentals, for a week-long stay from March 15 to March 22, 2024, for two adults.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--23", "ques": "Identify a hotel in Tokyo with a spa and wellness center, rated 9 or above, with availability for a five-night stay starting on February 20, 2024. Check if free cancellation is offered.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--24", "ques": "Find a hotel in Barcelona for a stay from February 25-28, 2024. Please sort the results by distance from the beach and make sure they offer free Wi-Fi and breakfast.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--25", "ques": "Search for a hotel in Lisbon with airport shuttle, rated 8.5 or above, available for a six-night stay from March 1 to March 7, 2024, for two adults, breakfast included.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--26", "ques": "Check Booking.com for a 3-star hotel or higher in Paris with a guest rating above 8.0 and available parking for dates February 20-23, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--27", "ques": "Locate a hotel in Melbourne offering free parking and free WiFi, for a stay from February 28 to March 4, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--28", "ques": "Find a hotel in Dubai with a swimming pool, for a week-long stay from February 22 to February 29, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--29", "ques": "Search for a hotel in Toronto with a fitness center and a rating of 8+, available for a two-night stay from March 5 to March 7, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--30", "ques": "Search for hotels in London from March 20 to March 23, 2024, on Booking. How many hotels are left after applying the Breakfast included and Fitness center filters?", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--31", "ques": "Search for hotels in Rio de Janeiro from March 1-7, 2024, check the Brands filter to see which brand has the most hotels and which brand has the fewest.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--32", "ques": "Look for hotels in Sydney from February 24 to February 27, 2024, on Booking. Once the Swimming Pool and Airport Shuttle filters are applied, what is the total number of hotels available?", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--33", "ques": "Find the Customer Service on the Booking website, browse the questions about cancellation, and tell me 'how do I know whether my booking has been cancelled'.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--34", "ques": "Search for a hotel in Berlin available for a three-night stay from March 15 to March 18, 2024, for one adult. Tell me the price in USD and CNY for the three-night stay.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--35", "ques": "Browse the booking website to get inspiration for your next trip, and summarize at least three places mentioned in one of the travel articles.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--36", "ques": "Search for a budget hotel in Rome under $100 per night for one adult from March 20 to March 23, 2024. Sort the results by price, identify if any of top three results offer breakfast.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--37", "ques": "Search for a resort (not hotel) in Bali, detailing the available dates between March 20, 2024, and March 25, 2024, and checking any provided tour or cultural experiences.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--38", "ques": "Look up Vienna hotel options with availability for a 4-night stay from February 28 to March 4, 2024, with amenities that include a Parking, breakfast included, and a rating of 8+ on Booking.com.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--39", "ques": "Find a pet-friendly hotel with parking available in downtown Toronto for the stay of February 24-26, 2024.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--40", "ques": "I need to choose a hotel in Shenzhen, please select date (6 March to 8 March 2024) and click the search button. How much it costs when convert the price to Chinese Yuan on the page.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--41", "ques": "Browse Booking's homepage to find out which company it belongs to.", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--42", "ques": "Search for a hotel in Hokkaido for the period March 1 to March 7, 2024, with a rating of 9+, check out its user reviews, which categories are greater than 9 and which are less than 9?", "web": "https://www.booking.com/"} +{"web_name": "Booking", "id": "Booking--43", "ques": "Search for properties in Los Angeles, browse the results page to see what filters are available, list some of them.", "web": "https://www.booking.com/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--0", "ques": "Look up the pronunciation and definition of the word \"sustainability\" on the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--1", "ques": "Find the pronunciation, definition, and a sample sentence for the word 'serendipity'.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--2", "ques": "Look up the pronunciation, definition, and example sentence for the word \"ubiquitous\" in UK and US English.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--3", "ques": "Look up the definition, pronunciation, and examples of the word \"zeitgeist.\"", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--4", "ques": "Look for the British English pronunciation of the word \"innovate\" and write down the International Phonetic Alphabet (IPA) notation, then find one example sentence provided in the Cambridge Dictionary that uses this word.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--5", "ques": "Learn the UK and US pronunciation of the word \"procrastination\", and find one example sentence that reflects its use in context.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--6", "ques": "Search for the word \"sustainability\" on the Cambridge Dictionary, what is the translation of sustainability into Chinese and French in the dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--7", "ques": "Look up the meaning, pronunciation, and an example sentence of the word \"gestalt\" using the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--8", "ques": "Find three different meanings of \"dog\" in Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--9", "ques": "Look up the British pronunciation of the word \"euphoria\" and find an example sentence using that word on the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--10", "ques": "Look up the definition and pronunciation of the word \"impeccable\" and also find an example sentence using that word.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--11", "ques": "Look up the pronunciation and definition of the word \"ameliorate,\" and provide an example sentence using the word.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--12", "ques": "Find the pronunciation, definition, and a sample sentence for the word \"resilience\" in the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--13", "ques": "Find one word, one phase and one idiom related to euphoria in Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--14", "ques": "Use the Cambridge Dictionary to find the pronunciation, definition, and one example sentence for the word \"concatenate\".", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--15", "ques": "Find the pronunciation and a sample sentence for the word \"pandemic.\"", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--16", "ques": "Look up the definition of \"cryptocurrency\" on Cambridge Dictionary, provide the pronunciation, and use it in two example sentences that illustrate different contexts.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--17", "ques": "How many meanings of \"unblemished\" are given in Cambridge Dictionary? Please browse the page and give the number directly.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--18", "ques": "Search for \"to behave well\" in Cambridge Dictionary's Thesaurus and see which synonyms the dictionary gives.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--19", "ques": "Try a Cambridge Dictionary translation and tell me which company provided the translation.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--20", "ques": "Look up the definition, pronunciation (both UK and US), and find one example sentence for the word \"altruism\" in the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--21", "ques": "Search for the word \"ephemeral\" on Cambridge Dictionary and find its translation into Spanish.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--22", "ques": "Use the Cambridge Dictionary to find the definition, UK pronunciation, and an example sentence for the word \"quintessential.\"", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--23", "ques": "Find the US English pronunciation of the word \"meticulous\" using the Cambridge Dictionary and note the International Phonetic Alphabet (IPA) notation, then find one example sentence provided in the dictionary using this word.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--24", "ques": "Look up the definition and both UK and US pronunciation of the word \"reverie,\" and provide an example sentence using the word from Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--25", "ques": "Find two different meanings of the word \"harmony\" in the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--26", "ques": "Search for the word \"nostalgia\" in the Cambridge Dictionary and report the translation of this word into Chinese.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--27", "ques": "Look up the meaning, pronunciation, and an example sentence of the word \"solitude\" using the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--28", "ques": "Search for \"feel giddy\" in Cambridge Dictionary's Thesaurus and list the synonyms the dictionary provides.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--29", "ques": "Go to the Plus section of Cambridge Dictionary, find Image quizzes and do an easy quiz about Animals and tell me your final score.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--30", "ques": "Find the grammar for present perfect simple uses in English, including examples of affirmative, negative, and interrogative sentences, on the Cambridge Dictionary website.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--31", "ques": "Look up the use of modal verbs in grammar section for expressing possibility (e.g., 'might', 'could', 'may') and find examples of their usage in sentences on the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--32", "ques": "Search for the differences between \"fewer\" and \"less\" in grammar section, and provide examples illustrating their correct usage from the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--33", "ques": "Find explanations and examples of the passive voice in Grammar on the Cambridge Dictionary website.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--34", "ques": "Use the Cambridge Dictionary to understand the rules for forming and using comparative and superlative adjectives in English Grammar, including example sentences.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--35", "ques": "Find the most common prepositions that consist of groups of words on the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--36", "ques": "Search for guidelines on using indirect speech in English, with examples of how to change direct speech to indirect speech, on the Cambridge Dictionary.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--37", "ques": "Use Cambridge Dictionary to understand the use of articles ('a', 'an', 'the') in English Grammar, including examples of usage with both countable and uncountable nouns.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--38", "ques": "Go to the Plus section of Cambridge Dictionary, finish a recommended Grammar quiz without login and tell me your final score.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--39", "ques": "Try the Word Scramble game in the Plus section, Can you beat the clock by unscrambling the letters to spell the word? (Just try the first example.)", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--40", "ques": "Look up the definition, pronunciation in UK English, and at least one example using the word 'mitigate'.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--41", "ques": "Find and browse Cambridge Dictionary Shop section, listing 3 items.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Cambridge Dictionary", "id": "Cambridge Dictionary--42", "ques": "Convert the Cambridge Dictionary homepage from English (UK) to Deutsch.", "web": "https://dictionary.cambridge.org/"} +{"web_name": "Coursera", "id": "Coursera--0", "ques": "Find a beginner-level online course about '3d printing' which lasts 1-3 months, and is provided by a renowned university.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--1", "ques": "Search for a beginner-level online course about Python programming, suitable for someone who has no programming experience on Coursera.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--2", "ques": "Find a Beginner's Spanish Specialization on Coursera and show all the courses in this Specialization.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--3", "ques": "Identify a new course or Specialization on Coursera related to Python Data Science, sort the courses by newest, what the first course is and which institution offers it.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--4", "ques": "Identify a course or Specialization on Coursera that helps business process management with with a rating 4.7.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--5", "ques": "Identify a Specialization on Coursera that teaches C++ programming for beginners, provide the name and what the learning outcomes are.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--6", "ques": "Identify a course on Coursera related to 'Artificial Intelligence for Healthcare' and note the course duration along with the number of quizzes in Assessments.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--7", "ques": "Find a course on Coursera that teaches Reinforcement Learning for Intermediate with a rating of at least 4.5. Provide the name of the course, the institution offering it, and the number of reviews it has received.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--8", "ques": "Find a free course related to 'R for Data Science' available on Coursera. Scroll to find a course with the Free tag. What language the course is taught in?", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--9", "ques": "Identify a Coursera course on artificial intelligence ethics that has a duration of less than 20 hours to complete and has been rated 4+ stars by participants.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--10", "ques": "Locate an introductory course related to artificial intelligence on Coursera, ensuring it's suitable for beginners and contains at least one module discussing Ethical Considerations.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--11", "ques": "Search for a Specialization on Coursera about project management that is produced by a university, show a testimonial for this Specialization.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--12", "ques": "Look for a Coursera course (not Specialization) that teaches Java programming basics.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--13", "ques": "Look for a Specialization on Coursera that teaches Python programming, and identify the skills you will learn by taking this Specialization.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--14", "ques": "Find a course on Coursera related to Introductory Project Management that includes modules on Agile methodology.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--15", "ques": "Find a course on Coursera named 'Introduction to Mathematical Thinking' offered by Stanford, what is the percentage (rounded) of 5 star ratings in reviews and which level has the least percentage?.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--16", "ques": "Identify a course on Coursera named 'Introduction to Finance: The Basics', who is the course instructor and what other courses does he/she teach.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--17", "ques": "How many results are there for a search on Coursera for Machine Learning, then filtered by Credit Eligible and 1-4 Years duration?", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--18", "ques": "Identify a Coursera course that teaches JavaScript, which is beginner-friendly and includes a certificate upon completion.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--19", "ques": "Identify a course on Coursera that provides an introduction to Psychology, list the instructor's name, the institution offering it, and how many hours it will approximately take to complete.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--20", "ques": "Find an Intermediate-level online course on Coursera about 'Blockchain Technology' which lasts between 1 to 4 weeks, and is provided by a well-known institution. Also, note the course's main goals and the instructor's name.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--21", "ques": "Search for an online course on Coursera about 'Digital Marketing', suitable for beginner-level learners. Specify the course duration, the main learning outcomes, and the institution offering the course.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--22", "ques": "Identify a Specialization on Coursera that focuses on 'Human Resource', list the courses included in this Specialization, and the institution offering it.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--23", "ques": "Find a course on Coursera about 'Artificial Intelligence Ethics', which has a duration of less than 5 weeks and has been rated 4.5 stars or higher. Provide the course name and the instructor's name.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--24", "ques": "Locate an online course on Coursera related to 'Sustainability' that belongs to Physical Science and Engineering subject. The course should include a module on Measuring Sustainability. Note the course duration and the offering institution.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--25", "ques": "Find a course on Coursera about 'Relativity' for beginners. List the course's main topics and the estimated time (in hours) required to complete it.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--26", "ques": "Identify a Specialization on Coursera that offers an overview of 'Renewable Energy'. The Specialization should be beginner-level and include a course on Renewable Energy Futures. Note the instructor's name and the number of weeks required to complete the course if I spend 5 hours a week.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--27", "ques": "Search for a Specialization on Coursera about 'Data Visualization' that includes a project. Provide the name of the Specialization, the institution offering it, and the skills that will be developed by completing it.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--28", "ques": "Locate a Coursera Guided project related to 'Astrophysics' suitable for advanced learners. Mention the course duration, the institution offering it, and the main subjects covered in the course.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--29", "ques": "Browse the Coursera website and find the price required for one year of Coursera Plus. How much is the discount? Then list 3 companies that work with Coursera.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--30", "ques": "Locate the course 'Modern Art & Ideas' on Coursera offered by The Museum of Modern Art. Find out the percentage (rounded) of 3-star ratings in the reviews and note which star level has the lowest percentage.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--31", "ques": "Search for the course 'Exploring Quantum Physics' on Coursera, offered by the University of Maryland, College Park. Identify the percentage (rounded) of 5-star ratings in the reviews.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--32", "ques": "Search for 'Data Analysis' courses on Coursera. Apply filters to find courses that are 'Beginner Level' and have a duration ranging from 1 to 3 months. Determine the total count of courses that match these specifications.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--33", "ques": "Find a beginner level Coursera course related to \"Internet of Things (IoT)\" with a high rating. Provide the course name, instructor's name, and a brief summary of the skills that will be taught.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--34", "ques": "Find the course on Coursera named 'Essentials of Global Health'. Determine the instructor of this course and summarize his bio, note if there are any additional courses he offers on Coursera.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--35", "ques": "Find a Coursera course on Sustainable Agriculture practices, and detail the course's objectives and the background of the lead instructor.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--36", "ques": "Browse Coursera, which universities offer Master of Advanced Study in Engineering degrees? Tell me what is the latest application deadline for this degree?", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--37", "ques": "Browse the Coursera homepage and list at least three free courses.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--38", "ques": "Browse Coursera, which universities and companies from Australia are partners of Coursera? List all of them.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--39", "ques": "Find the Space Safety course offered by TUM on Coursera. How many videos are there in module 2? What is the name of each video?", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--40", "ques": "Browse Coursera for Business and Coursera for Teams and summarise some of their advantages.", "web": "https://www.coursera.org/"} +{"web_name": "Coursera", "id": "Coursera--41", "ques": "Browse online degrees section on Coursera and list 3 Bachelor's degree programmes.", "web": "https://www.coursera.org/"} +{"web_name": "ESPN", "id": "ESPN--0", "ques": "Look up the current standings for the NBA Eastern Conference on ESPN.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--1", "ques": "Check the latest articles on ESPN for updates on any trades that occurred in the NBA within the past 2 days.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--2", "ques": "Show the scores and main highlight of the Milwaukee Bucks game that took place within the last 2 days on ESPN.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--3", "ques": "Retrieve the final score from the most recent NBA game broadcast on ESPN, including the playing teams' names and the date of the match.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--4", "ques": "Check ESPN for the final scores of NBA games that were played yesterday.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--5", "ques": "Identify the top scorer in the NBA from the latest completed game and note down the points scored, the team they play for, and their position on the team.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--6", "ques": "Find the result of the latest basketball game between the Los Angeles Lakers and the Boston Celtics, including the final score and top scorer from the match.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--7", "ques": "Retrieve the final score and a brief summary of the latest NBA game played by the Los Angeles Lakers as reported on ESPN.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--8", "ques": "Find information on ESPN about the top three scoring leaders in the NBA as of the last day of the regular season, and note which teams they play for.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--9", "ques": "Search on ESPN for how many teams have Los Angeles in their name and how many of them are NBA.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--10", "ques": "Check ESPN for the score and a brief recap of the latest college football championship game.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--11", "ques": "How many NBA teams are there and list all the teams with 'New' in their name.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--12", "ques": "The first three Top Headlines in the current ESPN home page correspond to which sports leagues?", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--13", "ques": "Identify today's top headline in the Basketball section of ESPN, and summarize the main points of that article.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--14", "ques": "Find the latest news about NBA trades or player movements on ESPN and report the most recent trade deal OR player acquisition.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--15", "ques": "Check the scores of the NBA games played on December 25, 2023.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--16", "ques": "Check the schedule for the NBA game on December 25, 2023, and provide the teams that are playing and their current standings in their respective conferences.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--17", "ques": "Check out the NBA Basketball Power Index 2023-24 to see which teams are in first place and which are in last place.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--18", "ques": "How many sports leagues can you choose from on the ESPN home page?", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--19", "ques": "Who has the highest salary in Boston Celtics Roster 2023-24?", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--20", "ques": "Look up the current leaders in rebounds and assists in the NBA Western Conference on ESPN.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--21", "ques": "Show the scores and main highlight of the Denver Nuggets game that occurred within the last 3 days on ESPN.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--22", "ques": "Find the latest Team transactions in the NBA within the past week.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--23", "ques": "Find the result of the latest basketball game between the Miami Heat and the New York Knicks, including the final score and top rebounder from the match.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--24", "ques": "Find the final score from the most recent NFL game broadcast on ESPN, including the teams' names and the date of the match.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--25", "ques": "Identify the player with the most assists in the latest NBA game and show me the assists, the team they play for, and their position.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--26", "ques": "Find information on ESPN NBA schedule. Tell me yesterday's matchups in which the loser high was higher than the winner high.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--27", "ques": "Search on ESPN for how many teams have 'Golden' in their name and how many of them are in the NHL.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--28", "ques": "How many MLB teams are there and list all the teams with 'City' in their name.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--29", "ques": "Identify today's top headline in the Soccer section of ESPN, and summarize the main points of that article.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--30", "ques": "Check out the NHL Standings 2023-24 on ESPN to see which teams are at the top and which are at the bottom in Eastern and Western Conference. What about the situation in Division.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--31", "ques": "Who has the heaviest weight among infielders in the New York Yankees Roster 2023-24?", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--32", "ques": "Review yesterday's NHL game results on ESPN, focusing on teams' performance.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--33", "ques": "Locate the latest ESPN articles discussing potential MVP candidates in the NFL for 2023 season.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--34", "ques": "Visit ESPN to view the Philadelphia 76ers' latest injuries.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--35", "ques": "Browse ESPN to find out when the next game of the Los Angeles Lakers will start. Then navigate to the ticket purchasing website from ESPN, what is the cheapest ticket available.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--36", "ques": "Search for Lionel Messi's last 5 games, which teams has he played for, and what are the results?", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--37", "ques": "Check out LeBron James' Stats to see how many games he has played in his career so far.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--38", "ques": "Check Los Angeles Lakers Stats 2023-24, calculate Anthony Davis' games played (GP) percentage, tell me if there are other players with the same games played percentage as Anthony Davis.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--39", "ques": "Check the New York Jets Depth Chart in the NFL section of ESPN and identify the players listed as injured in the 2ND position.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--40", "ques": "Browse the ESPN+ page from ESPN for a brief summary of what ESPN+ Tools is used for.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--41", "ques": "Find out which four teams the NFC North contains in the NFL on ESPN.", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--42", "ques": "Check out NCAAM standings on ESPN, what are the teams with equal wins and losses in the America East Conference currently?", "web": "https://www.espn.com/"} +{"web_name": "ESPN", "id": "ESPN--43", "ques": "Check out NCAAW recruiting on ESPN, what colleges are the top three players from?", "web": "https://www.espn.com/"} +{"web_name": "GitHub", "id": "GitHub--0", "ques": "Search for an open-source project related to 'climate change data visualization' on GitHub and report the project with the most stars.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--1", "ques": "Search for an open-source repository for machine learning in Python, specifically focused on decision trees, updated within the last 2 days.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--2", "ques": "Look for the trending Python repositories on GitHub with most stars.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--3", "ques": "Find out how much more package storage the Enterprise version has over Team in GitHub Pricing.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--4", "ques": "Find a popular JavaScript repository created in the last 30 days on GitHub with a Readme file.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--5", "ques": "Find a Python repository on GitHub that has been updated in the past 2 days and has at least 500 stars.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--6", "ques": "Search for an open-source project related to 'cryptocurrency wallet' updated in the past 30 days and provide the top three contributors.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--7", "ques": "Find the official GitHub repository for ALBERT and show me what files the repo changed in the most recent commit.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--8", "ques": "Look up the latest stable release version of Vuex and find out when it was published.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--9", "ques": "Locate a repository on GitHub that was created in the last week and has 50 or more stars. Provide brief details about the project's purpose and its programming language.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--10", "ques": "If I start using Copilot Individual, how much US dollars will it cost per year and what features does it have?", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--11", "ques": "Find a newly created open-source project on GitHub related to 'climate change' that has been initiated in January 2023; check the main programming language used and the project's description.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--12", "ques": "Retrieve the latest release from the 'electron/electron' repository on GitHub and note down the release version number and date.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--13", "ques": "Identify the latest top-trending open-source project in the category of 'Machine Learning' on GitHub, and check the number of stars it has received.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--14", "ques": "Locate the repository for the open-source project \"vscode\" and identify the top three contributors.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--15", "ques": "Locate a repository on GitHub related to 'quantum computing' that has been updated within the last week and has at least 50 stars. Provide a brief description of the project.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--16", "ques": "Find the GitHub Skill section and how many courses are under the 'First day on GitHub' heading.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--17", "ques": "Locate a C++ project on GitHub that has been recently updated in the last week and has at least 500 stars, then describe its main purpose.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--18", "ques": "Identify and report the most popular (in terms of stars) open-source image processing tool on GitHub.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--19", "ques": "Look up the most recently updated Python repository on GitHub that is tagged with 'web scraping' and has over 100 stars.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--20", "ques": "Open GitHub Copilot's FAQs to find the official answer to when Copilot chat can be used on mobile.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--21", "ques": "Find the Security topic in GitHub Resources and answer the role of GitHub Advanced Security.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--22", "ques": "Find an open-source repository on GitHub focused on natural language processing in Ruby, updated within the last week.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--23", "ques": "Find the wiki page of ohmyzsh on GitHub and tell me how to change the theme of zsh to agnoster.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--24", "ques": "Locate the GitHub repository for the open-source project \"angular\" and identify the last three issues closed.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--25", "ques": "Search for a 'virtual reality' related repository on GitHub updated in the last 10 days with at least 200 stars and summarize its main objective.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--26", "ques": "Find the Resolve merge conflicts course in GitHub Skills and what actions learners will perform in this course.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--27", "ques": "Find a Ruby repository on GitHub that has been updated in the past 3 days and has at least 1000 stars.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--28", "ques": "Identify the most starred JavaScript repositories on GitHub that were created after 2023-12-29.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--29", "ques": "Compare the maximum number of private repositories allowed in the Free and Pro plans in GitHub Pricing.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--30", "ques": "Search for an open-source project related to 'blockchain technology' on GitHub updated in the past 15 days and list the top five contributors.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--31", "ques": "Find the official GitHub repository for TensorFlow and list the files changed in the last commit. Tell me the name of changed files, total additions and total deletion.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--32", "ques": "Discover the latest C# repository on GitHub related to 'game development' and having over 150 stars, and describe its main features.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--33", "ques": "Find Customer Stories on the GitHub page and list the 2 stories that appear on the web page.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--34", "ques": "Search for an open-source project on GitHub related to 'Protein prediction' and identify the project with the highest number of forks.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--35", "ques": "Check the latest release version of React and the date it was published on GitHub.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--36", "ques": "Identify a new open-source project on GitHub related to 'AI agriculture' that created in 2022, and note its main programming language and description.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--37", "ques": "List the 3 features mentioned in GitHub's Copilot product page.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--38", "ques": "Identify and report the most popular (by stars) open-source repo related to cybersecurity on GitHub.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--39", "ques": "Browse the GitHub Trending and find out which developer is currently ranked first this month and the corresponding repository.", "web": "https://github.com/"} +{"web_name": "GitHub", "id": "GitHub--40", "ques": "Select Sign up on the GitHub homepage to see if email 'test123@gmail.com' already exists.", "web": "https://github.com/"} +{"web_name": "Google Flights", "id": "Google Flights--0", "ques": "Book a journey with return option on same day from Edinburg to Manchester on December 28th and show me the lowest price option available.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--1", "ques": "Show me the list of one-way flights today (February 17, 2024) from Chicago to Paris.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--2", "ques": "Find the lowest fare from all eligible one-way flights for 1 adult from JFK to Heathrow on Jan. 22.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--3", "ques": "Search for the one-way flight available from Calgary to New York on Jan. 1st with the lowest carbon dioxide emissions.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--4", "ques": "Search for one-way flights from New York to London on Dec. 26th and filter the results to show only non-stop flights.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--5", "ques": "Find flights from Chicago to London on 20 December and return on 23 December.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--6", "ques": "Search for a flight on December 19 and return on December 26 from Tel Aviv to Venice and Select First Class.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--7", "ques": "Find a round trip from Phoenix to Miami (Dec. 25th - Dec. 28th), show the First Class plane tickets for me that do not exceed $1320..", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--8", "ques": "Search a one-way filght from Dublin To Athens Greece for 1 Adult that leaves on December 30 and analyse the price graph for the next 2 months.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--9", "ques": "Find a one way economy flight from Pune to New York in Jan. 15th and show me how long it will take for flight transfer.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--10", "ques": "Locate the cheapest round-trip flights from New York to Tokyo leaving on January 25, 2024, and returning on February 15, 2024.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--11", "ques": "Compare the prices for round-trip flights from New York to Tokyo for a departure on February 10, 2024, and a return on February 24, 2024, and select the option with the least number of stops.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--12", "ques": "Find the best-priced round-trip flight from New York to London leaving on December 25, 2023, and returning on January 5, 2024, with one stop or fewer.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--13", "ques": "Find the cheapest round-trip flight option from New York City to Tokyo for a departure on January 10, 2024, and a return on January 24, 2024.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--14", "ques": "Compare flight options and find the lowest round trip fare from New York to London departing on January 10, 2024, and returning on January 17, 2024.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--15", "ques": "Compare the prices and total duration of non-stop flights from New York to Tokyo Narita Airport departing on February 12th, 2024, and returning on February 26th, 2024.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--16", "ques": "Find the cheapest one-way flight from New York to Tokyo departing on January 15, 2024, and provide the airline and total flight duration.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--17", "ques": "Find the cheapest round-trip flight from New York to Paris leaving on December 27, 2023, and returning on January 10, 2024.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--18", "ques": "Compare flight options from New York to Tokyo for a round trip leaving on January 25, 2024, and returning on February 15, 2024, for one adult. Prioritize the comparisons by the shortest travel time.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--19", "ques": "Find the cheapest one-way flight from London to Paris, departing on January 25, 2024. Include the airline, total travel time, and layovers for the chosen flight.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--20", "ques": "Book a round-trip flight from San Francisco to Berlin, departing on March 5, 2024, and returning on March 12, 2024, and find the option with the shortest total travel time.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--21", "ques": "Locate the lowest-priced one-way flight from Tokyo to Sydney for an adult, departing on February 25, 2024, and include the flight duration and number of layovers.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--22", "ques": "Find a round-trip flight from Rio de Janeiro to Los Angeles, leaving on March 15, 2024, and returning on March 22, 2024, and select the option with the least carbon dioxide emissions.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--23", "ques": "Search for a one-way flight from Mumbai to Vancouver on February 28, 2024, filtering the results to show only 1-stop flights.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--24", "ques": "Compare prices for economy class round-trip flights from Dubai to Rome, departing on March 1, 2024, and returning on March 8, 2024, and select the option with the fewest stops.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--25", "ques": "Find a one-way business class flight from Buenos Aires to Amsterdam on March 10, 2024, and provide the details of the flight with the shortest duration.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--26", "ques": "Search for the cheapest round-trip flights from Bangkok to Madrid, leaving on February 26, 2024, and returning on February 28, 2024, and provide options under $1000.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--27", "ques": "Locate a one-way flight from Johannesburg to Toronto on March 30, 2024, for one adult, and analyze the price trends for the following month.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--28", "ques": "Find the best-priced round-trip flight from Seattle to Paris, departing on February 27, 2024, and returning on March 1, 2024, with a maximum of one stop.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--29", "ques": "Compare the prices and total travel time of non-stop flights from Mexico City to Frankfurt, departing on March 5, 2024, and returning on March 15, 2024.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--30", "ques": "Find the most affordable one-way flight from Cape Town to Singapore, departing on March 20, 2024, and include the airline and total number of layovers.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--31", "ques": "Find a one-way economy flight from Auckland to Honolulu on March 25, 2024, browse the full page and display a flight option with the most stops.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--32", "ques": "Search for round-trip flights from Stockholm to Toronto, departing on March 3, 2024, and returning on March 10, 2024, and sort the results to find the shortest total travel time.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--33", "ques": "Find a one-way flight from Shanghai to Vancouver on February 27, 2024, and compare the options based on carbon dioxide emissions.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--34", "ques": "Compare business class flight options from Lisbon to Singapore for a one-way trip on March 15, 2024, select one of the flights and see which websites offer its booking options. Which one is the cheapest.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--35", "ques": "Find the lowest-priced one-way flight from Cairo to Montreal on February 21, 2024, including the total travel time and number of stops.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--36", "ques": "Search for round-trip flights from Helsinki to New Delhi, departing on March 28, 2024, and returning on April 4, 2024, and filter the results to show only flights under $1000.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--37", "ques": "Locate a round-trip flight from Buenos Aires to Beijing, leaving on February 28, 2024, and returning on March 3, 2024, check out one of the options and tell me if the airline for my return flight is the same as my departure flight.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--38", "ques": "Compare the prices and flight durations for economy class flights from Oslo to Dubai, departing on March 8, 2024, and show the options with no more than two layovers.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--39", "ques": "Find a one-way flight from Prague to a city in Japan on March 20, 2024, which city in Japan is cheaper to go to, Tokyo or a certain city in Hokkaido?", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--40", "ques": "Browse destinations on the Google Flights homepage from Seattle, look at destinations on a map, and recommend some famous places to travel that are within a reasonable distance and price.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Flights", "id": "Google Flights--41", "ques": "Choose one way business class ticket from Hong Kong to Glacier National Park on 8 March 2024, offering a 1 stop ticket.", "web": "https://www.google.com/travel/flights/"} +{"web_name": "Google Map", "id": "Google Map--0", "ques": "Find 5 beauty salons with ratings greater than 4.8 in Seattle, WA.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--1", "ques": "Tell me one bus stop that is nearest to the intersection of main street and Amherst street in Altavista.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--2", "ques": "Find Apple Stores close to zip code 90028", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--3", "ques": "The least amount of walking from Central Park Zoo to the Broadway Theater in New York.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--4", "ques": "Plan a trip from Boston Logan Airport to North Station.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--5", "ques": "Search for a parking garage near Thalia Hall in Chicago that isn't open 24 hours.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--6", "ques": "Find all Uniqlo locations in Chicago, IL.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--7", "ques": "Find bus stops in Alanson, MI", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--8", "ques": "Find a place to climb within 2 miles of zip code 90028.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--9", "ques": "Find the art gallery that is nearest to Los Angeles Hindu Temple.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--10", "ques": "Search for a park in the state of California called Castle Mountains National Monument and find out it's Basic Information.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--11", "ques": "Locate a large store in Washington that has kids' and maternity products, also check if it has a parking lot.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--12", "ques": "Find 5 places that serve burgers near 44012 zip code and sort these 5 places by highest rating.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--13", "ques": "Find a parking lot in Gloucester and book a ride from there to North Plymouth, view the map to understand the route better.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--14", "ques": "Find motorcycle parking near Radio City Music Hall.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--15", "ques": "Find daytime only parking nearest to Madison Square Garden. Summarize what people are saying about it. ", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--16", "ques": "Find EV charging supported parking closest to Smithsonian museum.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--17", "ques": "Search for locksmiths open now but not open 24 hours in Texas City.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--18", "ques": "Find a route between Chicago to Los Angeles, then print the route details.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--19", "ques": "I will arrive Pittsburgh Airport soon. Provide the name of the Hilton hotel closest to the airport. Then, tell me the the walking time to the nearest supermarket from the hotel.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--20", "ques": "Find Tesla Destination Charger closest to the National Air and Space Museum.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--21", "ques": "Identify the nearest bus stop to the corner of Elm Street and Oak Street in Massachusetts.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--22", "ques": "Find a Best Buy store near zip code 33139.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--23", "ques": "Determine the shortest walking route from The Metropolitan Museum of Art to Times Square in New York.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--24", "ques": "Plan a journey from San Francisco International Airport to Union Square via driving.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--25", "ques": "Search for a parking facility near the Fox Theater in Detroit that closes at night.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--26", "ques": "Search for Los Angeles on Google Map, try to print the map as PDF and summarize the information on the map.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--27", "ques": "Locate the Target stores in Atlanta, GA. How many results are shown on the map.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--28", "ques": "Find the search settings for Google Map, what options are shown on that page?", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--29", "ques": "Identify bus stops in Ypsilanti, MI, list three of them.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--30", "ques": "Locate a parking lot near the Brooklyn Bridge that open 24 hours. Review the user comments about it.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--31", "ques": "First search New York's Central Park Zoo on Google Map, and then find the way to share the map. What is the generated sharing link?", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--32", "ques": "Search for plumbers available now but not open 24 hours in Orlando, FL.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--33", "ques": "Check out Denver International Airport's information and tell me: 1) which level has the least proportion in reviews; 2) what are its Accessibility and Amenities.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--34", "ques": "Find a hiking trail within 2 miles of zip code 80202.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--35", "ques": "Search for a natural reserve in Texas called Big Bend National Park and gather its Basic Information.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--36", "ques": "Identify 5 restaurants serving pizza near the 30309 zip code and rank them by their ratings.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--37", "ques": "Locate a parking area in Salem and find a route from there to Marblehead, including map directions for better understanding.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--38", "ques": "Search for bicycle parking near the Empire State Building.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--39", "ques": "Find a route from Miami to New Orleans, and provide the detailed route information.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Map", "id": "Google Map--40", "ques": "Find a restaurant in Boston that eats Boston lobster and asks for a rating of 4.6 or higher, and check out what a one-star review says.", "web": "https://www.google.com/maps/"} +{"web_name": "Google Search", "id": "Google Search--0", "ques": "Find the initial release date for Guardians of the Galaxy Vol. 3 the movie.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--1", "ques": "Find Kevin Durant's bio", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--2", "ques": "Search for the latest news title about the NBA team the Los Angeles Lakers.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--3", "ques": "Show me a list of comedy movies, sorted by user ratings. Show me the Top 5 movies.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--4", "ques": "Show most played games in Steam. And tell me the number of players in In game at this time", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--5", "ques": "find the score of the latest nba game played by the phoenix suns.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--6", "ques": "Browse the monthly trending searches in Columbus.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--7", "ques": "Find the software requirements for iPhones that support AirDrop's ability to continue transmitting over the web when out of range.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--8", "ques": "Find the video on YouTube: 'Oscars 2023: Must-See Moments!'. Tell me who the first comment displayed under that video belongs to, and how many thumbs up and replies it has.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--9", "ques": "Show the rating of Prometheus movie on IMDb and Rotten Tomatoes.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--10", "ques": "Find the no. 1 weekly charts ranked artist based on Billboard and tell me 10 most played song by this artist until now.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--11", "ques": "According to FlightAware, tell me the busiest airport last week and its total arrivals and departures last week.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--12", "ques": "Find the year that Tom Brady had the most touchdowns in a single seasson.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--13", "ques": "What are Jerry Trainor's upcoming projects?", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--14", "ques": "Find the retired players the year before last named James Smith and tell me which club he has been a member of from 2020\u20132021.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--15", "ques": "Please try to log in to twitter with email: webagenttest@testmail.com and password: test123456. Let me know if the login was successful.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--16", "ques": "How many members are there in the OpenAI community on Reddit, and what is the hottest news right now?", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--17", "ques": "Tell me the names of Trump's kids", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--18", "ques": "When and where the most recent World Cup was held, and which team was the winner?", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--19", "ques": "What are the first 7 bits of the SHA of the Bert's latest commit on GitHub, and what exactly was changed in that commit.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--20", "ques": "Find the release date for the latest \"Fast & Furious\" movie.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--21", "ques": "Show a list of the top 5 highest-grossing animated movies, sorted by box office earnings.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--22", "ques": "Browse and list the top three trending topics this month in New York City.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--23", "ques": "Retrieve a short biography of LeBron James.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--24", "ques": "What is the name of the star system closest to the Solar System, and what are the discovered planets in it?", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--25", "ques": "Get the latest news headline about the English Premier League football club Manchester United.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--26", "ques": "Identify the hardware requirements for using the latest version of Adobe Photoshop on a Mac.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--27", "ques": "Check the current air quality index in Paris.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--28", "ques": "Check the IMDb and Metacritic scores of the movie \"Inception.\"", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--29", "ques": "Find out the current world record for the men's 100m sprint.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--30", "ques": "Find the current number one artist on the Spotify Global Top 50 chart and list his/her top 10 songs as of now.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--31", "ques": "Discover which year Cristiano Ronaldo scored the most goals in a single season.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--32", "ques": "Find out where and when the most recent UEFA Champions League final was held, and which team won.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--33", "ques": "Find and copy the SHA of the latest commit in the TensorFlow repository on GitHub, then find a textbox to paste and tell me what the SHA is.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--34", "ques": "Determine the distance from Earth to Mars as of today's date.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--35", "ques": "Look up the latest research paper related to black holes published in the journal \"Nature Astronomy\".", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--36", "ques": "Search for the most recent Nobel Prize winner in Physics and their contribution to the field.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--37", "ques": "Find the current top 3 super-earth planets and give a brief introduction to them.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--38", "ques": "Search for the next visible solar eclipse in North America and its expected date, and what about the one after that.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--39", "ques": "Identify the top-10 trending travel destination for 2024 through a blog, how many of them are in Asian.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--40", "ques": "Look up the elevation of Mount Kilimanjaro on Google Search.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--41", "ques": "Look up the current statistics of air pollution level in Los Angeles using Google Search.", "web": "https://www.google.com/"} +{"web_name": "Google Search", "id": "Google Search--42", "ques": " Use Google Search to find an article that explains the major differences between American English and British English.", "web": "https://www.google.com/"} +{"web_name": "Huggingface", "id": "Huggingface--0", "ques": "Find a pre-trained natural language processing model on Hugging Face that can perform sentiment analysis, and make sure the model's last update is within March 2023.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--1", "ques": "Use the Huggingface Inference API to generate a short story about a dragon and a wizard.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--2", "ques": "Discover three new and popular open-source NLP models for language translation released in the past month on Huggingface.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--3", "ques": "Look up a model with a license of cc-by-sa-4.0 with the most likes on Hugging face.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--4", "ques": "Locate an open-source conversational AI model on Hugging Face, trained in English and list its main features and applications.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--5", "ques": "Find a model released on Hugging Face for recipe generation. Retrieve the information of the model, including its name, model size and tensor type.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--6", "ques": "Find the model sentence-transformers/all-MiniLM-L6-v2 and use the Inference API on the webpage to get the similarity of the following two sentences: 'Tomorrow is Sunday', 'Eat a burger on Sunday'.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--7", "ques": "Which is the most downloaded audio related dataset on Hugging face currently.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--8", "ques": "Retrieve an example of a pre-trained language model in natural language processing and identify the tasks it is specifically designed for, like translation or text summarization.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--9", "ques": "Find the most download machine translation model on Huggingface which focuses on English and Japanese (en-ja) and report the evaluation metrics stated for it.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--10", "ques": "Open space: argilla/notux-chat-ui and interact with it by asking it 'which team trained you'. What is its answer.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--11", "ques": "Identify the latest updated image to video model available on Huggingface and summarize its main features.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--12", "ques": "Find the most recently updated machine learning model on Huggingface which focuses on Error Correction.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--13", "ques": "Search for LLaMA in the huggingface doc, what type is the spaces_between_special_tokens parameter in LlamaTokenizer and what is its default value.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--14", "ques": "How much is the Pro account of Hugging face for a month and what are the features?", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--15", "ques": "Identify the most downloaded models on Hugging face that use the PaddlePaddle library.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--16", "ques": "Find information on the latest (as of today's date) pre-trained language model on Huggingface suitable for text classification and briefly describe its intended use case and architecture.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--17", "ques": "Find the most recently updated open-source project related to natural language processing on the Huggingface platform. Provide the project's name, creator, and a brief description of its functionality.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--18", "ques": "Look up TRL's forward modelling in the hugging face documentation on how to add a margin to a loss.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--19", "ques": "Explore and summarize the features of the most recent open-source NLP model released by Hugging Face for English text summarization.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--20", "ques": "Locate a pre-trained natural language processing model on Hugging Face that specializes in named entity recognition (NER), confirm that the model was last updated in 2022 and has 1M+ downloads.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--21", "ques": "Look up the tour about how to use the 'pipeline' feature in the Hugging Face Transformers library for sentiment analysis, and identify the default model it uses.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--22", "ques": "Identify the steps to convert a PyTorch model to TensorFlow using the Hugging Face Transformers library as described in their documentation.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--23", "ques": "Identify three innovative and widely recognized open-source NLP models for automatic speech recognition released in the past month on Huggingface.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--24", "ques": "Search for a model on Hugging Face with an Apache-2.0 license that has received the highest number of likes.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--25", "ques": "In the Hugging Face documentation, find the tutorial on loading adapters with PEFT, tell me how to load in 8bit or 4bit.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--26", "ques": "Identify a model on Hugging Face designed for generating travel chats. Obtain information about the model, including its name, size and training framwork.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--27", "ques": "Determine the most downloaded dataset related to Text Retrieval in NLP on Hugging Face.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--28", "ques": "Retrieve an example of a pre-trained model on Hugging Face that is optimized for question answering tasks and detail the languages it supports.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--29", "ques": "Summarize the description of the recent open-source NLP model released on Hugging Face for medical summarization.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--30", "ques": "Identify the most downloaded English-Chinese (en-zh) machine translation model on Huggingface and report its latest performance metrics and usage guidelines.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--31", "ques": "Identify the latest machine learning model on Huggingface that specializes in detecting fake news, including the date of its last update.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--32", "ques": "On the Hugging Face website, search for the model 'GPT-J-6B' and find the 'temperature' parameter in its settings. What is the default value of this parameter?", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--33", "ques": "List three hugging face docs. How many GitHub stars have they earned so far?", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--34", "ques": "List the benefits of hugging face classroom mentioned on Hugging face website.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--35", "ques": "Find the latest Diffusion-related blog on Hugging Face, and read its intro or overview section to roughly summarize the content of the blog.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--36", "ques": "Summarize all the payment plans and their advantages in huggingface pricing.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--37", "ques": "Browse the daily paper on Hugging Face. What is the title of the first article, how many upvotes has it received, and is there any related model or data release?", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--38", "ques": "Investigate the 'transformers' library in the Hugging Face documentation, focusing on how to add new tokens to a tokenizer.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--39", "ques": "Investigate in the Hugging Face documentation how to utilize the 'Trainer' API for training a model on a custom dataset, and note the configurable parameters of the Trainer class.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--40", "ques": "Check out Text Embeddings Inference in Hugging face's Doc to summarise the strengths of the toolkit.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--41", "ques": "What is the current Text-to-3D model with the highest number of downloads and tell me are there Spaces that use the model.", "web": "https://huggingface.co/"} +{"web_name": "Huggingface", "id": "Huggingface--42", "ques": "Check the Dataset Viewer for ai2lumos/lumos_complex_qa_plan_onetime on Hugging face. what is the content corresponding to user in the first message?", "web": "https://huggingface.co/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--0", "ques": "derivative of x^2 when x=5.6", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--1", "ques": "Give a constraint on the set of inequalities for the inner region of the pentagram.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--2", "ques": "Calculate 3^71 and retain 5 significant figures in scientific notation.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--3", "ques": "Let g(x) be the integral of x^2 cos(2x). Write the expression of g(x).", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--4", "ques": "Pack 24 circles in a circle radius r. Compare Densest known packing and Square packing. Then tell me the radius of the inner circles.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--5", "ques": "Show the solution of y\"(z) + sin(y(z)) = 0 from wolframalpha.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--6", "ques": "Simplify x^5-20x^4+163x^3-676x^2+1424x-1209 so that it has fewer items.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--7", "ques": "Give the final angle and final length after 6s of a Spring pendulum with spring equilibrium length=0.12m, initial length=0.24m, initial angle=80deg, mass=1kg, spring constant=120 N/m .", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--8", "ques": "Give 12 lbs of 4-cyanoindole, converted to molar and indicate the percentage of C, H, N.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--9", "ques": "Annual energy production of Diablo Canyon 2 in 2010.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--10", "ques": "Give the geomagnetic field on June 20, 2023 in Oslo.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--11", "ques": "Show the electrical resistivity of UNS A92024 and UNS G10800 at 20 degrees Celsius.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--12", "ques": "Which character in unicode 8900 to 8920 looks like a snowflake", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--13", "ques": "What is 10,000 US dollars worth now in 1980 and in 1970?", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--14", "ques": "Compare the total Calories: whopper vs baconator vs big mac. Assume that each serving of food is 300g.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--15", "ques": "Show the blood relationship fraction between you and your father's mother's sister's son.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--16", "ques": "Weight lose for a male with current weight 90 kg, 40 year old, 175 cm. If he intakes 1500 calories every day, how long will it take to lose 17 kg.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--17", "ques": "Show the average price of movie ticket in Providence, Nashville, Boise in 2023.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--18", "ques": "Plot Albert Einstein curve with Parametric equations.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--19", "ques": "Standing in the sun from 11:00 am with SPF 5 in Australia. Approximate time to sunburn for each skin type.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--20", "ques": "Compute the integral of 3e^(2x) from x=0 to x=5.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--21", "ques": "Calculate (1+0.1*i)^8 + (1\u22120.2*i)^8 where i is a complex number.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--22", "ques": "Determine the area of a regular hexagon with a side length of 7 cm.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--23", "ques": "Calculate the population growth rate of Canada from 2020 to 2023 using Wolfram Alpha.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--24", "ques": "Solve the differential equation y''(t) - 2y'(t) + 10y(t) = 0 and display its general solution.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--25", "ques": "Calculate the final position and velocity of a projectile launched at 45 degrees with an initial speed of 30 m/s after 3 seconds.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--26", "ques": "Convert 15 kilograms of sulfuric acid to moles and display the percentage composition of H, S, and O by weight.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--27", "ques": "Display the thermal conductivity of Copper (Cu) and Aluminum (Al) at 25 degrees Celsius.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--28", "ques": "Identify the character in Unicode range 9632 to 9650 that represents a hollow parallelogram.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--29", "ques": "Create a plot of cat curve using wolfram alpha.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--30", "ques": "Calculate the estimated time to sunburn for different skin types when exposed to the sun at 1:00 pm with SPF 1 in Brazil.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--31", "ques": "Using Wolfram Alpha, determine the current temperature and wind speed in Chicago, IL.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--32", "ques": "Print all prime numbers between 1000 and 1200 using Wolfram alpha.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--33", "ques": "Identify the electrical energy output of a hydroelectric power plant named Itaipu Dam in 2023 using Wolfram Alpha.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--34", "ques": "Calculate the mass of Jupiter compared to Earth using Wolfram Alpha. Also, find the length of one day on Jupiter.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--35", "ques": "Calculate the determinant of a 6x6 Hilbert matrix.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--36", "ques": "Determine the convergence or divergence of the series \u03a3 (n=1 to \u221e) of 1/(n^3 + 1).", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--37", "ques": "How many days are there between February 12, 2024 and August 9, 2050?", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--38", "ques": "Compute the length of a curve defined by y = 2x^3 - 3x^2 + 4x - 5 from x = 0 to x = 3.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--39", "ques": "Use Wolfram alpha to write the expression of the ellipse x^2 + 3 y^2 = 4 rotated 33 degrees counterclockwise.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--40", "ques": "Approximate amount of fat burned by a 28yo, 172cm tall, 70kg woman running for 30min at a pace of 6min/mile.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--41", "ques": "What is the approximate Heart Rate Reserve of a 50 year old man who has a heart rate of 60bpm at rest.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--42", "ques": "What is the raw memory of a 100.2\" * 123.5\" true colour picture at 72 ppi?", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--43", "ques": "A polyominoes of order 6 means you have 6 identical squares to combine different shapes (2-sided). How many combinations are there? Looking at all the shapes in the result, how many of them have only 2 rows in total?", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--44", "ques": "Solve the ODE, g' + cos(g) = 0, if there is a constant in the result, determine the value of the constant by the condition that g(0) = 1.", "web": "https://www.wolframalpha.com/"} +{"web_name": "Wolfram Alpha", "id": "Wolfram Alpha--45", "ques": "A 175cm tall, 85kg, 40yo man climbs 2500 steps at about 18cm per step and 40 steps per minute. summarise the Metabolic properties.", "web": "https://www.wolframalpha.com/"} \ No newline at end of file diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/data/reference_answer.json b/benchmarks/adapters/webvoyager/src/webvoyager/data/reference_answer.json new file mode 100644 index 0000000..72d55f2 --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/data/reference_answer.json @@ -0,0 +1,3292 @@ +{ + "Allrecipes": { + "notice": " note that review information is real-time", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "'Vegetarian Four Cheese Lasagna', 4.6-star, 181 reviews, Servings 8" + }, + { + "id": 1, + "type": "possible", + "ans": "\"Debbie's Vegetable Lasagna\", 4.7-star, include zucchini" + }, + { + "id": 2, + "type": "possible", + "ans": "'Easy Vegetarian Red Beans Lasagna', 496 Calories, prep time 20 mins" + }, + { + "id": 3, + "type": "golden", + "ans": "'Vegan Chocolate Chip, Oatmeal, and Nut Cookies', 4.9 star, 67 viewers (> 60)" + }, + { + "id": 4, + "type": "possible", + "ans": "'Baked Dijon Salmon', 4.6-star, prep time 15 mins" + }, + { + "id": 5, + "type": "possible", + "ans": "\"World's Best Pasta Sauce!\", 4.7-star, 818 reviews, " + }, + { + "id": 6, + "type": "possible", + "ans": "'Spinach Lasagna', 4.7-star, 501 reviews" + }, + { + "id": 7, + "type": "possible", + "ans": "'Best Chocolate Chip Cookies', , " + }, + { + "id": 8, + "type": "possible", + "ans": "'Beef Wellington', " + }, + { + "id": 9, + "type": "possible", + "ans": "'Spicy Vegetarian Lasagna', , prep time 30 mis, cook time 1 hour 10 mins" + }, + { + "id": 10, + "type": "golden", + "ans": "'Swedish Meatballs I', prep time 25 mins, total time 1 hour 25 mins" + }, + { + "id": 11, + "type": "possible", + "ans": "'Chocolate Cupcake', 1261 reviews, prep time 15 mins" + }, + { + "id": 12, + "type": "possible", + "ans": "'Best Chocolate Chip Cookies', 4.6-star, 14493 reviews, " + }, + { + "id": 13, + "type": "possible", + "ans": "'Crispy Fried Fish', Iron: 15mg" + }, + { + "id": 14, + "type": "possible", + "ans": "'Slow Cooked Chicken Stew', prep time 20 mins" + }, + { + "id": 15, + "type": "possible", + "ans": "'Ultimate Chocolate Dessert', 4.7-star, prep time 15 mins" + }, + { + "id": 16, + "type": "possible", + "ans": "'Chocolate Chip Cookie Cups', 5.0-star, 3 reviews, total time 45 mins, " + }, + { + "id": 17, + "type": "golden", + "ans": "Easy to make and very delicious" + }, + { + "id": 18, + "type": "possible", + "ans": "'Eggplant Lasagna', 4.7-star, 305 reviews" + }, + { + "id": 19, + "type": "possible", + "ans": "'Vegan Lasagna II', 9 Ingredients, 4.2-star, prep time 30 mins, cook time 1 hour, " + }, + { + "id": 20, + "type": "possible", + "ans": "'Cauliflower Pizza Crust', 4.2 stars, Prep Time: 15 mins, 59 Calories per serving" + }, + { + "id": 21, + "type": "possible", + "ans": "'Gluten-Free Fudge Brownies', 4.1 stars, 69 reviews, , Prep Time: 15 mins, Total Time: 1 hr" + }, + { + "id": 22, + "type": "possible", + "ans": "'Avocado Salad', 4.7 stars, 253 reviews, Prep Time: 15 mins, Nutrition Facts: 126 Calories, 10g Fat, 10g Carbs, 2g Protein" + }, + { + "id": 23, + "type": "possible", + "ans": "'Baked Chicken Schnitzel', 4.5 stars, 250 reviews, Prep Time: 20 mins, " + }, + { + "id": 24, + "type": "possible", + "ans": "'Eggplant Parmesan', 4.5 stars, 2711 reviews, Prep Time: 25 mins, Servings: 10" + }, + { + "id": 25, + "type": "possible", + "ans": "'Easy Quinoa Salad', 4.8 stars, 1107 reviews, Prep Time: 20 mins, Cook Time: 15 mins, " + }, + { + "id": 26, + "type": "possible", + "ans": "'The Best Vegetarian Chili in the World', 4.7 stars, 1681 reviews, Cook Time: 1 hr, , " + }, + { + "id": 27, + "type": "possible", + "ans": "'Indian Chicken Curry (Murgh Kari)', 4.7 stars, 955 reviews, , Prep Time: 20 mins, " + }, + { + "id": 28, + "type": "possible", + "ans": "'Vegan Brownies', 4.6 stars, 828 reviews, , Prep Time: 15 mins, Cook Time: 30 mins, " + }, + { + "id": 29, + "type": "possible", + "ans": "'Branzino Mediterranean', 36 reviews, include olive oil, , Prep Time: 15 mins, Cook Time: 25 mins, Total Time: 40 mins" + }, + { + "id": 30, + "type": "possible", + "ans": "'Spinach and Banana Power Smoothie', 4.8 stars, 72 reviews, Ingredients: 1 cup plain soy milk, 3/4 cup packed fresh spinach leaves, 1 large banana, sliced; Prep Time: 10 mins; " + }, + { + "id": 31, + "type": "possible", + "ans": "'Easy Paella', 4.6 stars, 470 reviews, , , Total Time: 1 hr" + }, + { + "id": 32, + "type": "possible", + "ans": "'Slow Cooker Beef Stew', 3994 reviews, Cook Time: 4 hrs, " + }, + { + "id": 33, + "type": "possible", + "ans": "'Low-Carb Bacon Spinach Egg Cups', 99 reviews, 237 Calories, 18g Fat, 4g Carbs, 17g Protein" + }, + { + "id": 34, + "type": "possible", + "ans": "'Baked Salmon', 4.7 stars, 2339 reviews, Cook Time: 35 mins, " + }, + { + "id": 35, + "type": "possible", + "ans": "'Italian Turkey Meatballs', 4.7 stars, 234 reviews, Cook Time: 15 mins, meat: 1/2 pounds ground lean turkey" + }, + { + "id": 36, + "type": "possible", + "ans": "'All American Apple Pie', 4.6 stars, 490 reviews, 350 degrees F (175 degrees C)" + }, + { + "id": 37, + "type": "possible", + "ans": "'Greek Salad', 4.6 stars, 192 reviews, 1 cup crumbled feta cheese, ground black pepper to taste..." + }, + { + "id": 38, + "type": "possible", + "ans": "'Ratatouille', 4.6 stars, 793 reviews, vegetables: 1 eggplant, cut into 1/2 inch cubes; 2 zucchini, sliced; 2 large tomatoes, chopped" + }, + { + "id": 39, + "type": "possible", + "ans": "'Smoked Salmon Sushi Roll', 78 reviews, Nutrition Facts (per serving): 291 Calories, 7g Fat, 45g Carbs, 11g Protein, ; You can refrigerate them in an airtight container for up to two days." + }, + { + "id": 40, + "type": "golden", + "ans": "The Allrecipes Allstars: Social media influencers, registered dietitians, grillmasters, and more seasoned home cooks make up our enthusiastic squad of 100+ brand ambassadors. This diverse, food-loving crew spans the U.S. geographically and represents many different cultures, ethnicities, and family makeups. Since 2011, the Allstars have created tens of thousands of original recipes, photos, and reviews plus shared their cooking expertise via flat and video content on our website, social media, plus more marketing channels." + }, + { + "id": 41, + "type": "possible", + "ans": "Ground Beef-Spinach Casserole; Mexican Ground Beef Casserole; Retro Ground Beef Casserole with Biscuits" + }, + { + "id": 42, + "type": "possible", + "ans": "'Banana Banana Bread', 4.7 stars, 12649 reviews" + }, + { + "id": 43, + "type": "possible", + "ans": "'Amazing Vegan Pumpkin Pie', 5.0 stars, Cook Time: 1 hr 55 mins" + }, + { + "id": 44, + "type": "possible", + "ans": "THANKSGIVING RECIPES; CHRISTMAS RECIPES; LUNAR NEW YEAR RECIPES; HANUKKAH RECIPES; PURIM RECIPES; MARDI GRAS RECIPES ..." + } + ] + }, + "Amazon": { + "notice": " Products results are related to time and location.", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "Xbox Core Wireless Gaming Controller - Velocity Green; 4.7-star" + }, + { + "id": 1, + "type": "possible", + "ans": "PUMA Golf 2019 Men's Rotation Polo; $50.00" + }, + { + "id": 2, + "type": "possible", + "ans": "HP Victus 15L Gaming Desktop with Windows 11 Home and 1TB disk size" + }, + { + "id": 3, + "type": "possible", + "ans": "First 3 results after sort" + }, + { + "id": 4, + "type": "possible", + "ans": "Nintendo Switch Lite - Blue; Used Good: $170" + }, + { + "id": 5, + "type": "possible", + "ans": "Apple iPhone 12 Pro, 128GB, Pacific Blue - Fully Unlocked (Renewed); Action: ADD_TO_CHART" + }, + { + "id": 6, + "type": "possible", + "ans": "Baby Trend Expedition Jogger, Dash Black; 22146 reviews; 4.7-star" + }, + { + "id": 7, + "type": "possible", + "ans": "Filter: 4-star, waterproof, size 6" + }, + { + "id": 8, + "type": "possible", + "ans": "Samsung Galaxy Tab S 10.5in 16GB Android Tablet - Titanium Gold (Renewed); $139.94" + }, + { + "id": 9, + "type": "possible", + "ans": "Gulokoka Large Dog Bed for Crate Comfortable Washable Pet Mat for Dogs, Cats, Gray" + }, + { + "id": 10, + "type": "possible", + "ans": "Sony Playstation PS4 1TB Black Console; 2-Year Protection for $30.99" + }, + { + "id": 11, + "type": "possible", + "ans": "Transolid STDE33226-2 Kitchen Sink, Stainless Steel; $120.89" + }, + { + "id": 12, + "type": "possible", + "ans": "Worth every penny" + }, + { + "id": 13, + "type": "possible", + "ans": "adidas Men's Essentials Fleece Hoodie; 500+ bought in past month" + }, + { + "id": 14, + "type": "possible", + "ans": "Surge Protector Power Strip $15.99, 8 Outlets, 4.7-star" + }, + { + "id": 15, + "type": "possible", + "ans": "Damyuan Men's Sport Gym Running Shoes Walking Shoes Casual Lace Up Lightweight; black, size 7, 4.0-star, $29.99" + }, + { + "id": 16, + "type": "golden", + "ans": "FREE Returns, 1. Go to Your Orders to start the return; 2. Print the return shipping label; 3. Ship it!" + }, + { + "id": 17, + "type": "possible", + "ans": "Johnson's Baby Care Essentials Gift Set, $7.55; SWEET DOLPHIN 12 Pack Muslin Burp Cloths Large 100% Cotton Hand Washcloths for Baby, $9.98" + }, + { + "id": 18, + "type": "possible", + "ans": "Gevi Household V2.0 Countertop Nugget Ice Maker, 20% off; Osmo - Little Genius Starter Kit for iPad & iPhone, 7% off;" + }, + { + "id": 19, + "type": "possible", + "ans": "THE HISTORY OF THE DECLINE AND FALL OF THE ROMAN EMPIRE (All 6 Volumes), released on January 10, 2024." + }, + { + "id": 20, + "type": "possible", + "ans": "Logitech Wave Keys Wireless Ergonomic Keyboard, $57.99, 4.6 stars, 26005 ratings" + }, + { + "id": 21, + "type": "possible", + "ans": "Braun BrewSense 12-Cup Drip Coffee Maker, Stainless Steel, 4.3 stars, $129.95" + }, + { + "id": 22, + "type": "possible", + "ans": "CAROTE 11pcs Nonstick Cookware Set, Non Stick, Oven Safe, $129.99 ($11.82 / Count)" + }, + { + "id": 23, + "type": "possible", + "ans": "Smartwatch for Men Android iPhone, Waterproof, Heart Rate, $54.99" + }, + { + "id": 24, + "type": "possible", + "ans": "Dash DMAF360GBAQ02 Aircrisp\u00ae Pro Digital Air Fryer, Digital Display, Auto Shut Off, 3qt, $90.10" + }, + { + "id": 25, + "type": "possible", + "ans": "2 Inch 7-Zone Memory Foam Mattress Topper Queen with 100% Bamboo Rayon Cover, Cooling Gel-Infused Swirl Egg Crate Memory Foam, $99.99" + }, + { + "id": 26, + "type": "possible", + "ans": "Portable Bluetooth Speaker, IPX7 Waterproof Wireless Speaker, 25W Super Bass 24H Playtime, $29.97" + }, + { + "id": 27, + "type": "possible", + "ans": "Hiearcool USB C Hub, USB C Multi-Port Adapter for MacBook Pro, 7IN1, include 4K HDMI USB3.0 and SD/TF Card Reader, $24.99" + }, + { + "id": 28, + "type": "possible", + "ans": "Retrospec Solana Yoga Mat 1\" Thick, Non Slip, $38.51" + }, + { + "id": 29, + "type": "possible", + "ans": "KelvinLux Solar Garden Lights Outdoor, 12 Packs, 12 LEDs, $35.99 ($3.00 / Count)" + }, + { + "id": 30, + "type": "possible", + "ans": "The Women Library Binding \u2013 Large Print, March 1, 2024, 4.8 stars" + }, + { + "id": 31, + "type": "possible", + "ans": "5K Digital Camera for Photography Autofocus, 16X Digital Zoom, 5.0 stars, $129.99" + }, + { + "id": 32, + "type": "possible", + "ans": "COMFEE' Stainless Steel Electric Kettle, 1.7 Liter, 4.6 stars" + }, + { + "id": 33, + "type": "possible", + "ans": "price compare: 1) Shinco 10,000 BTU Portable Air Conditioner, $314.99; 2) Renogy 8,000 BTU Portable Air Conditioners, $283.09; 3) SereneLife Compact Freestanding Portable Air Conditioner, $247.54" + }, + { + "id": 34, + "type": "possible", + "ans": "Complete Acrylic Paint Set, 24\u0445 Rich Pigment Colors, for Painting Canvas, $16.97" + }, + { + "id": 35, + "type": "possible", + "ans": "STAY FINE Top Grain Leather Wallet for Men, RFID Blocking, Slim Billfold with 8 Card Slots, FREE delivery Friday, March 1" + }, + { + "id": 36, + "type": "possible", + "ans": "UNGLINGA 150 Experiments Science Kits for Kids Age 6-8-10-12-14, 4.6 stars, $29.99" + }, + { + "id": 37, + "type": "possible", + "ans": "NEWLAKE Cotton Bedspread Quilt Sets-Reversible Patchwork Coverlet Set, Blue Classic Royal Pattern, Queen Size" + }, + { + "id": 38, + "type": "possible", + "ans": "Bird Feeder for Outdoors Hanging, Squirrel Proof, FREE delivery Friday, March 1" + }, + { + "id": 39, + "type": "possible", + "ans": "Japan Travel Guide 2024: The Ultimate Route to Authentic Ramen and Beyond \u2013 Tips, Maps, and Must-Sees for Every Traveler, February 1, 2024, 38 ratings" + }, + { + "id": 40, + "type": "possible", + "ans": "ProsourceFit Extra Thick Yoga Pilates Exercise Mat, 1/2\", 4.6 stars, $21.99, 7 colors, FREE delivery Friday, March 1 on orders shipped by Amazon over $35" + } + ] + }, + "Apple": { + "notice": "", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "MacBook Air 13-inch M1 chip: from $999; 13-inch M2 chip: from $1099; 15-inch M2 chip: from $1299" + }, + { + "id": 1, + "type": "possible", + "ans": "StandBy delivers a new full-screen experience; AirDrop makes it easier to share and connect; Enhancements to the keyboard;... compatible" + }, + { + "id": 2, + "type": "possible", + "ans": "14 Pro: Available at authorized resellers, A16 Bionic chip, 6-core CPU, 5-core GPU, 16-core Neural Engine; 15 Pro: Starting at $999, A17 Pro chip, 6-core CPU, 6-core GPU, 16-core Neural Engine" + }, + { + "id": 3, + "type": "possible", + "ans": "iPhone 15 pro starts from $999, 6.1-inch screen; iPhone 15 pro max starts from $1199, 6.7-inch screen" + }, + { + "id": 4, + "type": "possible", + "ans": "$4,199.00 or $349.91/mo.per month for 12 mo.*" + }, + { + "id": 5, + "type": "possible", + "ans": "iPhone 15 ($799) or pro ($999) or pro Max ($1199); September 22, 2023" + }, + { + "id": 6, + "type": "possible", + "ans": "4" + }, + { + "id": 7, + "type": "possible", + "ans": "Available early 2024 in the U.S." + }, + { + "id": 8, + "type": "possible", + "ans": "iPad Pro, storage options: 128GB, 256GB, 512GB, 1TB, 2TB; processor type: Apple M2 chip; display features: 11\u2011inch with Liquid Retina display, 12.9\u2011inch with Liquid Retina XDR display" + }, + { + "id": 9, + "type": "possible", + "ans": "iPhone 15; Schedule an in-store pickup" + }, + { + "id": 10, + "type": "possible", + "ans": "Macbook Pro; processor type: Apple M3 chip, Apple M3 Pro chip, Apple M3 Max chip; memory size: 8GB, 16GB, 18GB, 24GB, 36GB, 48GB, 64GB, 96GB, 128GB; storage capacity: 512GB, 1TB, 2TB, 4TB, 8TB" + }, + { + "id": 11, + "type": "possible", + "ans": "sixth-generation iPad Pro 11\u2011inch, iPad Pro 12.9\u2011inch; release date: October 26, 2022; base storage capacity 128 GB, starting price $799" + }, + { + "id": 12, + "type": "golden", + "ans": "Any 2 of 'Send your product to Apple', 'Find an Apple Authorized Service Provider', 'Visit a Genius at an Apple Store', 'Independent Repair Providers', 'Self Service Repair'" + }, + { + "id": 13, + "type": "possible", + "ans": "4, Silver, Starlight, Space Gray, and Midnight" + }, + { + "id": 14, + "type": "possible", + "ans": "Base model:$1599, difference: $1020" + }, + { + "id": 15, + "type": "possible", + "ans": "16" + }, + { + "id": 16, + "type": "possible", + "ans": "2 types, price difference $10" + }, + { + "id": 17, + "type": "golden", + "ans": "Apple Tower Theatre" + }, + { + "id": 18, + "type": "golden", + "ans": "There are trade-in offers." + }, + { + "id": 19, + "type": "golden", + "ans": "If you can dream it, Mac can do it; Mind-blowing. Head-turning" + }, + { + "id": 20, + "type": "possible", + "ans": "From $899 or $37.45/mo.per month for 24 mo.months" + }, + { + "id": 21, + "type": "possible", + "ans": "128GB, 256GB, 512GB, 1TB, and 2TB" + }, + { + "id": 22, + "type": "possible", + "ans": "iPhone 13 Pro Max, Up to $500" + }, + { + "id": 23, + "type": "possible", + "ans": "Apple Watch SE From $249, Apple Watch Series 9 From $399" + }, + { + "id": 24, + "type": "possible", + "ans": "$1299.00" + }, + { + "id": 25, + "type": "possible", + "ans": "Apple TV 4K: A15 Bionic chip" + }, + { + "id": 26, + "type": "possible", + "ans": "4K video recording at 24 fps, 25 fps, 30 fps, or 60 fps" + }, + { + "id": 27, + "type": "possible", + "ans": "Available in multiple colors: Space Gray, Blue, Yellow, White, and Orange." + }, + { + "id": 28, + "type": "golden", + "ans": "Yes. Mac mini Apple M2 Pro chip, Configurable to: 19-core GPU" + }, + { + "id": 29, + "type": "possible", + "ans": "Up to 15 hours wireless web" + }, + { + "id": 30, + "type": "possible", + "ans": "11-inch, 128GB from $799, 256GB from $899, 512GB from $1099, 1TB from $1499, and 2TB from $1899." + }, + { + "id": 31, + "type": "golden", + "ans": "Smarter. Brighter. Mightier." + }, + { + "id": 32, + "type": "possible", + "ans": "iPhone 11 Pro Max\tUp to $270" + }, + { + "id": 33, + "type": "possible", + "ans": "Blue, Green, Pink, Silver, Yellow, Orange, Purple" + }, + { + "id": 34, + "type": "possible", + "ans": "Height: 1.2 inches (31 mm), Width: 3.66 inches (93 mm), Depth: 3.66 inches (93 mm); Siri Remote features" + }, + { + "id": 35, + "type": "possible", + "ans": "3, Apple Pencil (2nd generation), Apple Pencil (USB-C), Apple Pencil (1st generation); Apple Pencil (2nd generation) supports Wireless pairing and charging." + }, + { + "id": 36, + "type": "possible", + "ans": "Lauren Daigle, Megan Moroney, Olivia Rodrigo ..." + }, + { + "id": 37, + "type": "golden", + "ans": "iPhone 13 pro: Alpine Green, Silver, Gold, Graphite, Sierra Blue; iPhone 14 pro: Deep Purple, Gold, Silver, Space Black; iPhone 15 pro: Natural Titanium, Blue Titanium, White Titanium, Black Titanium" + }, + { + "id": 38, + "type": "possible", + "ans": "Apple Vision Pro Battery; Apple Vision Pro Travel Case; ZEISS Optical Inserts ..." + }, + { + "id": 39, + "type": "possible", + "ans": "The fastest and easiest way to reset your password is with your iPhone or other trusted Apple device \u2014 one that you're already signed in to with your Apple ID, so that we know that it's yours." + }, + { + "id": 40, + "type": "possible", + "ans": "Device Weight, 21.2\u201322.9 ounces (600\u2013650 g); Built\u2011in Apps: App Store, Encounter Dinosaurs, Files, Freeform, Keynote..." + }, + { + "id": 41, + "type": "possible", + "ans": "$649" + }, + { + "id": 42, + "type": "possible", + "ans": "see https://www.apple.com/watch/compare/, " + } + ] + }, + "ArXiv": { + "notice": " real-time", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "Any paper related to quantum computing (latest)" + }, + { + "id": 1, + "type": "possible", + "ans": "Paper related to quantum computing (latest 2 days)" + }, + { + "id": 2, + "type": "possible", + "ans": "cs.CL paper, " + }, + { + "id": 3, + "type": "possible", + "ans": "math.AT paper, , <authors>, <abstract>" + }, + { + "id": 4, + "type": "possible", + "ans": "22 Dec 2023, 18 (real-time)" + }, + { + "id": 5, + "type": "possible", + "ans": "23081 results, searching in archive quant-ph; 39482 results, search in all archives" + }, + { + "id": 6, + "type": "golden", + "ans": "2 Figures, 8 Tables." + }, + { + "id": 7, + "type": "possible", + "ans": "Latest cs.LG paper" + }, + { + "id": 8, + "type": "possible", + "ans": "'Accessibility update: arXiv now offers papers in HTML format' (December 21, 2023)" + }, + { + "id": 9, + "type": "possible", + "ans": "Latest paper related to neural networks" + }, + { + "id": 10, + "type": "golden", + "ans": "If your submission has not yet become publicly available you may delete or delay it. To do either of these things go to your user page and select either the Delete or Unsubmit icon." + }, + { + "id": 11, + "type": "golden", + "ans": "-----" + }, + { + "id": 12, + "type": "golden", + "ans": "3" + }, + { + "id": 13, + "type": "possible", + "ans": "4" + }, + { + "id": 14, + "type": "golden", + "ans": "3" + }, + { + "id": 15, + "type": "possible", + "ans": "2" + }, + { + "id": 16, + "type": "possible", + "ans": "Latest gravitational waves paper, <summary>" + }, + { + "id": 17, + "type": "golden", + "ans": "Mon, 27 Mar 2023 17:46:54 UTC" + }, + { + "id": 18, + "type": "golden", + "ans": "2 formulas, the second one is loss function" + }, + { + "id": 19, + "type": "possible", + "ans": "Cornell University, 16071 UNDERGRADUATE STUDENTS" + }, + { + "id": 20, + "type": "possible", + "ans": "stat.ML paper, <abstract>" + }, + { + "id": 21, + "type": "possible", + "ans": "cs paper related to 'neural networks for image processing'," + }, + { + "id": 22, + "type": "possible", + "ans": "To: arch-ive@arxiv.org \\n Subject: subscribe Your Full Name" + }, + { + "id": 23, + "type": "possible", + "ans": "eess.SY paper related to autonomous vehicles" + }, + { + "id": 24, + "type": "possible", + "ans": "paper related to graph neural networks" + }, + { + "id": 25, + "type": "golden", + "ans": "6, arXiv Logo Shirt, arXiv Logo Mug, arXiv is Open Science, Gift cards, arXiv Morning Mug, arXiv Forever" + }, + { + "id": 26, + "type": "possible", + "ans": "astro-ph.EP paper related to climate change modeling last week" + }, + { + "id": 27, + "type": "golden", + "ans": "Econometrics (econ.EM), General Economics (econ.GN), and Theoretical Economics (econ.TH)" + }, + { + "id": 28, + "type": "possible", + "ans": "'Persona-Coded Poly-Encoder: Persona-Guided Multi-Stream Conversational Sentence Scoring', Access include: HTML (experimental)" + }, + { + "id": 29, + "type": "possible", + "ans": "240+ (search by title)" + }, + { + "id": 30, + "type": "golden", + "ans": "Accepted figure formats: PostScript (PS, EPS) \u2014 requires LaTeX processing; JPEG, GIF, PNG or PDF figures \u2014 requires PDFLaTeX processing" + }, + { + "id": 31, + "type": "golden", + "ans": "7 papers" + }, + { + "id": 32, + "type": "possible", + "ans": "latest nlin.CD paper, <abstract>, <date>" + }, + { + "id": 33, + "type": "possible", + "ans": "eess.SY paper" + }, + { + "id": 34, + "type": "possible", + "ans": "Finite spectral triples for the fuzzy torus, Authors: John W. Barrett, James Gaunt, <abstract>" + }, + { + "id": 35, + "type": "possible", + "ans": "paper related to Quantum Physics" + }, + { + "id": 36, + "type": "golden", + "ans": "'CVPR 2023': 48 results; 'CVPR2023': 9 results" + }, + { + "id": 37, + "type": "golden", + "ans": "Ramin Zabih, Yoav Artzi, Stephanie Orphan, Steinn Sigurdsson, and Charles Frankston." + }, + { + "id": 38, + "type": "possible", + "ans": "'Attention arXiv users: Re-implemented RSS', January 31, 2024, <summary>" + }, + { + "id": 39, + "type": "golden", + "ans": "One of the main goals of developing such models is to improve their ability to understand and generate natural language text, particularly in more complex and nuanced scenarios." + }, + { + "id": 40, + "type": "possible", + "ans": "astro-ph.SR paper, latest 3 days" + }, + { + "id": 41, + "type": "golden", + "ans": "QR code image, Action: add to chart" + }, + { + "id": 42, + "type": "golden", + "ans": "'Using a Support-Vector Machine for Japanese-to-English Translation of Tense, Aspect, and Modality'" + } + ] + }, + "BBC News": { + "notice": " real time, answer based on American BBC", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "<report> (about developments in renewable energy technologies in the UK)" + }, + { + "id": 1, + "type": "possible", + "ans": "<summary> (about latest health-related article)" + }, + { + "id": 2, + "type": "possible", + "ans": "<article> (within the last 2 days)" + }, + { + "id": 3, + "type": "possible", + "ans": "Mauritius Open; 5" + }, + { + "id": 4, + "type": "possible", + "ans": "<summary> (economic implications of climate change in Europe)" + }, + { + "id": 5, + "type": "golden", + "ans": "This recent climate change has been caused by human activity, mainly the widespread use of fossil fuels - coal, oil and gas - in homes, factories and transport." + }, + { + "id": 6, + "type": "possible", + "ans": "Latest news in Innovation - Technology" + }, + { + "id": 7, + "type": "possible", + "ans": "Analyse the first image in story." + }, + { + "id": 8, + "type": "possible", + "ans": "CPTPP trade deal, <summary>; 16th July 2023" + }, + { + "id": 9, + "type": "possible", + "ans": "Taylor Swift" + }, + { + "id": 10, + "type": "possible", + "ans": "News about UK's plan to tackle climate change" + }, + { + "id": 11, + "type": "possible", + "ans": "12 teams, 15:00, 2 Jan 2024" + }, + { + "id": 12, + "type": "possible", + "ans": "ramen, Tokyo" + }, + { + "id": 13, + "type": "possible", + "ans": "<summary> (about Trump)" + }, + { + "id": 14, + "type": "possible", + "ans": "<title>, <author>, <summary> (impact of the recent tech industry layoffs on the global economy)" + }, + { + "id": 15, + "type": "possible", + "ans": "Name that whale! How AI aces animal spotting" + }, + { + "id": 16, + "type": "possible", + "ans": "<summary> (Brexit negotiations)" + }, + { + "id": 17, + "type": "possible", + "ans": "2" + }, + { + "id": 18, + "type": "possible", + "ans": "2 of them: Believe in Magic, The Gift, Vishal, A Very British Cult, People Who Knew Me, History's Secret Heroes" + }, + { + "id": 19, + "type": "possible", + "ans": "17th - 18th February 2024" + }, + { + "id": 20, + "type": "possible", + "ans": "Earth - Green Living, <article>, <summary>" + }, + { + "id": 21, + "type": "possible", + "ans": "News - World, <headline>, <region>" + }, + { + "id": 22, + "type": "possible", + "ans": "Business, <article>, <summary>, economic implications" + }, + { + "id": 23, + "type": "possible", + "ans": "Innovation - Science & Health, <article>, <summary>" + }, + { + "id": 24, + "type": "possible", + "ans": "Search for space exploration, eg. SpaceX blasts private firm's lunar lander into orbit" + }, + { + "id": 25, + "type": "possible", + "ans": "Sport - Football - Leagues & Cups - Premier League, <article>" + }, + { + "id": 26, + "type": "possible", + "ans": "Earth - Weather & Science, eg. Indonesia hit by some of strongest winds recorded" + }, + { + "id": 27, + "type": "possible", + "ans": "Archaeological discoveries: eg, Historical 10,000BC artefacts found on road project, Significant discoveries" + }, + { + "id": 28, + "type": "golden", + "ans": "Business - Market Data, Source: Morningstar" + }, + { + "id": 29, + "type": "possible", + "ans": "Audio - Podcasts - New Releases ..." + }, + { + "id": 30, + "type": "possible", + "ans": "Culture - Film & TV, <review>, <summary>" + }, + { + "id": 31, + "type": "possible", + "ans": "Sunday 11th February, Aston Villa 1:2 Manchester United" + }, + { + "id": 32, + "type": "possible", + "ans": "Innovation - Artificial Intelligence, <headline>, <companies>" + }, + { + "id": 33, + "type": "possible", + "ans": "News - Israel-Gaza War, <article>, <summary>" + }, + { + "id": 34, + "type": "possible", + "ans": "Sydney, New York, Tenerife ..." + }, + { + "id": 35, + "type": "possible", + "ans": "News - World - Asia, <article>, <summary>" + }, + { + "id": 36, + "type": "possible", + "ans": "News - World - Africa, <article>, <summary>" + }, + { + "id": 37, + "type": "possible", + "ans": "Culture - Books, eg, Sloane Crosley: What to do when you lose a friend" + }, + { + "id": 38, + "type": "possible", + "ans": "Earth - Weather & Science, article about severe weather, eg, You can't hear it, but this sound can reveal that a tornado is on its way" + }, + { + "id": 39, + "type": "possible", + "ans": "eg, 2024-01-30: Chepstow Summer Sessions Handicap Chase, 13 runners" + }, + { + "id": 40, + "type": "possible", + "ans": "News - Israel-Gaza War, <article>" + }, + { + "id": 41, + "type": "possible", + "ans": "Sport - Golf - Leaderboard - Women's Majors, most in top20: American, best in Australian: Grace Kim in 36" + } + ] + }, + "Booking": { + "notice": " real-time, check task requirements, date and other requirements (may need sort)", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "Be Local" + }, + { + "id": 1, + "type": "possible", + "ans": "OYO 3755 Sweet Home, US$14" + }, + { + "id": 2, + "type": "possible", + "ans": "Berlin Heritage Inn, US$549 for 3 adults and 2 rooms" + }, + { + "id": 3, + "type": "possible", + "ans": "Freehand Los Angeles" + }, + { + "id": 4, + "type": "possible", + "ans": "Moonlight Residency, Breakfast included, US$14" + }, + { + "id": 5, + "type": "possible", + "ans": "Palasari Villa, free WiFi and air conditioning" + }, + { + "id": 6, + "type": "possible", + "ans": "La Quinta by Wyndham LAX" + }, + { + "id": 7, + "type": "possible", + "ans": "Fragrance Hotel - Ocean View" + }, + { + "id": 8, + "type": "possible", + "ans": "OYO Flagship Valasaravakkam" + }, + { + "id": 9, + "type": "possible", + "ans": "The Birds Nest Hostel; Umbrella Properties London Excel; Umbrella Properties London Woolwich" + }, + { + "id": 10, + "type": "possible", + "ans": "Villa Alessandra" + }, + { + "id": 11, + "type": "possible", + "ans": "Pendry Chicago" + }, + { + "id": 12, + "type": "possible", + "ans": "Mode Paris Aparthotel" + }, + { + "id": 13, + "type": "possible", + "ans": "Le Bellevue" + }, + { + "id": 14, + "type": "possible", + "ans": "Nolinski Paris" + }, + { + "id": 15, + "type": "possible", + "ans": "Rhinoceros; rating 9.2; cost US$5771; Amenities: air conditioning, free WiFi..." + }, + { + "id": 16, + "type": "possible", + "ans": "Zoku Paris; 48 Avenue de la Porte de Clichy, 17th arr., Paris; US$210 per night" + }, + { + "id": 17, + "type": "possible", + "ans": "Villa-des-Pr\u00e9s" + }, + { + "id": 18, + "type": "possible", + "ans": "Cromwell Serviced Apartments; Cromwell Serviced Apartments is an apartment featuring rooms with free Wifi and air conditioning in the center of London" + }, + { + "id": 19, + "type": "possible", + "ans": "H\u00f4tel des Arts Montmartre; Bulgari Hotel Paris; Four Seasons Hotel George V Paris" + }, + { + "id": 20, + "type": "possible", + "ans": "47 Boutique Hotel, 8.6 ratings, breakfast included, free cancellation" + }, + { + "id": 21, + "type": "possible", + "ans": "Lexie Suites, 9.1 ratings, free Wi-Fi and parking" + }, + { + "id": 22, + "type": "possible", + "ans": "nhow Amsterdam Rai, 9.0 ratings, bicycle rentals" + }, + { + "id": 23, + "type": "possible", + "ans": "The Peninsula Tokyo, 9.2 ratings, Spa and Fitness center" + }, + { + "id": 24, + "type": "possible", + "ans": "Unite Hostel Barcelona, 8.2 ratings, 400m from beach, free Wi-Fi and breakfast" + }, + { + "id": 25, + "type": "possible", + "ans": "The Homeboat Company Parque das Na\u00e7\u00f5es-Lisboa, 9.5 ratings, airport shuttle, breakfast included" + }, + { + "id": 26, + "type": "possible", + "ans": "InterContinental Paris Le Grand, an IHG Hotel, US$2208, 8.6 ratings, 5-star, parking" + }, + { + "id": 27, + "type": "possible", + "ans": "Nesuto Docklands, 8.9 ratings, free parking and free WiFi" + }, + { + "id": 28, + "type": "possible", + "ans": "Park Regis by Prince Dubai Islands, swimming pool" + }, + { + "id": 29, + "type": "possible", + "ans": "Fairmont Royal York Hotel, 8.3 ratings, fitness center" + }, + { + "id": 30, + "type": "possible", + "ans": "After applying the Breakfast included and Fitness center: 228 hotels" + }, + { + "id": 31, + "type": "possible", + "ans": "Brands has the most hotels: Windsor, Rede Atl\u00e2ntico; Brands has the fewest hotels: Ramada" + }, + { + "id": 32, + "type": "possible", + "ans": "Swimming Pool and Airport Shuttle filters are applied: 1 hotel" + }, + { + "id": 33, + "type": "golden", + "ans": "After you cancel a booking with us, you should get an email confirming the cancellation. Make sure to check your inbox and spam/junk mail folders. If you don\u2019t receive an email within 24 hours, contact the property to confirm they got your cancellation." + }, + { + "id": 34, + "type": "possible", + "ans": "Hotel Adlon Kempinski Berlin, US$1185, CNY 8528" + }, + { + "id": 35, + "type": "possible", + "ans": "Ace Hotel, Downtown Los Angeles; The Hollywood Roosevelt; Hotel Indigo, an IHG Hotel" + }, + { + "id": 36, + "type": "possible", + "ans": "ROMA GONDOLA SRLS, US$81, no breakfast" + }, + { + "id": 37, + "type": "possible", + "ans": "Kappa Senses Ubud, resort, Activity include: Tour or class about local culture" + }, + { + "id": 38, + "type": "possible", + "ans": "ARCOTEL Wimberger Wien, 8.2 ratings, Parking, breakfast included" + }, + { + "id": 39, + "type": "possible", + "ans": "One King West Hotel and Residence, pet-friendly hotel, parking" + }, + { + "id": 40, + "type": "possible", + "ans": "Four Seasons Hotel Shenzhen, US$522, CNY 3760" + }, + { + "id": 41, + "type": "golden", + "ans": "Booking Holdings Inc." + }, + { + "id": 42, + "type": "possible", + "ans": "Heiseikan Shiosaitei Hanatsuki, 9.0 ratings, high: Staff 9.3, Facilities 9.0, Cleanliness 9.4, Comfort 9.3. low: Value for money 8.2, Location 8.7, Free WiFi 8.1" + }, + { + "id": 43, + "type": "possible", + "ans": "Breakfast Included, Wonderful: 9+, Fitness center ..." + } + ] + }, + "Cambridge Dictionary": { + "notice": " pronunciation (If there is no requirement to provide both US & UK pronunciations, one correct pronunciation is sufficient.) Examples should from Cambridge Dictionary if requested.", + "answers": [ + { + "id": 0, + "type": "golden", + "ans": "UK: /s\u0259\u02ccste\u026a.n\u0259\u02c8b\u026al.\u0259.ti/, US: /s\u0259\u02ccste\u026a.n\u0259\u02c8b\u026al.\u0259.t\u032ci/; the quality of being able to continue over a period of time" + }, + { + "id": 1, + "type": "possible", + "ans": "UK: /\u02ccser.\u0259n\u02c8d\u026ap.\u0259.ti/, US: /\u02ccser.\u0259n\u02c8d\u026ap.\u0259.t\u032ci/; the fact of finding interesting or valuable things by chance; There is a real element of serendipity in archaeology." + }, + { + "id": 2, + "type": "possible", + "ans": "UK: /ju\u02d0\u02c8b\u026ak.w\u026a.t\u0259s/, US: /ju\u02d0\u02c8b\u026ak.w\u0259.t\u032c\u0259s/; seeming to be everywhere; Leather is very much in fashion this season, as is the ubiquitous denim." + }, + { + "id": 3, + "type": "possible", + "ans": "UK: /\u02c8tsa\u026at.\u0261a\u026ast/ or /\u02c8za\u026at.\u0261a\u026ast/, US: /\u02c8tsa\u026at.\u0261a\u026ast/ or /\u02c8za\u026at.\u0261a\u026ast/; the general set of ideas, beliefs, feelings, etc. that is typical of a particular period in history; Our methods of working, then, were facilitated and in some ways strongly encouraged by the technologies available to us, the products of a zeitgeist of convergence." + }, + { + "id": 4, + "type": "possible", + "ans": "UK: /\u02c8\u026an.\u0259.ve\u026at/; Above all, this proposal aims to correct the allocative inefficiencies of the existing patent system, while preserving the dynamic incentives to innovate." + }, + { + "id": 5, + "type": "possible", + "ans": "UK: /pr\u0259\u02cckr\u00e6s.t\u026a\u02c8ne\u026a.\u0283\u0259n/, US: /pro\u028a\u02cckr\u00e6s.t\u026a\u02c8ne\u026a.\u0283\u0259n/; Vacillation and procrastination, out of fears of recession or otherwise, would run grave risks." + }, + { + "id": 6, + "type": "golden", + "ans": "\u53ef\u6301\u7eed\u6027; durabilit\u00e9 , viabilit\u00e9" + }, + { + "id": 7, + "type": "possible", + "ans": "UK: /\u0261\u0259\u02c8\u0283t\u00e6lt/, US: /\u0261\u0259\u02c8\u0283t\u0251\u02d0lt/; something such as a structure or experience that, when considered as a whole, has qualities that are more than the total of all its parts; In the comic and cartoon mythoses, however, most gestalts have one default transformation." + }, + { + "id": 8, + "type": "possible", + "ans": "a common animal with four legs, especially kept by people as a pet or to hunt or guard things; a man who is unpleasant or not to be trusted; to follow someone closely and continuously" + }, + { + "id": 9, + "type": "possible", + "ans": "UK: /ju\u02d0\u02c8f\u0254\u02d0.ri.\u0259/; They were in a state of euphoria for days after they won the prize." + }, + { + "id": 10, + "type": "possible", + "ans": "UK: /\u026am\u02c8pek.\u0259.b\u0259l/, US: /\u026am\u02c8pek.\u0259.b\u0259l/; perfect, with no problems or bad parts; His English is impeccable." + }, + { + "id": 11, + "type": "possible", + "ans": "UK: /\u0259\u02c8mi\u02d0l.j\u0259.re\u026at/, US: /\u0259\u02c8mi\u02d0l.j\u0259.re\u026at/; to make a bad or unpleasant situation better; Foreign aid is badly needed to ameliorate the effects of the drought." + }, + { + "id": 12, + "type": "possible", + "ans": "UK: /r\u026a\u02c8z\u026al.j\u0259ns/, US: /r\u026a\u02c8z\u026al.j\u0259ns/; the ability to be happy, successful, etc. again after something difficult or bad has happened; Trauma researchers emphasize the resilience of the human psyche." + }, + { + "id": 13, + "type": "possible", + "ans": "beatitude; bed of roses; for fun" + }, + { + "id": 14, + "type": "possible", + "ans": "UK: /k\u0259n\u02c8k\u00e6t.\u0259.ne\u026at/, US: /k\u0259n\u02c8k\u00e6t\u032c.\u0259.ne\u026at/; to put things together as a connected series; The filename is a series of concatenated words with no spaces." + }, + { + "id": 15, + "type": "possible", + "ans": "UK: /p\u00e6n\u02c8dem.\u026ak/, US: /p\u00e6n\u02c8dem.\u026ak/; In some parts of the world malaria is still pandemic." + }, + { + "id": 16, + "type": "possible", + "ans": "UK: /\u02c8kr\u026ap.t\u0259\u028a\u02cck\u028cr.\u0259n.si/, US: /\u02c8kr\u026ap.to\u028a\u02cck\u025d\u02d0.\u0259n.si/; It is one of several prominent efforts to enable complex financial functions in a cryptocurrency; Vice versa, a cryptocurrency can be a legal tender, in which case it is not a virtual currency." + }, + { + "id": 17, + "type": "golden", + "ans": "2" + }, + { + "id": 18, + "type": "golden", + "ans": "behaves themselves; be on their best behaviour" + }, + { + "id": 19, + "type": "golden", + "ans": "Microsoft" + }, + { + "id": 20, + "type": "possible", + "ans": "UK: /\u02c8\u00e6l.tru.\u026a.z\u0259m/, US: /\u02c8\u00e6l.tru.\u026a.z\u0259m/; Def: willingness to do things that bring advantages to others, even if it results in disadvantage for yourself; She's not known for her altruism." + }, + { + "id": 21, + "type": "golden", + "ans": "ef\u00edmero" + }, + { + "id": 22, + "type": "possible", + "ans": "UK: /\u02cckw\u026an.t\u026a\u02c8sen.\u0283\u0259l/, US:/\u02cckw\u026an.t\u026a\u02c8sen.\u0283\u0259l/; Def: being the most typical example or most important part of something; Sheep's milk cheese is the quintessential Corsican cheese." + }, + { + "id": 23, + "type": "possible", + "ans": "US: /m\u0259\u02c8t\u026ak.j\u0259.l\u0259s/; Many hours of meticulous preparation have gone into writing the book." + }, + { + "id": 24, + "type": "possible", + "ans": "UK: /\u02c8rev.\u0259r.i/, US:/\u02c8rev.\u025a.i/; He was lost in reverie until he suddenly heard someone behind him." + }, + { + "id": 25, + "type": "possible", + "ans": "Meaning 1: a pleasant musical sound made by different notes being played or sung at the same time; Meaning 2: a situation in which people are peaceful and agree with each other, or when things seem right or suitable together" + }, + { + "id": 26, + "type": "golden", + "ans": "\u6000\u65e7" + }, + { + "id": 27, + "type": "possible", + "ans": "UK: /\u02c8s\u0252l.\u026a.t\u0283u\u02d0d/, US: /\u02c8s\u0251\u02d0.l\u0259.tu\u02d0d/; the situation of being alone without other people; After months of solitude at sea it felt strange to be in company." + }, + { + "id": 28, + "type": "golden", + "ans": "Synonyms: feel dizzy; whirl; spin; reel" + }, + { + "id": 29, + "type": "possible", + "ans": "Action: finish an easy Image quiz about Animals" + }, + { + "id": 30, + "type": "possible", + "ans": "Present perfect simple: uses; I\u2019ve been there a couple of times before; We haven\u2019t met before, have we?; Have you ever tried to write your name and address with your left hand?" + }, + { + "id": 31, + "type": "possible", + "ans": "She might sell her house; We could have lunch early; It may be possible for him to get home tonight." + }, + { + "id": 32, + "type": "possible", + "ans": "Article: 'Less or fewer?'; I do less work at weekends than I used to; Better cycle routes would mean fewer cars and fewer accidents." + }, + { + "id": 33, + "type": "possible", + "ans": "Cambridge University Press published this book. (active); This book was published by Cambridge University Press. (passive)" + }, + { + "id": 34, + "type": "possible", + "ans": "This car is more expensive than my last one; Joe used to be the slowest runner in the class." + }, + { + "id": 35, + "type": "possible", + "ans": "ahead of; except for; instead of; owing to; apart from; in addition to ..." + }, + { + "id": 36, + "type": "possible", + "ans": "Example: direct: \u2018I\u2019m tired,\u2019 I said; indirect: I told them (that) I was tired." + }, + { + "id": 37, + "type": "possible", + "ans": "<understandings>, Countable nouns: I have a sister and a brother. That was an excellent meal. The lion roared. Uncountable nouns: I hope we have nice weather. The weather was awful last summer..." + }, + { + "id": 38, + "type": "possible", + "ans": "Action: finish a recommended Grammar quiz" + }, + { + "id": 39, + "type": "possible", + "ans": "Action: finish the Word Scramble game in the Plus section" + }, + { + "id": 40, + "type": "possible", + "ans": "UK: /\u02c8m\u026at.\u026a.\u0261e\u026at/, US: /\u02c8m\u026at\u032c.\u0259.\u0261e\u026at/; to make something less harmful, unpleasant, or bad; It is unclear how to mitigate the effects of tourism on the island." + }, + { + "id": 41, + "type": "possible", + "ans": "Shop: Cambridge Dictionary organic cotton Hoodie; On top of the world organic cotton T shirt - white writing variety; Multitasking Mug" + }, + { + "id": 42, + "type": "golden", + "ans": "Action: Click English (UK), change language to: Deutsch" + } + ] + }, + "Coursera": { + "notice": "", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "Rapid Prototyping Using 3D Printing, Specialization" + }, + { + "id": 1, + "type": "possible", + "ans": "Python for Data Science, AI & Development" + }, + { + "id": 2, + "type": "possible", + "ans": "Learn Spanish: Basic Spanish Vocabulary, Specialization; Spanish Vocabulary: Meeting People; Spanish Vocabulary: Cultural Experience; Spanish Vocabulary: Sports, Travel, and the Home; Spanish Vocabulary: Careers and Social Events; Spanish Vocabulary Project" + }, + { + "id": 3, + "type": "possible", + "ans": "Data Science with NumPy, Sets, and Dictionaries; Duke University" + }, + { + "id": 4, + "type": "possible", + "ans": "Business Foundations, Specialization" + }, + { + "id": 5, + "type": "possible", + "ans": "Coding for Everyone: C and C++, Specialization; Outcomes: Learn in-demand skills from university and industry experts; Master a subject or tool with hands-on projects; Develop a deep understanding of key concepts; Earn a career certificate from University of California, Santa Cruz" + }, + { + "id": 6, + "type": "possible", + "ans": "Fundamentals of Machine Learning for Healthcare; 14 hours (approximately); 19 quizzes" + }, + { + "id": 7, + "type": "possible", + "ans": "Reinforcement Learning, Specialization; University of Alberta; 3.3K reviews" + }, + { + "id": 8, + "type": "possible", + "ans": "Introducci\u00f3n a Data Science: Programaci\u00f3n Estad\u00edstica con R; Taught in Spanish" + }, + { + "id": 9, + "type": "possible", + "ans": "Artificial Intelligence: Ethics & Societal Challenges" + }, + { + "id": 10, + "type": "possible", + "ans": "Introduction to Artificial Intelligence (AI)" + }, + { + "id": 11, + "type": "possible", + "ans": "Project Management, Specialization; Felipe M. \"To be able to take courses at my own pace and rhythm has been an amazing experience. I can learn whenever it fits my schedule and mood.\"" + }, + { + "id": 12, + "type": "possible", + "ans": "Introduction to Java" + }, + { + "id": 13, + "type": "possible", + "ans": "Python 3 Programming, Specialization; Learn Python 3 basics, from the basics to more advanced concepts like lists and functions; Practice and become skilled at solving problems and fixing errors in your code; Gain the ability to write programs that fetch data from internet APIs and extract useful information." + }, + { + "id": 14, + "type": "possible", + "ans": "Agile Project Management" + }, + { + "id": 15, + "type": "possible", + "ans": "85%; 2-star" + }, + { + "id": 16, + "type": "possible", + "ans": "Xi Yang; Introduction to Finance: The Role of Financial Markets" + }, + { + "id": 17, + "type": "possible", + "ans": "23" + }, + { + "id": 18, + "type": "possible", + "ans": "Programming with JavaScript" + }, + { + "id": 19, + "type": "possible", + "ans": "Instructor: Paul Bloom; Yale University; 14 hours" + }, + { + "id": 20, + "type": "possible", + "ans": "Introduction to Supply Chain Finance & Blockchain Technology; New York Institute of Finance; Instructors: Oliver Belin, Jack Farmer; <summary of main goals>" + }, + { + "id": 21, + "type": "possible", + "ans": "Foundations of Digital Marketing and E-commerce; Google; Instructors: Google Career Certificates; <outcomes>; duration: 1 - 4 weeks or 25 hours (approximately)" + }, + { + "id": 22, + "type": "possible", + "ans": "Human Resource Management: HR for People Managers Specialization; University of Minnesota; Course 1: Preparing to Manage Human Resources; Course 2: Recruiting, Hiring, and Onboarding Employees; Course 3: Managing Employee Performance; Course 4: Managing Employee Compensation; Course 5: Human Resources Management Capstone: HR for People Managers" + }, + { + "id": 23, + "type": "possible", + "ans": "Artificial Intelligence: Ethics & Societal Challenges; Lund University; 4.6 stars; Instructors: Maria Hedlund, Lena Lindstr\u00f6m, Erik Persson" + }, + { + "id": 24, + "type": "possible", + "ans": "Introduction to Sustainability; University of Illinois at Urbana-Champaign; Instructors: Dr. Jonathan Tomkin; duration: Approx. 25 hours to complete, 3 weeks at 8 hours a week" + }, + { + "id": 25, + "type": "possible", + "ans": "Understanding Einstein: The Special Theory of Relativity; <topic>; Approx. 80 hours to complete" + }, + { + "id": 26, + "type": "possible", + "ans": "Renewable Energy Specialization; Instructors: Stephen R. Lawrence, Paul Komor; 2 months" + }, + { + "id": 27, + "type": "possible", + "ans": "Data Visualization with Tableau Specialization; University of California, Davis; <skills>" + }, + { + "id": 28, + "type": "possible", + "ans": "Explore Einstein's theories of Relativity using Wolfram; Coursera Project Network; 2 hours; <main subjects>" + }, + { + "id": 29, + "type": "possible", + "ans": "$399/year, discount: 59 / month * 12 - 399 = 309; Google, IBM, and Imperial College London ..." + }, + { + "id": 30, + "type": "possible", + "ans": "3 stars: 2.5%; 1 star has the lowest percentage" + }, + { + "id": 31, + "type": "possible", + "ans": "52.6%" + }, + { + "id": 32, + "type": "possible", + "ans": "568 results" + }, + { + "id": 33, + "type": "possible", + "ans": "Introduction and Programming with IoT Boards; Instructor: James Won-Ki HONG; <summary>" + }, + { + "id": 34, + "type": "possible", + "ans": "Instructor: Richard Skolnik; <summary> of bio; no other course" + }, + { + "id": 35, + "type": "possible", + "ans": "Introduction to Sustainability; <objectives>; Instructor: Dr. Jonathan Tomkin" + }, + { + "id": 36, + "type": "possible", + "ans": "Master of Advanced Study in Engineering; UC Berkeley College of Engineering; Fall 2024; March 1, 2024: Fall 2024 Priority Application Deadline; April 1, 2024: Fall 2024 Final Application Deadline" + }, + { + "id": 37, + "type": "possible", + "ans": "Business Analytics with Excel: Elementary to Advanced; Cybersecurity for Everyone; Financial Markets ..." + }, + { + "id": 38, + "type": "golden", + "ans": "Macquarie University; The University of Melbourne; The University of Sydney; University of Western Australia; UNSW Sydney (The University of New South Wales)" + }, + { + "id": 39, + "type": "golden", + "ans": "6 videos; Introduction; Space Debris; Mitigation; Measurements; Protection; Atmospheric Re-entry" + }, + { + "id": 40, + "type": "possible", + "ans": "Coursera for Business: Strengthen critical skills with content you can trust; Develop, retain, and advance critical talent; Lower training costs without sacrificing quality; Track and measure skills to demonstrate ROI; Coursera for Teams: Upskill 5 to 125 employees; Unlimited access to 10,250+ learning opportunities; Program setup and launch tools; Analytics and benchmarking dashboard" + }, + { + "id": 41, + "type": "possible", + "ans": "BSc Computer Science, University of London; Bachelor of Science in Cybersecurity Technology, University of Maryland Global Campus; Bachelor of Information Technology, Illinois Institute of Technology" + } + ] + }, + "ESPN": { + "notice": " real-time", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "<standings> (NBA Eastern Conference)" + }, + { + "id": 1, + "type": "possible", + "ans": "<article> (trades), maybe no article" + }, + { + "id": 2, + "type": "possible", + "ans": "<score> (Milwaukee Bucks vs xxx); <highlight>" + }, + { + "id": 3, + "type": "possible", + "ans": "<score> (most recent NBA game)" + }, + { + "id": 4, + "type": "possible", + "ans": "<score> (yesterday)" + }, + { + "id": 5, + "type": "possible", + "ans": "<player>; <PTS>; <team>; <position> (eg, James Harden; scored 35 points; LA Clippers; Shooting Guard (SG))" + }, + { + "id": 6, + "type": "possible", + "ans": "Los Angeles Lakers vs Boston Celtics, 115 - 126; Kristaps Porzingis" + }, + { + "id": 7, + "type": "possible", + "ans": "<score> (latest, Los Angeles Lakers vs xxx); <summary>" + }, + { + "id": 8, + "type": "possible", + "ans": "Joel Embiid (PHI) with 34.4 PPG, Luka Doncic (DAL) with 32.9 PPG, and Giannis Antetokounmpo (MIL) with 31.4 PPG." + }, + { + "id": 9, + "type": "golden", + "ans": "10 teams have Los Angeles in their name; 2 teams are NBA" + }, + { + "id": 10, + "type": "possible", + "ans": "<score>; <summary> (latest college football championship game)" + }, + { + "id": 11, + "type": "golden", + "ans": "30; New York Knicks; New Orleans Pelicans" + }, + { + "id": 12, + "type": "possible", + "ans": "<League 1>; <League 2>; <League 3>" + }, + { + "id": 13, + "type": "possible", + "ans": "<headline>; <summary>" + }, + { + "id": 14, + "type": "possible", + "ans": "News about NBA trades" + }, + { + "id": 15, + "type": "golden", + "ans": "(US Time) Bucks vs Knicks, 122 - 129; Warriors vs Nuggets, 114 - 120; Celtics vs Lakers, 126 - 115; 76ers vs Heat, 113 - 119; Mavericks vs Suns, 128 - 114" + }, + { + "id": 16, + "type": "possible", + "ans": "teams and current standings" + }, + { + "id": 17, + "type": "golden", + "ans": "Boston Celtics; San Antonio Spurs" + }, + { + "id": 18, + "type": "golden", + "ans": "31 (in ESPN America)" + }, + { + "id": 19, + "type": "golden", + "ans": "Jrue Holiday" + }, + { + "id": 20, + "type": "possible", + "ans": "For Western, rebounds: Domantas Sabonis; assists: Luka Doncic" + }, + { + "id": 21, + "type": "possible", + "ans": "<score> within 3 days; <highlight>" + }, + { + "id": 22, + "type": "possible", + "ans": "Team transaction: eg, February 1, TRANSACTION: Dallas Mavericks, Assigned F Olivier-Maxence Proster to the Texas Legends of the G League." + }, + { + "id": 23, + "type": "possible", + "ans": "NBA <score>, latest, Miami Heat - New York Knicks, eg, January 28, 2024, 109 - 125, Top rebounder: B. Adebayo, P. Achiuwa" + }, + { + "id": 24, + "type": "possible", + "ans": "NFL <score>, latest, eg, January 29, 2024, Chiefs - Ravens, 17 - 10" + }, + { + "id": 25, + "type": "possible", + "ans": "NBA game, latest, eg, February 2, 2024, Lakers - Celtics, 114 - 105, most assist: 14, D. Russell, position: PG, team: Los Angeles Lakers" + }, + { + "id": 26, + "type": "possible", + "ans": "NBA game, yesterday, eg, January 26, 2024, Philadelphia - Indiana, 134 - 122, winner high 26 - loser high 31; Denver - New York, 122 - 84, winner high 26 - loser high 31; Chicago - Los Angeles, 141 - 132, winner high 29 - loser high 32" + }, + { + "id": 27, + "type": "golden", + "ans": "30 teams in search results, 1 team Vegas Golden Knights (NHL)" + }, + { + "id": 28, + "type": "golden", + "ans": "30 teams in search results, Kansas City Royals" + }, + { + "id": 29, + "type": "possible", + "ans": "<headline> today" + }, + { + "id": 30, + "type": "possible", + "ans": "NHL Standings 2023-24, top - bottom, Eastern Conference: New York Rangers - Columbus Blue Jackets; Western Conference: Vancouver Canucks - Chicago Blackhawks; Division: ATLANTIC, Boston Bruins - Montreal Canadiens; METROPOLITAN: New York Rangers - Columbus Blue Jackets; CENTRAL: Dallas Stars - Chicago Blackhawks; PACIFIC: Vancouver Canucks - San Jose Sharks" + }, + { + "id": 31, + "type": "golden", + "ans": "Carlos Rodon, 255 lbs" + }, + { + "id": 32, + "type": "possible", + "ans": "NHL <score> yesterday" + }, + { + "id": 33, + "type": "possible", + "ans": "Article, '2023 NFL MVP: Ranking five finalists, plus stats'" + }, + { + "id": 34, + "type": "possible", + "ans": "Philadelphia 76ers - Injuries, latest" + }, + { + "id": 35, + "type": "possible", + "ans": "next game of Los Angeles Lakers, <price>" + }, + { + "id": 36, + "type": "possible", + "ans": "<games>; Inter Miami CF, <results>" + }, + { + "id": 37, + "type": "possible", + "ans": "1471" + }, + { + "id": 38, + "type": "possible", + "ans": "54/58 = 93.1%, no other players, https://www.espn.com/nba/team/stats/_/name/lal/los-angeles-lakers" + }, + { + "id": 39, + "type": "possible", + "ans": "check IR on https://www.espn.com/nfl/team/depth/_/name/nyj/new-york-jets" + }, + { + "id": 40, + "type": "possible", + "ans": "Bracket Predictor, Bracket Analyzer, Custom Dollar Value Generator" + }, + { + "id": 41, + "type": "golden", + "ans": "Chicago Bears, Detroit Lions, Green Bay Packers, and Minnesota Vikings" + }, + { + "id": 42, + "type": "possible", + "ans": "check America East Conference on https://www.espn.com/mens-college-basketball/standings" + }, + { + "id": 43, + "type": "possible", + "ans": "espnW Rankings Class of 2023, Judea Watkins from USC, Mikaylah Williams from LSU, Jadyn Donovan from Duke" + } + ] + }, + "GitHub": { + "notice": " repos are real-time", + "answers": [ + { + "id": 0, + "type": "golden", + "ans": "resource-watch/resource-watch" + }, + { + "id": 1, + "type": "possible", + "ans": "google/yggdrasil-decision-forests" + }, + { + "id": 2, + "type": "possible", + "ans": "myshell-ai/OpenVoice" + }, + { + "id": 3, + "type": "golden", + "ans": "48GB" + }, + { + "id": 4, + "type": "possible", + "ans": "<repo> (use advanced search like 'javascript created:>2023-12-10 language:JavaScript')" + }, + { + "id": 5, + "type": "possible", + "ans": "<repo> (stars:\"> 500\" language:Python), then choose recently undated" + }, + { + "id": 6, + "type": "possible", + "ans": "blocknetdx/blocknet; laanwj, sipa, theuni" + }, + { + "id": 7, + "type": "golden", + "ans": "classifier_utils.py and squad_utils.py" + }, + { + "id": 8, + "type": "golden", + "ans": "Latest v4.0.2 on Jun 17, 2021" + }, + { + "id": 9, + "type": "possible", + "ans": "<repo> (stars:>=50 created:>=xxxx-xx-xx)" + }, + { + "id": 10, + "type": "golden", + "ans": "$100 per year; Code completions, Chat, and more for indie developers and freelancers." + }, + { + "id": 11, + "type": "possible", + "ans": "TheAIDojo/AI-for-Climate-Change; Jupyter Notebook; Repository of notebooks and associated code that covers the fundamental concepts of deep learning and its application to climate science." + }, + { + "id": 12, + "type": "possible", + "ans": "v29.0.0-alpha.5, 19 hours ago (real-time release)" + }, + { + "id": 13, + "type": "possible", + "ans": "microsoft/ML-For-Beginners" + }, + { + "id": 14, + "type": "possible", + "ans": "bpasero; jrieken; mjbvz" + }, + { + "id": 15, + "type": "possible", + "ans": "desireevl/awesome-quantum-computing" + }, + { + "id": 16, + "type": "golden", + "ans": "3" + }, + { + "id": 17, + "type": "possible", + "ans": "microsoft/terminal; The new Windows Terminal and the original Windows console host, all in the same place!" + }, + { + "id": 18, + "type": "golden", + "ans": "OpenCV" + }, + { + "id": 19, + "type": "possible", + "ans": "scrapy/scrapy" + }, + { + "id": 20, + "type": "golden", + "ans": "'Chat in GitHub Mobile is coming soon.' OR 'We do not have a set timeline for making Copilot Chat available on mobile. We\u2019ll continue to update this page with the latest information on new capabilities for various plans.'" + }, + { + "id": 21, + "type": "possible", + "ans": "With AI-powered application security testing tools embedded in your development workflow, GitHub Advanced Security outperforms non-native add-ons by delivering 7x faster remediation rates for identified vulnerabilities." + }, + { + "id": 22, + "type": "possible", + "ans": "<repo> (natural language processing language:Ruby)" + }, + { + "id": 23, + "type": "golden", + "ans": "edit the .zshrc file and set the ZSH_THEME variable to \"agnoster\"" + }, + { + "id": 24, + "type": "possible", + "ans": "recently closed issue in repo angular/angular: https://github.com/angular/angular/issues?q=is%3Aissue+is%3Aclosed" + }, + { + "id": 25, + "type": "possible", + "ans": "<repo> (virtual reality stars:>=200), <summary>" + }, + { + "id": 26, + "type": "golden", + "ans": "Create a pull request. Resolve a merge conflict. Create a merge conflict. Merge your pull request." + }, + { + "id": 27, + "type": "possible", + "ans": "<repo> (language:Ruby stars:>1000)" + }, + { + "id": 28, + "type": "possible", + "ans": "<repo> (language:JavaScript created:>2023-12-29), sort by Most stars" + }, + { + "id": 29, + "type": "golden", + "ans": "Unlimited" + }, + { + "id": 30, + "type": "possible", + "ans": "eg, aptos-labs/aptos-core, contributors: davidiw, gregnazario, JoshLind, bmwill, rustielin" + }, + { + "id": 31, + "type": "possible", + "ans": "Tensorflow latest commit" + }, + { + "id": 32, + "type": "possible", + "ans": "<repo> (game development language:C# stars:>150), <features>" + }, + { + "id": 33, + "type": "possible", + "ans": "Philips builds and deploys digital health technology faster with innersource on GitHub. Shopify keeps pushing eCommerce forward with help from GitHub tools." + }, + { + "id": 34, + "type": "possible", + "ans": "kexinhuang12345/DeepPurpose" + }, + { + "id": 35, + "type": "golden", + "ans": "18.2.0 (June 14, 2022)" + }, + { + "id": 36, + "type": "possible", + "ans": "<repo> (AI agriculture created:2022)" + }, + { + "id": 37, + "type": "possible", + "ans": "The AI coding assistant elevating developer workflows. Get AI-based suggestions in real time. Docs that feel tailored for you." + }, + { + "id": 38, + "type": "golden", + "ans": "WerWolv/ImHex" + }, + { + "id": 39, + "type": "possible", + "ans": "find info on https://github.com/trending/developers" + }, + { + "id": 40, + "type": "golden", + "ans": "Perform Action. email 'test123@gmail.com' already exists" + } + ] + }, + "Google Flights": { + "notice": " real-time, round trip (only provide the departure flight is OK). Answers can from best options (or sort). Because the best options are in line with the user's needs, if you go through the sort you may get a very unreasonable flight (maybe long duration).", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "Aer Lingus 11:40am - 4:45pm, $412 (real-time)" + }, + { + "id": 1, + "type": "possible", + "ans": "Air France 5:30\u202fPM \u2013 8:25\u202fAM (+1), United 6:30\u202fPM \u2013 9:55\u202fAM(+1), Delta 12:00\u202fPM \u2013 8:10\u202fAM(+1)... (real-time)" + }, + { + "id": 2, + "type": "possible", + "ans": "Tap Air Portugal 10:00\u202fPM \u2013 5:30\u202fPM(+1), $355 (real-time)" + }, + { + "id": 3, + "type": "possible", + "ans": "WestJet 9:55\u202fAM \u2013 4:34\u202fPM, emission: 225 kg CO2, $704 (real-time)" + }, + { + "id": 4, + "type": "possible", + "ans": "Norse Atlantic UK 6:10\u202fPM \u2013 6:00\u202fAM(+1), $331, Nonstop (real-time)" + }, + { + "id": 5, + "type": "possible", + "ans": "Scandinavian Airlines 9:45\u202fPM \u2013 4:00\u202fPM(+1), $1456 (real-time)" + }, + { + "id": 6, + "type": "possible", + "ans": "flydubai, Emirates, and AccesRail, 12:40 PM - 8:34 PM(+1), $8991 (real-time)" + }, + { + "id": 7, + "type": "possible", + "ans": "American Airlines, 5:44 AM \u2013 1:25 PM, $1,247 (real-time)" + }, + { + "id": 8, + "type": "possible", + "ans": "Analyse the picture of Price graph (real-time)" + }, + { + "id": 9, + "type": "possible", + "ans": "Air India, LOT, 3:55\u202fPM \u2013 8:35\u202fPM(+1), transfer time: 18 hours 20 mins (real-time, Transfer time only.)" + }, + { + "id": 10, + "type": "possible", + "ans": "Air Canada, 9:15\u202fAM \u2013 4:50\u202fPM(+1), $1169 (real-time)" + }, + { + "id": 11, + "type": "possible", + "ans": "United flight, 11:15\u202fAM \u2013 3:35\u202fPM(+1), $1366, Nonstop (real-time)" + }, + { + "id": 12, + "type": "possible", + "ans": "Norse Atlantic UK, 6:10\u202fPM \u2013 6:00\u202fAM(+1), $757, Nonstop (real-time)" + }, + { + "id": 13, + "type": "possible", + "ans": "Turkish Airlines, 8:00\u202fPM \u2013 8:30\u202fAM(+2), $1142, 1 stop (real-time)" + }, + { + "id": 14, + "type": "possible", + "ans": "Norse Atlantic UK, 6:10\u202fPM \u2013 6:00\u202fAM(+1), $546 (real-time)" + }, + { + "id": 15, + "type": "possible", + "ans": "Only one flight, United flight, 11:15\u202fAM \u2013 3:35\u202fPM(+1), $1316 (real-time)" + }, + { + "id": 16, + "type": "possible", + "ans": "Norse Atlantic UK, Air China, 6:10\u202fPM \u2013 1:40\u202fPM(+2), $671, 2 stops (real-time)" + }, + { + "id": 17, + "type": "possible", + "ans": "Scandinavian Airlines, 5:35\u202fPM \u2013 1:25\u202fPM(+1), $608, 2 stops (real-time)" + }, + { + "id": 18, + "type": "possible", + "ans": "United, 11:15\u202fAM \u2013 3:35\u202fPM(+1), duration 14 hr 20 min, $1316 (real-time)" + }, + { + "id": 19, + "type": "possible", + "ans": "easyJet, 6:35 PM - 8:55 PM, $35, nonstop (real-time)" + }, + { + "id": 20, + "type": "possible", + "ans": "Lufthansa United, 2:40\u202fPM \u2013 12:55\u202fPM(+1), 13 hr 15 min" + }, + { + "id": 21, + "type": "possible", + "ans": "Jetstar JAL, Qantas, 8:10\u202fPM \u2013 10:40\u202fAM(+1), 12 hr 30 min, 1 stop" + }, + { + "id": 22, + "type": "possible", + "ans": "Gol, Aeromexico, 7:00\u202fAM \u2013 10:22\u202fPM, 746 kg CO2" + }, + { + "id": 23, + "type": "possible", + "ans": "Air Canada Lufthansa, 4:25\u202fAM \u2013 4:15\u202fPM; Air India, Air Canada, 6:35\u202fAM \u2013 4:15\u202fPM; ...(1 stop)" + }, + { + "id": 24, + "type": "possible", + "ans": "Etihad ITA, 2:25\u202fAM \u2013 5:45\u202fAM, 6 hr 20 min, Nonstop" + }, + { + "id": 25, + "type": "possible", + "ans": "KLM, 4:25\u202fPM \u2013 9:40\u202fAM(+1), 13 hr 15 min, EZE\u2013AMS, Nonstop, $3912, 3251 kg CO2" + }, + { + "id": 26, + "type": "possible", + "ans": "Royal Jordanian, 2:20\u202fAM \u2013 2:05\u202fPM" + }, + { + "id": 27, + "type": "possible", + "ans": "British Airways, American, 7:45\u202fPM \u2013 6:28\u202fPM(+1), <analyze the price graph>" + }, + { + "id": 28, + "type": "possible", + "ans": "Icelandair, 2:35\u202fPM \u2013 12:00\u202fPM(+1), 1 stop, $1602" + }, + { + "id": 29, + "type": "possible", + "ans": "Only one flight, Lufthansa, 9:00\u202fPM \u2013 2:40\u202fPM(+1), 10 hr 40 min" + }, + { + "id": 30, + "type": "possible", + "ans": "Ethiopian, 2:35\u202fPM \u2013 2:50\u202fPM(+1), 1 stop, $633" + }, + { + "id": 31, + "type": "possible", + "ans": "Qantas, Qatar Airways, AlaskaEmirates, Mar 25, 4:05\u202fPM \u2013 11:59\u202fPM(+1), most: 3 stops" + }, + { + "id": 32, + "type": "possible", + "ans": "Icelandair, 12:50\u202fPM \u2013 6:15\u202fPM, 11 hr 25 min" + }, + { + "id": 33, + "type": "possible", + "ans": "Korean Air, 2:00\u202fPM \u2013 11:15\u202fAM, 13 hr 15 min, 816 kg CO2; EVA AirAir Canada, 8:10\u202fPM \u2013 6:35\u202fPM, 3,672 kg CO2; ..." + }, + { + "id": 34, + "type": "possible", + "ans": "Emirates, 8:45\u202fPM \u2013 9:15\u202fPM(+1), booking options: Emirates, Gotogate, Martigo, Expedia, kiss&fly, eDreams ... cheapest: Gotogate" + }, + { + "id": 35, + "type": "possible", + "ans": "EgyptAir, Lufthansa, Air Canada, 10:05\u202fAM \u2013 6:20\u202fPM, 15 hr 15 min, 1 stop, $644" + }, + { + "id": 36, + "type": "possible", + "ans": "Finnair, 6:00\u202fPM \u2013 6:05\u202fAM(+1), $744 ..." + }, + { + "id": 37, + "type": "possible", + "ans": "Lufthansa, 5:50\u202fPM \u2013 9:30\u202fAM(+2), return flight can be Lufthansa, 11:20\u202fAM \u2013 7:55\u202fAM(+1), the same as departure flight" + }, + { + "id": 38, + "type": "possible", + "ans": "Emirates, 2:10\u202fPM \u2013 11:55\u202fPM, Nonstop ..." + }, + { + "id": 39, + "type": "possible", + "ans": "Prague to Tokyo, British Airways, Air China, 7:05 AM \u2013 1:40 PM(+1)" + }, + { + "id": 40, + "type": "possible", + "ans": "Seattle to Las Vegas $21, Seattle to Los Angeles $42" + }, + { + "id": 41, + "type": "possible", + "ans": "United, Operated by Skywest DBA United Express, 10:30\u202fPM \u2013 12:45\u202fPM(+1), 1 stop" + } + ] + }, + "Google Map": { + "notice": "", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "Beehive Salon, Intermezzo Salon & Spa, Cindy's Beauty Salon, The Red Chair Salon, Ella and Oz Salon" + }, + { + "id": 1, + "type": "golden", + "ans": "'Amherst and 7th' or 'Main Street Middle'" + }, + { + "id": 2, + "type": "possible", + "ans": "Apple The Grove, Apple Beverly Center" + }, + { + "id": 3, + "type": "possible", + "ans": "Approximately 20 min" + }, + { + "id": 4, + "type": "possible", + "ans": "Drive via MA-1A S and take about 10 mins (based on real-time traffic conditions)" + }, + { + "id": 5, + "type": "possible", + "ans": "SP+ Parking in 1750 W 13th St, Chicago, IL 60608" + }, + { + "id": 6, + "type": "possible", + "ans": "UNIQLO State Street" + }, + { + "id": 7, + "type": "golden", + "ans": "Alanson, MI (EZ-Mart) Bus Stop" + }, + { + "id": 8, + "type": "golden", + "ans": "Hollywood Boulders" + }, + { + "id": 9, + "type": "golden", + "ans": "'Honor Fraser Gallery' or 'Walter Maciel Gallery'." + }, + { + "id": 10, + "type": "possible", + "ans": "located in Barstow, CA 92311; open 24 hours; phone number is (760) 252-6100" + }, + { + "id": 11, + "type": "possible", + "ans": "Village Maternity with a wheelchair accessible parking lot" + }, + { + "id": 12, + "type": "possible", + "ans": "Taki's Greek Kitchen - 4.7, Thai Chili - 4.7, Parker's Grille & Tavern - 4.5, Legacy Restaurant & Grille - 4.5, Jake's On the Lake - 4.5" + }, + { + "id": 13, + "type": "possible", + "ans": "Drive via MA-3 N and I-93 N, about 1.5 hours (based on real-time traffic conditions)." + }, + { + "id": 14, + "type": "possible", + "ans": "Rising Wolf Garage (should be motorcycle parking)" + }, + { + "id": 15, + "type": "possible", + "ans": "Quik Park; <reviews>" + }, + { + "id": 16, + "type": "possible", + "ans": "EVgo Charging Station" + }, + { + "id": 17, + "type": "possible", + "ans": "Protech Key and Locksmith (UTC 12:30)" + }, + { + "id": 18, + "type": "possible", + "ans": "Drive via I-80 W, about 29 hours" + }, + { + "id": 19, + "type": "possible", + "ans": "Hilton Garden Inn Pittsburgh Airport, walking time around 15min - 30min" + }, + { + "id": 20, + "type": "possible", + "ans": "Tesla Destination Charger, 1330 Maryland Ave SW, Washington, DC 20024" + }, + { + "id": 21, + "type": "golden", + "ans": "Elm Street & Oak Street, 18 Bay St, Amesbury, MA 01913" + }, + { + "id": 22, + "type": "possible", + "ans": "Best Buy, 1131 5th St, Miami Beach, FL 33139" + }, + { + "id": 23, + "type": "possible", + "ans": "around 42 min (1.9 miles) via 7th Ave" + }, + { + "id": 24, + "type": "possible", + "ans": "via US-101 N, around 19 min (current traffic condition), 14.6 miles" + }, + { + "id": 25, + "type": "possible", + "ans": "Park Rite Parking, Closes 11 PM" + }, + { + "id": 26, + "type": "golden", + "ans": "<Action>, print PDF" + }, + { + "id": 27, + "type": "possible", + "ans": "8" + }, + { + "id": 28, + "type": "golden", + "ans": "Privacy & Safety: Activity, Content, More options; Other settings" + }, + { + "id": 29, + "type": "possible", + "ans": "Ypsilanti Transit Center; Ellsworth + Michigan; YTC - Stop 5" + }, + { + "id": 30, + "type": "possible", + "ans": "2-68 Division St Garage, <reviews>" + }, + { + "id": 31, + "type": "golden", + "ans": "share link, https://maps.app.goo.gl/Bnp4Q67dTHoFZ4Lx8" + }, + { + "id": 32, + "type": "possible", + "ans": "Drain Genie Plumbing Services" + }, + { + "id": 33, + "type": "golden", + "ans": "star 2 has the least proportion; Accessibility: Assistive hearing loop; Wheelchair accessible entrance; Wheelchair accessible parking lot; Wheelchair accessible restroom; Wheelchair accessible seating; Amenities: Baggage storage; Wi-Fi; Free Wi-Fi" + }, + { + "id": 34, + "type": "possible", + "ans": "Speer Blvd Park ..." + }, + { + "id": 35, + "type": "possible", + "ans": "Big Bend National Park, TX; (432) 477-2251; 6PXX+WW Big Bend National Park, Texas; Tickets: $30 ..." + }, + { + "id": 36, + "type": "possible", + "ans": "Varasano's Pizzeria - Buckhead, 4.9; DaVinci's Pizzeria, 4.4; Mellow Mushroom Atlanta - Buckhead, 4.4; Vinny's N.Y. Pizza & Grill - Piedmont, 4.2; Gino's NY Pizza Bar, 4.0" + }, + { + "id": 37, + "type": "possible", + "ans": "Take Lafayette St and Pleasant St to Cross St in Marblehead, 14 min (3.9 mi); Drive to Rowland St, 1 min (0.1 mi)" + }, + { + "id": 38, + "type": "possible", + "ans": "Bike Parking, 104 W 38th St, New York, NY 10018" + }, + { + "id": 39, + "type": "possible", + "ans": "Miami, Florida to New Orleans, Louisiana; Get on I-95 N from S Miami Ave, 5 min (1.4 mi); Follow Florida's Tpke, I-75 N and I-10 W to Carondelet St in New Orleans. Take exit 12B from US-90 BUS W, 12 hr 6 min (864 mi); Turn left onto Carondelet St, 3 min (0.6 mi)" + }, + { + "id": 40, + "type": "possible", + "ans": "Boston Sail Loft, 4.6; one star review: Not sure about the rest of the seafood here since I left immediately after trying their AWFUL Chowder. I won't call it clam chowder since I didn't see a single piece of clam. This stuff was more like if you heated up half & Half then sprinkle dill and salt in it. It's too bad the tourist think this is how it's supposed to taste." + } + ] + }, + "Google Search": { + "notice": "", + "answers": [ + { + "id": 0, + "type": "golden", + "ans": "May 5, 2023" + }, + { + "id": 1, + "type": "possible", + "ans": "Born on September 29, 1988; Professional basketball player for the Phoenix Suns now." + }, + { + "id": 2, + "type": "possible", + "ans": "News Title (real-time)" + }, + { + "id": 3, + "type": "golden", + "ans": "Life Is Beautiful, Back to the Future, The Intouchables, City Lights, Modern Times" + }, + { + "id": 4, + "type": "possible", + "ans": "Counter-Strike 2, 602,898 players (real-time)" + }, + { + "id": 5, + "type": "possible", + "ans": "Suns 120-107 Trail Blazers (real-time)" + }, + { + "id": 6, + "type": "possible", + "ans": "New Year's Eve parties, Christmas markets, january, comedy shows... (real-time)" + }, + { + "id": 7, + "type": "golden", + "ans": "IOS 17.1" + }, + { + "id": 8, + "type": "possible", + "ans": "user: @melvinsmiley5295, 329 thumbs up and 2 replies (real-time)" + }, + { + "id": 9, + "type": "possible", + "ans": "IMDb 7.0/10, Rotten Tomatoes 73%" + }, + { + "id": 10, + "type": "possible", + "ans": "Taylor Swift, 10 songs (different sources have different results)" + }, + { + "id": 11, + "type": "possible", + "ans": "KATL, 13555 total arrivals and departures (real-time)" + }, + { + "id": 12, + "type": "golden", + "ans": "2007" + }, + { + "id": 13, + "type": "possible", + "ans": "Strange Planet, 2023" + }, + { + "id": 14, + "type": "golden", + "ans": "Yeovil Town" + }, + { + "id": 15, + "type": "golden", + "ans": "Not successful" + }, + { + "id": 16, + "type": "possible", + "ans": "880K, ChatGPT will soon have real-time news access (real-time)" + }, + { + "id": 17, + "type": "golden", + "ans": "Ivanka Trump, Barron Trump, Donald Trump Jr., Tiffany Trump, Eric Trump" + }, + { + "id": 18, + "type": "golden", + "ans": "Qatar; November 20 to December 18, 2022; Argentina" + }, + { + "id": 19, + "type": "golden", + "ans": "eedf571, Smaller BERT Models" + }, + { + "id": 20, + "type": "golden", + "ans": "April 4, 2025" + }, + { + "id": 21, + "type": "golden", + "ans": "The Lion King (2019); Frozen II (2019); The Super Mario Bros. Movie (2023); Frozen (2013); Incredibles 2 (2018)" + }, + { + "id": 22, + "type": "possible", + "ans": "trending topics: 1.valentines day events; 2.fashion week; 3.job fairs; 4.march; 5.february" + }, + { + "id": 23, + "type": "possible", + "ans": "<bio> LeBron James" + }, + { + "id": 24, + "type": "golden", + "ans": "Alpha Centauri star system; Proxima Centauri b, Proxima Centauri c, and Proxima Centauri d" + }, + { + "id": 25, + "type": "possible", + "ans": "eg, Manchester United 1-2 Fulham: Alex Iwobi scores in added time for huge away win" + }, + { + "id": 26, + "type": "possible", + "ans": "RAM 8 GB; Processor: Multicore Intel\u00ae or Apple Silicon processor (2 GHz or faster processor with SSE 4.2 or later) with 64-bit support; Operating system, macOS Big Sur (version 11.0) or later; Graphics card, GPU with Metal support, 1.5 GB of GPU memory ..." + }, + { + "id": 27, + "type": "possible", + "ans": "Current PM2.5 AQI\t43" + }, + { + "id": 28, + "type": "golden", + "ans": "IMDb score 8.8, Metacritic score 74%." + }, + { + "id": 29, + "type": "golden", + "ans": "9.58s held by Usain Bolt of Jamaica" + }, + { + "id": 30, + "type": "possible", + "ans": "real-time, Benson Boone; Beautiful Things, In The Stars, GHOST TOWN, To Love Someone, Before You, NIGHTS LIKE THESE, Sugar Sweet, ROOM FOR 2, Little Runaway, What Was" + }, + { + "id": 31, + "type": "golden", + "ans": "2014-15 season" + }, + { + "id": 32, + "type": "possible", + "ans": "Manchester City Football Club; June 10, 2023; Atat\u00fcrk Olympic Stadium, Istanbul, Turkey" + }, + { + "id": 33, + "type": "possible", + "ans": "<SHA> of latest Tensorflow" + }, + { + "id": 34, + "type": "possible", + "ans": "345,957,886 kilometers" + }, + { + "id": 35, + "type": "possible", + "ans": "eg, 19 February 2024, The accretion of a solar mass per day by a 17-billion solar mass black hole" + }, + { + "id": 36, + "type": "possible", + "ans": "French-Swedish physicist Anne L'Huillier, French scientist Pierre Agostini, and Hungarian-born Frank Krausz. <summary>" + }, + { + "id": 37, + "type": "possible", + "ans": "Gliese 667Cc, Kepler-22b, Kepler-69c" + }, + { + "id": 38, + "type": "possible", + "ans": "next: April 8, 2024. The one after that will take place on August 23, 2044." + }, + { + "id": 39, + "type": "possible", + "ans": "Tokyo, Japan; Seoul, South Korea; Halong Bay, Vietnam; Palawan Island, Philippines; Sapa, Vietnam; Bogota, Colombia; Pattaya, Thailand; Alajuela, Costa Rica; Phnom Penh, Cambodia; Kuala Lumpur, Malaysia. Asian: Tokyo, Japan; Seoul, South Korea; Halong Bay, Vietnam; Palawan Island, Philippines; Sapa, Vietnam; Kuala Lumpur, Malaysia; Phnom Penh, Cambodia" + }, + { + "id": 40, + "type": "golden", + "ans": "19,341 feet (5,895 meters)" + }, + { + "id": 41, + "type": "possible", + "ans": "current air pollution level in Los Angeles" + }, + { + "id": 42, + "type": "possible", + "ans": "The main difference between British English and American English is in pronunciation. Some words are also different in each variety of English, and there are also a few differences in the way they use grammar. Here are five of the most common grammatical differences between British and American English. 1. Present perfect and past simple; 2. got and gotten; 3. Verb forms with collective nouns; 4. have and take; 5. shall" + } + ] + }, + "Huggingface": { + "notice": " models are real-time", + "answers": [ + { + "id": 0, + "type": "possible", + "ans": "distilroberta-finetuned-financial-news-sentiment-analysis" + }, + { + "id": 1, + "type": "possible", + "ans": "<story> (generated by Inference API)" + }, + { + "id": 2, + "type": "possible", + "ans": "<model 1>; <model 2>; <model 3>; (last month, recently created)" + }, + { + "id": 3, + "type": "possible", + "ans": "replit/replit-code-v1-3b" + }, + { + "id": 4, + "type": "possible", + "ans": "TinyLlama/TinyLlama-1.1B-Chat-v1.0; TinyLlama can be plugged and played in many open-source projects built upon Llama. Besides, TinyLlama is compact with only 1.1B parameters; Applications: cater to a multitude of applications demanding a restricted computation and memory footprint." + }, + { + "id": 5, + "type": "possible", + "ans": "flax-community/t5-recipe-generation; 223M params; F32" + }, + { + "id": 6, + "type": "golden", + "ans": "0.550" + }, + { + "id": 7, + "type": "golden", + "ans": "autumnjohnson/ceti_audio" + }, + { + "id": 8, + "type": "possible", + "ans": "microsoft/phi-2; Text generation" + }, + { + "id": 9, + "type": "golden", + "ans": "Helsinki-NLP/opus-mt-ja-en; BLEU 41.7\t; chr-F 0.589" + }, + { + "id": 10, + "type": "golden", + "ans": "Mistral AI team" + }, + { + "id": 11, + "type": "possible", + "ans": "motexture/VSeq2VSeq; Text to video diffusion model with variable length frame conditioning for infinite length video generation." + }, + { + "id": 12, + "type": "possible", + "ans": "Jaagup/errors_corrections_min3" + }, + { + "id": 13, + "type": "golden", + "ans": "bool, defaults to False" + }, + { + "id": 14, + "type": "golden", + "ans": "$9/month; Pro Account: Get a PRO badge on your profile, Early access to new features, Unlock Inference for PROs, Higher tier for AutoTrain" + }, + { + "id": 15, + "type": "possible", + "ans": "junnyu/roformer_chinese_base" + }, + { + "id": 16, + "type": "possible", + "ans": "<model> (today, text classification)" + }, + { + "id": 17, + "type": "possible", + "ans": "<model>; <creator>; <description> (recent, NLP)" + }, + { + "id": 18, + "type": "golden", + "ans": "As in the Llama 2 paper, you can add a margin to the loss by adding a margin column to the dataset. The reward collator will automatically pass it through and the loss will be computed accordingly." + }, + { + "id": 19, + "type": "possible", + "ans": "<model> (Most recent, English text summarization)" + }, + { + "id": 20, + "type": "golden", + "ans": "ckiplab/bert-base-chinese-ner" + }, + { + "id": 21, + "type": "golden", + "ans": "from transformers import pipeline \\n classifier = pipeline(\"sentiment-analysis\") \\n classifier(\"We are very happy to show you the \ud83e\udd17 Transformers library.\") ... distilbert/distilbert-base-uncased-finetuned-sst-2-english" + }, + { + "id": 22, + "type": "possible", + "ans": "<summary> of https://huggingface.co/docs/transformers/main/en/add_tensorflow_model#4-model-implementation" + }, + { + "id": 23, + "type": "possible", + "ans": "eg, openai/whisper-large-v3" + }, + { + "id": 24, + "type": "golden", + "ans": "mistralai/Mixtral-8x7B-Instruct-v0.1" + }, + { + "id": 25, + "type": "golden", + "ans": "Add the load_in_8bit or load_in_4bit parameters to from_pretrained() and set device_map=\"auto\" to effectively distribute the model to your hardware. (Or use code)" + }, + { + "id": 26, + "type": "possible", + "ans": "PhilipTheGreat/DiabloGPT-small-Traveller, GPT2LMHeadModel, 510 MB" + }, + { + "id": 27, + "type": "golden", + "ans": "nlphuji/mscoco_2014_5k_test_image_text_retrieval" + }, + { + "id": 28, + "type": "possible", + "ans": "eg, /roberta-base-squad2, language: English" + }, + { + "id": 29, + "type": "possible", + "ans": "<summary> of Falconsai/medical_summarization (T5 Large for Medical Text Summarization)" + }, + { + "id": 30, + "type": "golden", + "ans": "Helsinki-NLP/opus-mt-en-zh; testset, BLEU, chr-F: Tatoeba-test.eng.zho, 31.4, 0.268" + }, + { + "id": 31, + "type": "possible", + "ans": "eg, Hawat/make-believe-fakenews-detection, Updated Jan 16 2024" + }, + { + "id": 32, + "type": "golden", + "ans": "\"temperature\": 1.0" + }, + { + "id": 33, + "type": "possible", + "ans": "eg, Transformers - 119,672 stars, Diffusers - 20,775 stars, Datasets - 17,960 stars." + }, + { + "id": 34, + "type": "possible", + "ans": "Empower your students with state-of-the-art resources; Give your students unlimited access to modern machine learning tools; Easily manage your classroom ..." + }, + { + "id": 35, + "type": "possible", + "ans": "eg, Accelerating SD Turbo and SDXL Turbo Inference with ONNX Runtime and Olive, Published January 15, 2024, <summary>" + }, + { + "id": 36, + "type": "possible", + "ans": "summary of https://huggingface.co/pricing" + }, + { + "id": 37, + "type": "possible", + "ans": "huggingface posts, https://huggingface.co/posts" + }, + { + "id": 38, + "type": "golden", + "ans": "use add_tokens method" + }, + { + "id": 39, + "type": "possible", + "ans": "Trainer example, https://huggingface.co/docs/evaluate/main/en/transformers_integrations#trainer" + }, + { + "id": 40, + "type": "possible", + "ans": "Streamlined Deployment; Efficient Resource Utilization; Dynamic Batching ..." + }, + { + "id": 41, + "type": "golden", + "ans": "openai/shap-e; there are Spaces like hysts/Shap-E ..." + }, + { + "id": 42, + "type": "golden", + "ans": "content: Please provide a reasonable subgoal-based plan to solve the given task.\\nTask: What was the opening date of the museum dedicated to the war that, after it occurred, Boston became one of the wealthiest international ports?; Initial Environment Description: None." + } + ] + }, + "Wolfram Alpha": { + "notice": "", + "answers": [ + { + "id": 0, + "type": "golden", + "ans": "11.2" + }, + { + "id": 1, + "type": "possible", + "ans": "2 a + 3 sqrt(5) x + 5 x>=sqrt(2 (5 + sqrt(5))) y AND 2 a + sqrt(50 + 22 sqrt(5)) y>=(5 + sqrt(5)) x AND sqrt(5) a + 2 sqrt(5) x + 2 sqrt(5 + 2 sqrt(5)) y <= a ... (Search inner region of the pentagram on Wolfram)" + }, + { + "id": 2, + "type": "golden", + "ans": "7.5095 * 10^33" + }, + { + "id": 3, + "type": "golden", + "ans": "1/4 (2 x cos(2 x) + (-1 + 2 x^2) sin(2 x)) + Constant" + }, + { + "id": 4, + "type": "golden", + "ans": "Densest known packing: 0.176939r; Square packing: 0.163961r" + }, + { + "id": 5, + "type": "golden", + "ans": "y(z) = \u00b1 2 am(1/2 sqrt((c_1 + 2) (z + c_2)^2), 4/(c_1 + 2)), am(x, m) is the Jacobi amplitude function" + }, + { + "id": 6, + "type": "golden", + "ans": "7 + 3 (-4 + x)^3 + (-4 + x)^5" + }, + { + "id": 7, + "type": "golden", + "ans": "-73.26\u00b0 from vertical; 0.252 m" + }, + { + "id": 8, + "type": "possible", + "ans": "approximately: 38.3 mol; 76.0% C; 4.3% H; 19.7% N" + }, + { + "id": 9, + "type": "golden", + "ans": "9752 GW h/yr (gigawatt hours per year)" + }, + { + "id": 10, + "type": "golden", + "ans": "geomagnetic field, total 51.5 uT;" + }, + { + "id": 11, + "type": "golden", + "ans": "UNS A92024: 4.9\u00d710^-6 \u03a9 cm (ohm centimeters) (at 20 \u00b0C); UNS G10800: 1.8\u00d710^-5 \u03a9 cm (ohm centimeters)" + }, + { + "id": 12, + "type": "golden", + "ans": "8902 (U+22C6)" + }, + { + "id": 13, + "type": "possible", + "ans": "approximately: 36430; 77325" + }, + { + "id": 14, + "type": "possible", + "ans": "approximately: Whopper, 657 Cal; Baconator, 902 Cal; Big Mac, 730 Cal" + }, + { + "id": 15, + "type": "golden", + "ans": "3.125%" + }, + { + "id": 16, + "type": "possible", + "ans": "intake 1500 Cal/d for 3 months 12 days to lose 17 kg with a sedentary activity level" + }, + { + "id": 17, + "type": "golden", + "ans": "Providence $13.81; Nashville $12.65; Boise $12.65" + }, + { + "id": 18, + "type": "possible", + "ans": "show a Albert Einstein curve with parametric equations" + }, + { + "id": 19, + "type": "possible", + "ans": "<sunborn time> (real-time date)" + }, + { + "id": 20, + "type": "golden", + "ans": "approximately 33038" + }, + { + "id": 21, + "type": "golden", + "ans": "approximately 0.717183 - 0.425258 i" + }, + { + "id": 22, + "type": "golden", + "ans": "127.306 cm^2 or 147 \\sqrt(3) / 2" + }, + { + "id": 23, + "type": "golden", + "ans": "mean population growth rate of Canada from 2020 to 2023 is 0.9998% per year" + }, + { + "id": 24, + "type": "golden", + "ans": "y(t) = c1 e^t sin(3t) + c2 e^t cos(3t)" + }, + { + "id": 25, + "type": "golden", + "ans": "if g=9.81; x = 63.64m, y = 19.49m; Vx = 21.21 m/s, Vy = -8.22 m/s" + }, + { + "id": 26, + "type": "possible", + "ans": "if no H2O, 153 moles, hydrogen (H), 32.69% for sulfur (S), and 65.25% for oxygen (O)." + }, + { + "id": 27, + "type": "golden", + "ans": "401.2 W/(m K); 236.9 W/(m K)" + }, + { + "id": 28, + "type": "golden", + "ans": "9649 or U+25B1" + }, + { + "id": 29, + "type": "possible", + "ans": "any cat curve" + }, + { + "id": 30, + "type": "possible", + "ans": "real-time, search query: sunburn 1:00 pm with SPF 1 in Brazil" + }, + { + "id": 31, + "type": "possible", + "ans": "real-time, search query: current temperature and wind speed in Chicago, IL." + }, + { + "id": 32, + "type": "golden", + "ans": "1009, 1013, 1019, 1021, 1031, 1033, 1039, 1049, 1051, 1061, 1063, 1069, 1087, 1091, 1093, 1097, 1103, 1109, 1117, 1123, 1129, 1151, 1153, 1163, 1171, 1181, 1187, 1193." + }, + { + "id": 33, + "type": "golden", + "ans": "89.5 TWh (terawatt hours)" + }, + { + "id": 34, + "type": "golden", + "ans": "317.8 times that of Earth, and the length of one day on Jupiter is approximately 9.925 hours" + }, + { + "id": 35, + "type": "golden", + "ans": "1/186313420339200000" + }, + { + "id": 36, + "type": "golden", + "ans": "converges" + }, + { + "id": 37, + "type": "golden", + "ans": "9675" + }, + { + "id": 38, + "type": "golden", + "ans": "around 39.2" + }, + { + "id": 39, + "type": "golden", + "ans": "x^2(\\sin(\frac{2\u03c0}{15}) - 2) + 2xy \\cos(\frac{2\u03c0}{15}) + 4 = y^2(2 + \\sin(\frac{2\u03c0}{15}))" + }, + { + "id": 40, + "type": "golden", + "ans": "around 0.078 kg" + }, + { + "id": 41, + "type": "golden", + "ans": "110 bpm" + }, + { + "id": 42, + "type": "golden", + "ans": "192 MB" + }, + { + "id": 43, + "type": "golden", + "ans": "35; 12" + }, + { + "id": 44, + "type": "possible", + "ans": "g(x) = 2 cos^(-1)((sinh(x) (cos(1/2) - sin(1/2)) + cosh(x) (cos(1/2) - sin(1/2)) + sin(1/2) + cos(1/2))/(sqrt(2) sqrt(-(sin(1) - 1) sinh(2 x) - (sin(1) - 1) cosh(2 x) + 1 + sin(1)))) OR ..." + }, + { + "id": 45, + "type": "golden", + "ans": "energy expenditure | 2720 kJ (kilojoules); average energy expenditure per step | 1.1 kJ/step (kilojoules per step); fat burned | 0.0842 kg (kilograms); oxygen consumption | 129.9 L (liters); metabolic equivalent | 7 metabolic equivalents" + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/main.py b/benchmarks/adapters/webvoyager/src/webvoyager/main.py new file mode 100644 index 0000000..0df9d22 --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/main.py @@ -0,0 +1,76 @@ +"""CLI: generate Harbor tasks for the WebVoyager benchmark.""" + +from __future__ import annotations + +import argparse +import logging +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from webvoyager.adapter import WebVoyagerAdapter +else: + from .adapter import WebVoyagerAdapter + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def _default_output_dir() -> Path: + return Path(__file__).resolve().parents[3] / ".tasks" + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate Harbor tasks for the WebVoyager benchmark", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=_default_output_dir(), + help="Directory to write generated tasks (default: adapters/webvoyager/.tasks)", + ) + parser.add_argument( + "--limit", + "--num-tasks", + type=int, + dest="limit", + default=None, + help="Generate only the first N tasks", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Overwrite existing task dirs", + ) + parser.add_argument( + "--task-ids", + nargs="+", + default=None, + help="Only generate these task IDs (source id or local dir id)", + ) + parser.add_argument( + "--refresh", + action="store_true", + help="Re-fetch the vendored dataset from upstream before generating", + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + adapter = WebVoyagerAdapter( + output_dir=args.output_dir, + limit=args.limit, + overwrite=args.overwrite, + task_ids=args.task_ids, + refresh=args.refresh, + ) + logger.info(f"Loaded {len(adapter.tasks)} WebVoyager records") + adapter.run() + logger.info(f"Done. Tasks in: {args.output_dir.resolve()}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/environment/kernel.json b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/environment/kernel.json new file mode 100644 index 0000000..fa9051d --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/environment/kernel.json @@ -0,0 +1,5 @@ +{ + "start_url": "{start_url}", + "stealth": true, + "viewport": { "width": 1280, "height": 1024 } +} diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/instruction.md b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/instruction.md new file mode 100644 index 0000000..bdd04f8 --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/instruction.md @@ -0,0 +1,9 @@ +You are operating a live web browser that is already open at {start_url} ({web_name}). + +Complete this task: + +{ques} + +When you are done, finish your final message with a single concise sentence that +directly states the answer or result (the specific item, value, page, or confirmation +the task asked for). Do not ask for human help; only interact with the browser provided. diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/solution/solve.sh b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/solution/solve.sh new file mode 100644 index 0000000..a6434b5 --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/solution/solve.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Oracle plumbing check: write the human reference answer where the verifier reads +# the agent answer. WebVoyager has no exact key (the judge weighs the screenshot), so +# this exercises the verifier path end to end rather than asserting correctness. +mkdir -p /logs/agent +printf '%s' '{reference_answer}' > /logs/agent/answer.txt diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml new file mode 100644 index 0000000..bbc8c24 --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml @@ -0,0 +1,32 @@ +schema_version = "1.0" + +[task] +name = "webvoyager/webvoyager__{task_id}" +authors = [{ name = "MinorJerry" }] +keywords = ["webvoyager", "web-agent", "{web_name_slug}"] + +[metadata] +difficulty = "medium" +category = "web-navigation" +source = "MinorJerry/WebVoyager" +web_name = "{web_name}" +start_url = "{start_url}" +reference_answer = "{reference_answer}" +reference_type = "{reference_type}" + +[verifier] +timeout_sec = 600.0 + +[verifier.env] +ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +JUDGE_MODEL = "${WEBVOYAGER_JUDGE_MODEL:-claude-sonnet-4-5}" +MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-3}" + +[agent] +timeout_sec = 1800.0 + +[environment] +build_timeout_sec = 300.0 +cpus = 1 +memory_mb = 2048 +storage_mb = 10240 diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh new file mode 100644 index 0000000..5323832 --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +pip install -q "anthropic>=0.40.0" +python /tests/webjudge.py diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py new file mode 100644 index 0000000..01630db --- /dev/null +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py @@ -0,0 +1,128 @@ +"""WebVoyager's single-call multimodal judge, ported to the Anthropic Messages API. + +Reads the agent's final answer (``/logs/agent/answer.txt``) and the last ``MAX_IMAGES`` +screenshots the cua agent spilled (``/logs/agent/shots/shot-<n>.png``), sends them to a +vision model with WebVoyager's verbatim SYSTEM_PROMPT, and writes a single 0/1 reward to +``/logs/verifier/reward.txt`` (verdict ``NOT SUCCESS`` -> 0, ``SUCCESS`` -> 1, ambiguous -> +fail-closed 0). Upstream ``evaluation/auto_eval.py`` used the OpenAI GPT-4V payload; the +prompt, last-k screenshots, and verdict parse are unchanged. +""" + +import base64 +import json +import os +import re +from pathlib import Path + +from anthropic import Anthropic + +AGENT_DIR = Path("/logs/agent") +ANSWER_FILE = AGENT_DIR / "answer.txt" +SHOTS_DIR = AGENT_DIR / "shots" +GROUND_TRUTH = Path("/tests/ground_truth.json") +VERIFIER_DIR = Path("/logs/verifier") + +# Verbatim from MinorJerry/WebVoyager evaluation/auto_eval.py SYSTEM_PROMPT. +SYSTEM_PROMPT = """As an evaluator, you will be presented with three primary components to assist you in your role: + +1. Web Task Instruction: This is a clear and specific directive provided in natural language, detailing the online activity to be carried out. These requirements may include conducting searches, verifying information, comparing prices, checking availability, or any other action relevant to the specified web service (such as Amazon, Apple, ArXiv, BBC News, Booking etc). + +2. Result Screenshots: This is a visual representation of the screen showing the result or intermediate state of performing a web task. It serves as visual proof of the actions taken in response to the instruction. + +3. Result Response: This is a textual response obtained after the execution of the web task. It serves as textual result in response to the instruction. + +-- You DO NOT NEED to interact with web pages or perform actions such as booking flights or conducting searches on websites. +-- You SHOULD NOT make assumptions based on information not presented in the screenshot when comparing it to the instructions. +-- Your primary responsibility is to conduct a thorough assessment of the web task instruction against the outcome depicted in the screenshot and in the response, evaluating whether the actions taken align with the given instructions. +-- NOTE that the instruction may involve more than one task, for example, locating the garage and summarizing the review. Failing to complete either task, such as not providing a summary, should be considered unsuccessful. +-- NOTE that the screenshot is authentic, but the response provided by LLM is generated at the end of web browsing, and there may be discrepancies between the text and the screenshots. +-- Note the difference: 1) Result response may contradict the screenshot, then the content of the screenshot prevails, 2) The content in the Result response is not mentioned on the screenshot, choose to believe the content. + +You should elaborate on how you arrived at your final evaluation and then provide a definitive verdict on whether the task has been successfully accomplished, either as 'SUCCESS' or 'NOT SUCCESS'.""" + +USER_TMPL = "TASK: {task}\nResult Response: {answer}\n{n} screenshot(s) at the end:" + +_MEDIA_BY_SUFFIX = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".webp": "image/webp", +} +_SHOT_INDEX = re.compile(r"(\d+)") + + +def _shot_key(path: Path) -> int: + """Numeric sort key. Shots are ``shot-<n>.png`` (not zero-padded).""" + match = _SHOT_INDEX.search(path.stem) + return int(match.group(1)) if match else 0 + + +def _last_shots(k: int) -> list[Path]: + if not SHOTS_DIR.is_dir(): + return [] + shots = sorted( + (p for p in SHOTS_DIR.iterdir() if p.suffix.lower() in _MEDIA_BY_SUFFIX), + key=_shot_key, + ) + return shots[-k:] + + +def main() -> None: + VERIFIER_DIR.mkdir(parents=True, exist_ok=True) + reward_path = VERIFIER_DIR / "reward.txt" + + ground_truth = json.loads(GROUND_TRUTH.read_text()) + task = ground_truth["task"] + answer = ANSWER_FILE.read_text().strip() if ANSWER_FILE.exists() else "" + k = int(os.getenv("MAX_IMAGES", "3")) + shots = _last_shots(k) + + if not answer and not shots: + reward_path.write_text("0") + return + + blocks: list[dict] = [ + {"type": "text", "text": USER_TMPL.format(task=task, answer=answer, n=len(shots))} + ] + for shot in shots: + blocks.append( + { + "type": "image", + "source": { + "type": "base64", + "media_type": _MEDIA_BY_SUFFIX[shot.suffix.lower()], + "data": base64.b64encode(shot.read_bytes()).decode(), + }, + } + ) + blocks.append({"type": "text", "text": "Your verdict:\n"}) + + model = os.getenv("JUDGE_MODEL", "claude-sonnet-4-5") + client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) + response = client.messages.create( + model=model, + max_tokens=1000, + temperature=0, + system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": blocks}], + ) + verdict = "".join(part.text for part in response.content if part.type == "text") + + reward = 0 if "NOT SUCCESS" in verdict else (1 if "SUCCESS" in verdict else 0) + reward_path.write_text(str(reward)) + (VERIFIER_DIR / "grading_details.json").write_text( + json.dumps( + { + "verdict_raw": verdict, + "reward": reward, + "n_images": len(shots), + "answer": answer, + "model": model, + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/adapters/webvoyager/tests/test_adapter.py b/benchmarks/adapters/webvoyager/tests/test_adapter.py new file mode 100644 index 0000000..131bbca --- /dev/null +++ b/benchmarks/adapters/webvoyager/tests/test_adapter.py @@ -0,0 +1,151 @@ +"""Mocked unit tests for the WebVoyager adapter (no network, no live judge).""" + +from __future__ import annotations + +import json +import re +import sys +import tomllib +from pathlib import Path + +import pytest + +SRC = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(SRC)) + +from webvoyager.adapter import WebVoyagerAdapter, _index_reference, _toml_escape # noqa: E402 + + +@pytest.fixture +def adapter(tmp_path: Path) -> WebVoyagerAdapter: + return WebVoyagerAdapter(output_dir=tmp_path / "out", limit=3) + + +def test_loads_vendored_dataset() -> None: + adapter = WebVoyagerAdapter(output_dir=Path("/tmp/unused")) + assert len(adapter.tasks) == 643 + assert len({t.source_id for t in adapter.tasks}) == 643 + + +def test_make_local_task_id_slugifies_spaces() -> None: + assert WebVoyagerAdapter.make_local_task_id("Allrecipes--0") == "webvoyager-allrecipes--0" + # Multi-word sites must slugify spaces so the dir + registry name are valid. + assert ( + WebVoyagerAdapter.make_local_task_id("Google Flights--7") + == "webvoyager-google-flights--7" + ) + + +def test_registry_names_have_no_spaces() -> None: + # ORG_NAME_PATTERN forbids spaces; every site (incl. Google Flights, BBC News) must pass. + adapter = WebVoyagerAdapter(output_dir=Path("/tmp/unused")) + for task in adapter.tasks: + name = f"webvoyager/webvoyager__{WebVoyagerAdapter.normalize_id(task.source_id)}" + assert re.fullmatch(r"[a-z0-9][a-z0-9._-]*/[a-z0-9][a-z0-9._-]*", name), name + assert ".." not in name + + +def test_reference_indexing_attaches_answers() -> None: + adapter = WebVoyagerAdapter(output_dir=Path("/tmp/unused")) + first = next(t for t in adapter.tasks if t.source_id == "Allrecipes--0") + # Allrecipes id 0 in reference_answer.json is type "possible" with a non-empty ans. + assert first.reference_type == "possible" + assert first.reference_ans + + +def test_index_reference_keys_by_web_name_and_id() -> None: + ref = {"Site": {"answers": [{"id": 0, "type": "golden", "ans": "x"}]}} + index = _index_reference(ref) + assert index[("Site", 0)]["ans"] == "x" + + +def test_run_emits_expected_files(adapter: WebVoyagerAdapter) -> None: + adapter.run() + dirs = sorted(p for p in adapter.output_dir.iterdir() if p.is_dir()) + assert len(dirs) == 3 + for task_dir in dirs: + assert (task_dir / "instruction.md").is_file() + assert (task_dir / "environment" / "kernel.json").is_file() + assert (task_dir / "tests" / "test.sh").is_file() + assert (task_dir / "tests" / "webjudge.py").is_file() + assert (task_dir / "tests" / "ground_truth.json").is_file() + assert (task_dir / "solution" / "solve.sh").is_file() + assert (task_dir / "task.toml").is_file() + + +def test_kernel_json_has_start_url_no_placeholders(adapter: WebVoyagerAdapter) -> None: + adapter.run() + task_dir = adapter.output_dir / "webvoyager-allrecipes--0" + kernel = json.loads((task_dir / "environment" / "kernel.json").read_text()) + assert kernel["start_url"] == "https://www.allrecipes.com/" + assert kernel["stealth"] is True + assert kernel["viewport"] == {"width": 1280, "height": 1024} + + +def test_instruction_substituted(adapter: WebVoyagerAdapter) -> None: + adapter.run() + instruction = ( + adapter.output_dir / "webvoyager-allrecipes--0" / "instruction.md" + ).read_text() + assert "{ques}" not in instruction and "{start_url}" not in instruction + assert "https://www.allrecipes.com/" in instruction + assert "vegetarian lasagna" in instruction + + +def test_task_toml_valid_and_named(adapter: WebVoyagerAdapter) -> None: + adapter.run() + raw = (adapter.output_dir / "webvoyager-allrecipes--0" / "task.toml").read_text() + assert not re.search(r"\{[a-z_]+\}", raw) # every {placeholder} substituted + parsed = tomllib.loads(raw) + assert parsed["task"]["name"] == "webvoyager/webvoyager__allrecipes--0" + assert "allrecipes" in parsed["task"]["keywords"] + assert parsed["metadata"]["start_url"] == "https://www.allrecipes.com/" + assert parsed["verifier"]["env"]["ANTHROPIC_API_KEY"] == "${ANTHROPIC_API_KEY}" + + +def test_ground_truth_carries_task(adapter: WebVoyagerAdapter) -> None: + adapter.run() + gt = json.loads( + ( + adapter.output_dir / "webvoyager-allrecipes--0" / "tests" / "ground_truth.json" + ).read_text() + ) + assert gt["web_name"] == "Allrecipes" + assert "lasagna" in gt["task"] + + +def test_solution_embeds_reference_answer(adapter: WebVoyagerAdapter) -> None: + adapter.run() + solve = ( + adapter.output_dir / "webvoyager-allrecipes--0" / "solution" / "solve.sh" + ).read_text() + assert "/logs/agent/answer.txt" in solve + assert "{reference_answer}" not in solve + + +def test_limit_and_task_ids_select(tmp_path: Path) -> None: + adapter = WebVoyagerAdapter( + output_dir=tmp_path / "out", task_ids=["Amazon--3", "webvoyager-apple--1"] + ) + selected = adapter._select() + ids = {t.source_id for t in selected} + assert ids == {"Amazon--3", "Apple--1"} + + +def test_overwrite_false_skips_existing(adapter: WebVoyagerAdapter) -> None: + adapter.run() + target = adapter.output_dir / "webvoyager-allrecipes--0" / "instruction.md" + target.write_text("SENTINEL") + adapter.run() # overwrite=False + assert target.read_text() == "SENTINEL" + + +def test_toml_escape_quotes_and_backslashes() -> None: + assert _toml_escape('a"b\\c') == 'a\\"b\\\\c' + assert "\n" not in _toml_escape("line1\nline2") + + +def test_toml_escape_control_chars_roundtrip() -> None: + # A stray form-feed (Wolfram Alpha--39 reference answer) must survive into valid TOML. + raw = f'k = "{_toml_escape("frac\x0cbreak")}"' + assert tomllib.loads(raw)["k"] == "frac\x0cbreak" diff --git a/benchmarks/adapters/webvoyager/tests/test_webjudge.py b/benchmarks/adapters/webvoyager/tests/test_webjudge.py new file mode 100644 index 0000000..4b56f3b --- /dev/null +++ b/benchmarks/adapters/webvoyager/tests/test_webjudge.py @@ -0,0 +1,126 @@ +"""Mocked unit tests for the ported WebVoyager judge (no live Anthropic call).""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import types +from pathlib import Path + +import pytest + +TEMPLATE_TESTS = ( + Path(__file__).resolve().parents[1] / "src" / "webvoyager" / "task-template" / "tests" +) + + +def _load_webjudge(monkeypatch, agent_dir: Path, tests_dir: Path, captured: dict): + """Import webjudge.py with a stubbed ``anthropic`` and redirected I/O paths.""" + fake = types.ModuleType("anthropic") + + class _Resp: + def __init__(self, text: str): + self.content = [types.SimpleNamespace(type="text", text=text)] + + class _Anthropic: + def __init__(self, *a, **k): + self.messages = self + + def create(self, **kwargs): + captured.update(kwargs) + return _Resp(captured.get("_verdict", "Reasoning... SUCCESS")) + + fake.Anthropic = _Anthropic + monkeypatch.setitem(sys.modules, "anthropic", fake) + + spec = importlib.util.spec_from_file_location( + "wv_webjudge_under_test", TEMPLATE_TESTS / "webjudge.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + monkeypatch.setattr(module, "AGENT_DIR", agent_dir) + monkeypatch.setattr(module, "ANSWER_FILE", agent_dir / "answer.txt") + monkeypatch.setattr(module, "SHOTS_DIR", agent_dir / "shots") + monkeypatch.setattr(module, "GROUND_TRUTH", tests_dir / "ground_truth.json") + monkeypatch.setattr(module, "VERIFIER_DIR", tests_dir / "verifier") + return module + + +def _setup_dirs(tmp_path: Path, *, answer: str, shot_indices: list[int]): + agent_dir = tmp_path / "agent" + shots = agent_dir / "shots" + shots.mkdir(parents=True) + if answer is not None: + (agent_dir / "answer.txt").write_text(answer) + for n in shot_indices: + # 1x1 PNG bytes; content is irrelevant for the mocked judge. + (shots / f"shot-{n}.png").write_bytes(b"\x89PNG\r\n\x1a\n" + bytes([n % 256])) + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "ground_truth.json").write_text(json.dumps({"task": "do a thing"})) + return agent_dir, tests_dir + + +def test_shot_key_numeric_order(monkeypatch, tmp_path: Path): + agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1, 2, 10]) + module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) + ordered = module._last_shots(10) + assert [p.name for p in ordered] == ["shot-1.png", "shot-2.png", "shot-10.png"] + + +def test_last_k_takes_final_screenshots(monkeypatch, tmp_path: Path): + agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1, 2, 3, 4, 5]) + module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) + assert [p.name for p in module._last_shots(3)] == [ + "shot-3.png", + "shot-4.png", + "shot-5.png", + ] + + +@pytest.mark.parametrize( + "verdict,expected", + [ + ("The agent did it. SUCCESS", "1"), + ("It failed. NOT SUCCESS", "0"), + ("ambiguous waffle", "0"), # neither marker -> fail-closed + ("clearly NOT SUCCESS even though SUCCESS appears", "0"), # NOT SUCCESS wins + ], +) +def test_verdict_parsing(monkeypatch, tmp_path: Path, verdict: str, expected: str): + captured = {"_verdict": verdict} + agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1]) + module = _load_webjudge(monkeypatch, agent_dir, tests_dir, captured) + monkeypatch.setenv("ANTHROPIC_API_KEY", "test") + module.main() + assert (tests_dir / "verifier" / "reward.txt").read_text() == expected + details = json.loads((tests_dir / "verifier" / "grading_details.json").read_text()) + assert details["verdict_raw"] == verdict + assert details["n_images"] == 1 + + +def test_empty_answer_and_no_shots_fail_closed(monkeypatch, tmp_path: Path): + agent_dir, tests_dir = _setup_dirs(tmp_path, answer="", shot_indices=[]) + module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) + module.main() + assert (tests_dir / "verifier" / "reward.txt").read_text() == "0" + assert not (tests_dir / "verifier" / "grading_details.json").exists() + + +def test_payload_attaches_images_and_system_prompt(monkeypatch, tmp_path: Path): + captured: dict = {"_verdict": "SUCCESS"} + agent_dir, tests_dir = _setup_dirs(tmp_path, answer="my answer", shot_indices=[1, 2]) + module = _load_webjudge(monkeypatch, agent_dir, tests_dir, captured) + monkeypatch.setenv("ANTHROPIC_API_KEY", "test") + monkeypatch.setenv("MAX_IMAGES", "2") + module.main() + blocks = captured["messages"][0]["content"] + images = [b for b in blocks if b["type"] == "image"] + assert len(images) == 2 + assert images[0]["source"]["media_type"] == "image/png" + assert "do a thing" in blocks[0]["text"] + assert "my answer" in blocks[0]["text"] + assert captured["system"].startswith("As an evaluator") + assert captured["temperature"] == 0 diff --git a/benchmarks/adapters/webvoyager/uv.lock b/benchmarks/adapters/webvoyager/uv.lock new file mode 100644 index 0000000..cf4954c --- /dev/null +++ b/benchmarks/adapters/webvoyager/uv.lock @@ -0,0 +1,108 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566", size = 4755489, upload-time = "2026-06-25T17:20:37.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078", size = 10839665, upload-time = "2026-06-25T17:19:44.702Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b", size = 11208649, upload-time = "2026-06-25T17:19:48.787Z" }, + { url = "https://files.pythonhosted.org/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632", size = 10622638, upload-time = "2026-06-25T17:19:51.354Z" }, + { url = "https://files.pythonhosted.org/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd", size = 10984227, upload-time = "2026-06-25T17:19:54.044Z" }, + { url = "https://files.pythonhosted.org/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b", size = 10622882, upload-time = "2026-06-25T17:19:57.037Z" }, + { url = "https://files.pythonhosted.org/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267", size = 11474808, upload-time = "2026-06-25T17:20:00.357Z" }, + { url = "https://files.pythonhosted.org/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c", size = 12293094, upload-time = "2026-06-25T17:20:03.446Z" }, + { url = "https://files.pythonhosted.org/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae", size = 11526176, upload-time = "2026-06-25T17:20:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b", size = 11520767, upload-time = "2026-06-25T17:20:09.191Z" }, + { url = "https://files.pythonhosted.org/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487", size = 11500132, upload-time = "2026-06-25T17:20:13.602Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3", size = 10943828, upload-time = "2026-06-25T17:20:16.635Z" }, + { url = "https://files.pythonhosted.org/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053", size = 10645418, upload-time = "2026-06-25T17:20:19.4Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4", size = 11211770, upload-time = "2026-06-25T17:20:22.033Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460", size = 11618698, upload-time = "2026-06-25T17:20:25.259Z" }, + { url = "https://files.pythonhosted.org/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21", size = 10857322, upload-time = "2026-06-25T17:20:28.612Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415", size = 11993274, upload-time = "2026-06-25T17:20:31.871Z" }, + { url = "https://files.pythonhosted.org/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca", size = 11343498, upload-time = "2026-06-25T17:20:35.03Z" }, +] + +[[package]] +name = "webvoyager-adapter" +version = "0.1.0" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8" }, + { name = "ruff", specifier = ">=0.6" }, +] From f5455b41737b430199c2d06339f960c9c2989da4 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 11:00:18 +0000 Subject: [PATCH 02/11] webvoyager: make the verifier judge pip-free (stdlib HTTPS to Anthropic) The Kernel verifier VM has Python 3 but no pip/ensurepip, so the judge cannot install the anthropic SDK at grade time. Call the Messages API directly with urllib.request instead; drop the install step from test.sh and point docs at bare python3 for generation. Also gitignore _smoke_logs/. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- benchmarks/adapters/webvoyager/.gitignore | 2 + benchmarks/adapters/webvoyager/README.md | 12 +++-- .../adapters/webvoyager/run_webvoyager.yaml | 2 +- .../webvoyager/task-template/tests/test.sh | 5 ++- .../task-template/tests/webjudge.py | 45 ++++++++++++++----- .../webvoyager/tests/test_webjudge.py | 30 +++++-------- 6 files changed, 59 insertions(+), 37 deletions(-) diff --git a/benchmarks/adapters/webvoyager/.gitignore b/benchmarks/adapters/webvoyager/.gitignore index 8a7d7a5..9b758b0 100644 --- a/benchmarks/adapters/webvoyager/.gitignore +++ b/benchmarks/adapters/webvoyager/.gitignore @@ -1,6 +1,8 @@ .tasks/ +.validate/ jobs/ trials/ +_smoke_logs/ __pycache__/ *.egg-info/ .venv/ diff --git a/benchmarks/adapters/webvoyager/README.md b/benchmarks/adapters/webvoyager/README.md index 1efd5c9..360daf4 100644 --- a/benchmarks/adapters/webvoyager/README.md +++ b/benchmarks/adapters/webvoyager/README.md @@ -13,7 +13,7 @@ webvoyager-<site>--<n>/ ├── instruction.md # the ques + a "state the answer in your last line" directive ├── environment/kernel.json # { start_url: <web>, stealth: true, viewport: 1280x1024 } ├── tests/ -│ ├── test.sh # pip install anthropic; python webjudge.py +│ ├── test.sh # python3 webjudge.py (stdlib-only; the Kernel VM has no pip) │ ├── webjudge.py # WebVoyager single-call multimodal judge (Anthropic port) │ └── ground_truth.json # { task, web_name, start_url, reference_answer, reference_type } ├── solution/solve.sh # oracle plumbing: writes the reference answer to answer.txt @@ -28,11 +28,15 @@ re-fetches it from upstream. ```bash cd benchmarks -uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks # all 643 -uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --limit 20 # first 20 -uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --task-ids Allrecipes--0 Amazon--3 +SRC=adapters/webvoyager/src/webvoyager/main.py +python3 $SRC --output-dir adapters/webvoyager/.tasks # all 643 +python3 $SRC --output-dir adapters/webvoyager/.tasks --limit 20 # first 20 +python3 $SRC --output-dir adapters/webvoyager/.tasks --task-ids Allrecipes--0 Amazon--3 ``` +The adapter has no third-party deps, so it runs on bare `python3` (no `uv sync` for +generation). `uv sync`/`npm` are only needed for the `harbor run` below. + `.tasks/` is gitignored. ## Run (Kernel env + cua) diff --git a/benchmarks/adapters/webvoyager/run_webvoyager.yaml b/benchmarks/adapters/webvoyager/run_webvoyager.yaml index 9bdb5bd..d282897 100644 --- a/benchmarks/adapters/webvoyager/run_webvoyager.yaml +++ b/benchmarks/adapters/webvoyager/run_webvoyager.yaml @@ -1,7 +1,7 @@ # Reference config for a WebVoyager run on the Kernel env with the cua agent. # # cd benchmarks -# uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --limit 20 +# python3 adapters/webvoyager/src/webvoyager/main.py --output-dir adapters/webvoyager/.tasks --limit 20 # uv run harbor run \ # -p adapters/webvoyager/.tasks \ # -e kernel --environment-kwarg pool_size=8 \ diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh index 5323832..86b0ccd 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -pip install -q "anthropic>=0.40.0" -python /tests/webjudge.py +# The Kernel verifier VM has Python 3 but no pip; webjudge.py talks to the +# Anthropic API with the standard library, so no install step is needed. +python3 /tests/webjudge.py diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py index 01630db..c979bd1 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py @@ -6,22 +6,29 @@ ``/logs/verifier/reward.txt`` (verdict ``NOT SUCCESS`` -> 0, ``SUCCESS`` -> 1, ambiguous -> fail-closed 0). Upstream ``evaluation/auto_eval.py`` used the OpenAI GPT-4V payload; the prompt, last-k screenshots, and verdict parse are unchanged. + +The request is made with the standard library (``urllib.request``) rather than the +``anthropic`` SDK: the Kernel verifier VM has Python 3.10 but no ``pip``/``ensurepip``, so +nothing can be installed at grade time. A bare HTTPS POST to the Messages API needs no +dependencies. """ import base64 import json import os import re +import urllib.request from pathlib import Path -from anthropic import Anthropic - AGENT_DIR = Path("/logs/agent") ANSWER_FILE = AGENT_DIR / "answer.txt" SHOTS_DIR = AGENT_DIR / "shots" GROUND_TRUTH = Path("/tests/ground_truth.json") VERIFIER_DIR = Path("/logs/verifier") +ANTHROPIC_URL = "https://api.anthropic.com/v1/messages" +ANTHROPIC_VERSION = "2023-06-01" + # Verbatim from MinorJerry/WebVoyager evaluation/auto_eval.py SYSTEM_PROMPT. SYSTEM_PROMPT = """As an evaluator, you will be presented with three primary components to assist you in your role: @@ -67,6 +74,23 @@ def _last_shots(k: int) -> list[Path]: return shots[-k:] +def _call_anthropic(payload: dict, api_key: str) -> str: + """POST the Messages payload and return the concatenated text content.""" + request = urllib.request.Request( + ANTHROPIC_URL, + data=json.dumps(payload).encode(), + headers={ + "x-api-key": api_key, + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json", + }, + method="POST", + ) + with urllib.request.urlopen(request) as response: # noqa: S310 (pinned API host) + body = json.loads(response.read().decode()) + return "".join(part.get("text", "") for part in body.get("content", [])) + + def main() -> None: VERIFIER_DIR.mkdir(parents=True, exist_ok=True) reward_path = VERIFIER_DIR / "reward.txt" @@ -98,15 +122,14 @@ def main() -> None: blocks.append({"type": "text", "text": "Your verdict:\n"}) model = os.getenv("JUDGE_MODEL", "claude-sonnet-4-5") - client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) - response = client.messages.create( - model=model, - max_tokens=1000, - temperature=0, - system=SYSTEM_PROMPT, - messages=[{"role": "user", "content": blocks}], - ) - verdict = "".join(part.text for part in response.content if part.type == "text") + payload = { + "model": model, + "max_tokens": 1000, + "temperature": 0, + "system": SYSTEM_PROMPT, + "messages": [{"role": "user", "content": blocks}], + } + verdict = _call_anthropic(payload, os.getenv("ANTHROPIC_API_KEY", "")) reward = 0 if "NOT SUCCESS" in verdict else (1 if "SUCCESS" in verdict else 0) reward_path.write_text(str(reward)) diff --git a/benchmarks/adapters/webvoyager/tests/test_webjudge.py b/benchmarks/adapters/webvoyager/tests/test_webjudge.py index 4b56f3b..432e722 100644 --- a/benchmarks/adapters/webvoyager/tests/test_webjudge.py +++ b/benchmarks/adapters/webvoyager/tests/test_webjudge.py @@ -4,8 +4,6 @@ import importlib.util import json -import sys -import types from pathlib import Path import pytest @@ -16,30 +14,24 @@ def _load_webjudge(monkeypatch, agent_dir: Path, tests_dir: Path, captured: dict): - """Import webjudge.py with a stubbed ``anthropic`` and redirected I/O paths.""" - fake = types.ModuleType("anthropic") - - class _Resp: - def __init__(self, text: str): - self.content = [types.SimpleNamespace(type="text", text=text)] - - class _Anthropic: - def __init__(self, *a, **k): - self.messages = self - - def create(self, **kwargs): - captured.update(kwargs) - return _Resp(captured.get("_verdict", "Reasoning... SUCCESS")) - - fake.Anthropic = _Anthropic - monkeypatch.setitem(sys.modules, "anthropic", fake) + """Import webjudge.py with the HTTP call stubbed and I/O paths redirected. + ``_call_anthropic`` is replaced so no network request is made; the request + payload it would have sent is recorded in ``captured`` for assertions, and the + canned verdict comes from ``captured['_verdict']``. + """ spec = importlib.util.spec_from_file_location( "wv_webjudge_under_test", TEMPLATE_TESTS / "webjudge.py" ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + def fake_call(payload: dict, api_key: str) -> str: + captured.update(payload) + captured["_api_key"] = api_key + return captured.get("_verdict", "Reasoning... SUCCESS") + + monkeypatch.setattr(module, "_call_anthropic", fake_call) monkeypatch.setattr(module, "AGENT_DIR", agent_dir) monkeypatch.setattr(module, "ANSWER_FILE", agent_dir / "answer.txt") monkeypatch.setattr(module, "SHOTS_DIR", agent_dir / "shots") From 1b0ac9ccaff33ca1c48df9b44d401fe473923d37 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 11:25:31 +0000 Subject: [PATCH 03/11] webvoyager: record 20-task live smoke (10/20) + harden the judge Run the WebVoyager adapter end to end on Kernel browsers with cua as the agent and the Anthropic WebJudge as the verifier: 20 tasks, pass rate 10/20 (10/17 of graded tasks), 3 agent timeouts on heavy/anti-bot sites, no adapter bugs. SMOKE.md captures the per-task table and the observed failure taxonomy. Make the judge resilient across model generations and transient API failures: retry once without `temperature` when a model rejects it with a 400 (newer models do), and fail closed to reward 0 with the error recorded in grading_details.json instead of crashing a trial into a missing reward. --- benchmarks/adapters/webvoyager/SMOKE.md | 86 +++++++++++++++---- .../task-template/tests/webjudge.py | 34 +++++++- .../webvoyager/tests/test_webjudge.py | 55 ++++++++++++ 3 files changed, 155 insertions(+), 20 deletions(-) diff --git a/benchmarks/adapters/webvoyager/SMOKE.md b/benchmarks/adapters/webvoyager/SMOKE.md index b6d3146..23489c3 100644 --- a/benchmarks/adapters/webvoyager/SMOKE.md +++ b/benchmarks/adapters/webvoyager/SMOKE.md @@ -1,10 +1,9 @@ # WebVoyager adapter — smoke / validation notes -## Live smoke status: NOT RUN (deferred) +## Live smoke: 20 tasks on Kernel + cua, pass rate 10/20 -The live 20-task Kernel smoke was intentionally **not** run for this change (the task -scoped to build + lint + mocked unit tests only; the parent runs the live smoke after -review). The pipeline is wired and ready. To run it: +Ran the full pipeline live against Kernel browsers with cua as the agent and the +ported Anthropic WebJudge as the verifier. Command: ```bash cd benchmarks && uv sync && (cd node && npm install && npm run build) @@ -14,13 +13,66 @@ uv run harbor run \ -e kernel --environment-kwarg pool_size=8 \ --agent-import-path cua_harbor:CuaHarborAgent \ -m anthropic/claude-sonnet-4-6 \ + --agent-timeout-multiplier 0.5 \ --ae ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - -n 6 + -y -n 8 ``` -Validate 1–2 tasks green end to end (browser provisions, agent drives, `answer.txt` + -`shots/*.png` land, judge scores) before spending the full 20. If pool create/acquire -403s (quota/plan), drop `--environment-kwarg pool_size=8` to fall back to per-task create. +- **Config:** 20 tasks spanning 13 sites, `-n 8` with `--environment-kwarg + pool_size=8` (pools worked, no 403), judge `claude-sonnet-4-6`, agent + `claude-sonnet-4-6`. `--agent-timeout-multiplier 0.5` → 900s/task (the task + `[agent].timeout_sec` is 1800). Runtime ~18 min, cost ~$4.83. +- **Pre-flight:** a 2-task run (`-l 2`) was validated green first + (google-search--1 reward 1, coursera--19 reward 1) before the full 20. + +### Results + +`reward = 1` iff the WebJudge returns `SUCCESS`. 17/20 tasks reached the +verifier and got a reward; 3 hit the agent timeout and never produced an answer. + +| metric | value | +|---|---| +| tasks run | 20 | +| **pass rate** | **10/20 (50%)** — 10/17 = 59% of graded tasks | +| graded SUCCESS | 10 | +| graded NOT SUCCESS | 7 | +| agent timeouts (no answer) | 3 (`AgentTimeoutError`, 900s cap) | +| adapter bugs | 0 | + +Pass (reward 1): arxiv--3, arxiv--20, bbc-news--3, coursera--4, coursera--19, +espn--22, github--5, github--18, google-search--1, wolfram-alpha--0. + +Graded fail (reward 0): allrecipes--12, cambridge-dictionary, apple--2, +apple--15, amazon--6, google-search--10, huggingface--2. + +Timed out (no reward): allrecipes--0, amazon--6 [also above], apple--15 [also +above], booking--2, espn--7. (amazon--6/apple--15 ran 39/48 steps to the cap; +the agent kept acting without converging on a final answer.) + +### Observed failure taxonomy + +- **Agent timeout on heavy / anti-bot sites** (the dominant mode). Amazon, + Apple, Booking, Allrecipes drove 16–48 steps and either hit the 900s cap or + burned most of it without producing `answer.txt`. These are the + stealth-required, heavy-DOM surfaces; the agent doesn't converge inside a + 900s budget. A real parity run should drop the multiplier (full 1800s) and/or + add a residential `proxy_id` at the env level. +- **Judge strictness on visually-unconfirmed answers.** `apple--15` and + `huggingface--2` produced a plausible textual answer but the last-k + screenshots didn't *show* the supporting page, so the WebJudge scored + `NOT SUCCESS` / fail-closed (`huggingface--2`'s verdict had no clean + `SUCCESS` token and defaulted to 0). Inherent to a screenshot+answer LLM + judge; `grading_details.json` keeps the raw verdict for triage. +- **Teardown race on timed-out trials (cosmetic).** When a trial times out its + Kernel session is already gone, so the env's post-trial log download + session + stop log `Failed to upload agent logs` and `Error stopping Kernel session: 404 + browser not found`. The trial still records as errored; no reward is lost that + wasn't already lost to the timeout. Not an adapter bug, but noisy. + +The pipeline itself was clean on all 20: browser provisioned, kernel.json +overrides applied, agent drove + spilled `answer.txt` + `shots/`, verifier ran +the WebJudge in-VM (stdlib HTTPS, no pip) and wrote `reward.txt` + +`grading_details.json`. No generation or wiring bug surfaced. ## Build-time validation that WAS done @@ -82,13 +134,13 @@ full (no live browser/judge needed): - **Dataset vendored + pinned** under `src/webvoyager/data/` (commit `0915445`, checksums in `adapter_metadata.json`) for hermetic generation; `--refresh` re-fetches from upstream. -## Expected live failure taxonomy (from the design doc, to watch for) +## Failure taxonomy predicted by the design doc — all confirmed by the smoke -- **env-vs-task ambiguity**: a captcha/login wall scores `NOT SUCCESS`, indistinguishable - from a real miss in a 0/1 reward. `grading_details.json` records the raw verdict for - post-hoc triage. -- **site drift / anti-bot**: Amazon, Booking, Google properties are the block surface; - `stealth:true` is on for every task. An operator may add a residential `proxy_id` / - `profile` at the env level. -- **judge disagreement**: inherent to an LLM judge; `temperature=0` + pinned `JUDGE_MODEL`. -- **adapter bug**: none expected post-fix; generation validated on all 643. +The design doc predicted env-vs-task ambiguity, site drift / anti-bot (Amazon, +Booking, Google as the block surface), judge disagreement, and no adapter bugs. +The live smoke bore all of these out — see "Observed failure taxonomy" above. +The one the doc under-weighted: on the heavy/anti-bot sites the agent more often +**times out mid-task** than it reaches a wrong answer, so those land as +`AgentTimeoutError` rather than `NOT SUCCESS`. Mitigations to apply for a parity +run: full 1800s budget, residential `proxy_id` at the env level, `temperature=0` ++ pinned `JUDGE_MODEL`. diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py index c979bd1..1b610ab 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py @@ -17,6 +17,7 @@ import json import os import re +import urllib.error import urllib.request from pathlib import Path @@ -74,8 +75,8 @@ def _last_shots(k: int) -> list[Path]: return shots[-k:] -def _call_anthropic(payload: dict, api_key: str) -> str: - """POST the Messages payload and return the concatenated text content.""" +def _post(payload: dict, api_key: str) -> str: + """POST one Messages payload and return the concatenated text content.""" request = urllib.request.Request( ANTHROPIC_URL, data=json.dumps(payload).encode(), @@ -91,6 +92,23 @@ def _call_anthropic(payload: dict, api_key: str) -> str: return "".join(part.get("text", "") for part in body.get("content", [])) +def _call_anthropic(payload: dict, api_key: str) -> str: + """Call the judge, dropping ``temperature`` if the model rejects it. + + Newer Anthropic models (e.g. claude-opus-4-8) reject the ``temperature`` + parameter with a 400; on that specific error, retry once without it so the + same judge code works across model generations. + """ + try: + return _post(payload, api_key) + except urllib.error.HTTPError as exc: + detail = exc.read().decode(errors="replace") if exc.code == 400 else "" + if exc.code == 400 and "temperature" in detail and "temperature" in payload: + retry = {k: v for k, v in payload.items() if k != "temperature"} + return _post(retry, api_key) + raise + + def main() -> None: VERIFIER_DIR.mkdir(parents=True, exist_ok=True) reward_path = VERIFIER_DIR / "reward.txt" @@ -129,8 +147,17 @@ def main() -> None: "system": SYSTEM_PROMPT, "messages": [{"role": "user", "content": blocks}], } - verdict = _call_anthropic(payload, os.getenv("ANTHROPIC_API_KEY", "")) + error = None + try: + verdict = _call_anthropic(payload, os.getenv("ANTHROPIC_API_KEY", "")) + except urllib.error.HTTPError as exc: + verdict, error = "", f"HTTP {exc.code}: {exc.read().decode(errors='replace')[:500]}" + except OSError as exc: # connection reset / DNS / timeout + verdict, error = "", f"{type(exc).__name__}: {exc}" + + # A judge call that errors fails closed to 0 (recorded in grading_details), so a + # transient API hiccup never crashes the verifier into a missing-reward trial. reward = 0 if "NOT SUCCESS" in verdict else (1 if "SUCCESS" in verdict else 0) reward_path.write_text(str(reward)) (VERIFIER_DIR / "grading_details.json").write_text( @@ -141,6 +168,7 @@ def main() -> None: "n_images": len(shots), "answer": answer, "model": model, + "error": error, }, indent=2, ) diff --git a/benchmarks/adapters/webvoyager/tests/test_webjudge.py b/benchmarks/adapters/webvoyager/tests/test_webjudge.py index 432e722..c3a8686 100644 --- a/benchmarks/adapters/webvoyager/tests/test_webjudge.py +++ b/benchmarks/adapters/webvoyager/tests/test_webjudge.py @@ -3,7 +3,9 @@ from __future__ import annotations import importlib.util +import io import json +import urllib.error from pathlib import Path import pytest @@ -116,3 +118,56 @@ def test_payload_attaches_images_and_system_prompt(monkeypatch, tmp_path: Path): assert "my answer" in blocks[0]["text"] assert captured["system"].startswith("As an evaluator") assert captured["temperature"] == 0 + + +def _http_error(code: int, body: str) -> urllib.error.HTTPError: + return urllib.error.HTTPError( + url=MODULE_URL, code=code, msg="err", hdrs=None, fp=io.BytesIO(body.encode()) + ) + + +MODULE_URL = "https://api.anthropic.com/v1/messages" + + +def _import_webjudge(): + spec = importlib.util.spec_from_file_location( + "wv_webjudge_retry", TEMPLATE_TESTS / "webjudge.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_call_anthropic_drops_temperature_on_400(monkeypatch): + """A 400 citing temperature retries once without it (newer models reject it).""" + module = _import_webjudge() + calls: list[dict] = [] + + def fake_post(payload: dict, api_key: str) -> str: + calls.append(payload) + if "temperature" in payload: + raise _http_error(400, '{"error":{"message":"temperature is deprecated"}}') + return "SUCCESS" + + monkeypatch.setattr(module, "_post", fake_post) + out = module._call_anthropic({"model": "m", "temperature": 0}, "k") + assert out == "SUCCESS" + assert len(calls) == 2 + assert "temperature" not in calls[1] + + +def test_main_fails_closed_on_http_error(monkeypatch, tmp_path: Path): + """A judge HTTP error writes reward 0 + an error note, never crashes the trial.""" + agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1]) + module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) + + def boom(payload: dict, api_key: str) -> str: + raise _http_error(529, '{"error":{"message":"overloaded"}}') + + monkeypatch.setattr(module, "_call_anthropic", boom) + monkeypatch.setenv("ANTHROPIC_API_KEY", "test") + module.main() + assert (tests_dir / "verifier" / "reward.txt").read_text() == "0" + details = json.loads((tests_dir / "verifier" / "grading_details.json").read_text()) + assert details["reward"] == 0 + assert "529" in details["error"] From eb921de8c3c5b5b6ee92e151fc38ec6f2628dad9 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 11:27:37 +0000 Subject: [PATCH 04/11] webvoyager: reconcile SMOKE.md with harbor's final summary The mid-run snapshot under-counted exceptions; the final summary is 5 (4 AgentTimeoutError + 1 AddTestsDirError). Headline Mean 0.500 (10/20) unchanged. --- benchmarks/adapters/webvoyager/SMOKE.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/benchmarks/adapters/webvoyager/SMOKE.md b/benchmarks/adapters/webvoyager/SMOKE.md index 23489c3..c2bdac2 100644 --- a/benchmarks/adapters/webvoyager/SMOKE.md +++ b/benchmarks/adapters/webvoyager/SMOKE.md @@ -33,27 +33,30 @@ verifier and got a reward; 3 hit the agent timeout and never produced an answer. | metric | value | |---|---| | tasks run | 20 | -| **pass rate** | **10/20 (50%)** — 10/17 = 59% of graded tasks | +| **pass rate** | **10/20 (Mean 0.500)** — 10/17 = 59% of graded tasks | | graded SUCCESS | 10 | | graded NOT SUCCESS | 7 | -| agent timeouts (no answer) | 3 (`AgentTimeoutError`, 900s cap) | +| exceptions | 5 (4 `AgentTimeoutError` + 1 `AddTestsDirError`) | | adapter bugs | 0 | -Pass (reward 1): arxiv--3, arxiv--20, bbc-news--3, coursera--4, coursera--19, -espn--22, github--5, github--18, google-search--1, wolfram-alpha--0. +Harbor's headline is `Mean 0.500` over 20 tasks (17 graded + the verifier did +not run on the 5 exception trials). Pass (reward 1): arxiv--3, arxiv--20, +bbc-news--3, coursera--4, coursera--19, espn--22, github--5, github--18, +google-search--1, wolfram-alpha--0. Graded fail (reward 0): allrecipes--12, cambridge-dictionary, apple--2, apple--15, amazon--6, google-search--10, huggingface--2. -Timed out (no reward): allrecipes--0, amazon--6 [also above], apple--15 [also -above], booking--2, espn--7. (amazon--6/apple--15 ran 39/48 steps to the cap; -the agent kept acting without converging on a final answer.) +Exceptions (no reward): the 4 `AgentTimeoutError` were the heaviest sites +(allrecipes--0, booking--2, plus two that drove 39/48 steps to the 900s cap +without converging on a final answer); 1 `AddTestsDirError` was a one-off env +setup failure on a single trial. ### Observed failure taxonomy -- **Agent timeout on heavy / anti-bot sites** (the dominant mode). Amazon, - Apple, Booking, Allrecipes drove 16–48 steps and either hit the 900s cap or - burned most of it without producing `answer.txt`. These are the +- **Agent timeout on heavy / anti-bot sites** (the dominant exception mode). + Amazon, Apple, Booking, Allrecipes drove 16–48 steps and either hit the 900s + cap or burned most of it without producing `answer.txt`. These are the stealth-required, heavy-DOM surfaces; the agent doesn't converge inside a 900s budget. A real parity run should drop the multiplier (full 1800s) and/or add a residential `proxy_id` at the env level. @@ -63,6 +66,8 @@ the agent kept acting without converging on a final answer.) `NOT SUCCESS` / fail-closed (`huggingface--2`'s verdict had no clean `SUCCESS` token and defaulted to 0). Inherent to a screenshot+answer LLM judge; `grading_details.json` keeps the raw verdict for triage. +- **One-off env error (`AddTestsDirError`).** A single trial failed during the + env's tests-dir setup. Not reproduced on the other 19; noted, not chased. - **Teardown race on timed-out trials (cosmetic).** When a trial times out its Kernel session is already gone, so the env's post-trial log download + session stop log `Failed to upload agent logs` and `Error stopping Kernel session: 404 From 41d7733e8be85313fdedb2a98376bac078a07861 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 12:01:25 +0000 Subject: [PATCH 05/11] webvoyager: record the opus-4-8 live smoke (14/20, Mean 0.70) Replace the SMOKE notes with the claude-opus-4-8 agent + opus-4-8 judge run: 14/20 pass over 20 curated tasks across 12 sites, 0 judge/adapter exceptions. Failure taxonomy: 1 anti-bot (Cloudflare), 3 screenshot-coverage false-negatives (the MAX_IMAGES tension), 1 agent timeout (multi-constraint faceted search), and 1 env/session-lifetime error (session deleted before the shared-session verifier could attach). The judge hardening this run validated (temperature-drop retry + fail-closed on HTTP error) is already on the branch. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- benchmarks/adapters/webvoyager/SMOKE.md | 294 +++++++++++++----------- 1 file changed, 163 insertions(+), 131 deletions(-) diff --git a/benchmarks/adapters/webvoyager/SMOKE.md b/benchmarks/adapters/webvoyager/SMOKE.md index c2bdac2..7baeb81 100644 --- a/benchmarks/adapters/webvoyager/SMOKE.md +++ b/benchmarks/adapters/webvoyager/SMOKE.md @@ -1,151 +1,183 @@ -# WebVoyager adapter — smoke / validation notes +# WebVoyager adapter — live smoke -## Live smoke: 20 tasks on Kernel + cua, pass rate 10/20 +Live 20-task smoke on the Kernel env with the cua agent. Goal: exercise the real pipeline end to +end, learn failure modes, fix adapter bugs surfaced. Learning smoke, **not** a definitive +WebVoyager number. -Ran the full pipeline live against Kernel browsers with cua as the agent and the -ported Anthropic WebJudge as the verifier. Command: +## Setup + +- Agent model: `anthropic/claude-opus-4-8` (cua CUA loop). +- Judge: `claude-opus-4-8`, multimodal, last-3 screenshots (`WEBVOYAGER_JUDGE_MODEL=claude-opus-4-8`, + `WEBVOYAGER_MAX_IMAGES=3`). All 19 scored trials confirmed `model: claude-opus-4-8` in their + `grading_details.json`. +- Browser pool: `--environment-kwarg pool_size=5` (pools worked on this account; no 403 fallback needed). +- Tasks: 20 curated across 12 sites (Allrecipes, Amazon, Apple, ArXiv, BBC News, Booking, Cambridge + Dictionary, Coursera, ESPN, GitHub, Google Search, Huggingface, Wolfram Alpha), generated to a + gitignored `.tasks/`. The naive first-20 are all Allrecipes, so the set was hand-picked with + `--task-ids` for site diversity plus a couple of known anti-bot sites (Amazon/Booking) to probe + blocking. ```bash cd benchmarks && uv sync && (cd node && npm install && npm run build) -uv run python -m webvoyager.main --output-dir adapters/webvoyager/.tasks --limit 20 -uv run harbor run \ - -p adapters/webvoyager/.tasks \ - -e kernel --environment-kwarg pool_size=8 \ - --agent-import-path cua_harbor:CuaHarborAgent \ - -m anthropic/claude-sonnet-4-6 \ - --agent-timeout-multiplier 0.5 \ - --ae ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ - -y -n 8 +python3 adapters/webvoyager/src/webvoyager/main.py --output-dir adapters/webvoyager/.tasks \ + --task-ids Allrecipes--0 Allrecipes--12 Apple--2 Apple--15 ArXiv--3 ArXiv--20 \ + "Cambridge Dictionary--1" Coursera--4 Coursera--19 ESPN--7 ESPN--22 GitHub--5 GitHub--18 \ + Huggingface--2 "BBC News--3" "Google Search--1" "Google Search--10" Amazon--6 Booking--2 \ + "Wolfram Alpha--0" +WEBVOYAGER_JUDGE_MODEL=claude-opus-4-8 WEBVOYAGER_MAX_IMAGES=3 \ +uv run harbor run -y -p adapters/webvoyager/.tasks -e kernel --environment-kwarg pool_size=5 \ + --agent-import-path cua_harbor:CuaHarborAgent -m anthropic/claude-opus-4-8 \ + --ae ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY -n 4 ``` -- **Config:** 20 tasks spanning 13 sites, `-n 8` with `--environment-kwarg - pool_size=8` (pools worked, no 403), judge `claude-sonnet-4-6`, agent - `claude-sonnet-4-6`. `--agent-timeout-multiplier 0.5` → 900s/task (the task - `[agent].timeout_sec` is 1800). Runtime ~18 min, cost ~$4.83. -- **Pre-flight:** a 2-task run (`-l 2`) was validated green first - (google-search--1 reward 1, coursera--19 reward 1) before the full 20. +Validated on 2 tasks first (github--5, arxiv--3) — both green end to end (browser provisioned, +agent drove, `answer.txt` + sorted `shots/*.png` landed, opus judge scored), Mean 1.000 — before +spending the full 20. -### Results +## Results -`reward = 1` iff the WebJudge returns `SUCCESS`. 17/20 tasks reached the -verifier and got a reward; 3 hit the agent timeout and never produced an answer. +**Mean reward 0.70** (14 of 19 scored trials = 1; the 20th, espn--7, errored out of the mean). | metric | value | |---|---| | tasks run | 20 | -| **pass rate** | **10/20 (Mean 0.500)** — 10/17 = 59% of graded tasks | -| graded SUCCESS | 10 | -| graded NOT SUCCESS | 7 | -| exceptions | 5 (4 `AgentTimeoutError` + 1 `AddTestsDirError`) | -| adapter bugs | 0 | - -Harbor's headline is `Mean 0.500` over 20 tasks (17 graded + the verifier did -not run on the 5 exception trials). Pass (reward 1): arxiv--3, arxiv--20, -bbc-news--3, coursera--4, coursera--19, espn--22, github--5, github--18, -google-search--1, wolfram-alpha--0. - -Graded fail (reward 0): allrecipes--12, cambridge-dictionary, apple--2, -apple--15, amazon--6, google-search--10, huggingface--2. - -Exceptions (no reward): the 4 `AgentTimeoutError` were the heaviest sites -(allrecipes--0, booking--2, plus two that drove 39/48 steps to the 900s cap -without converging on a final answer); 1 `AddTestsDirError` was a one-off env -setup failure on a single trial. - -### Observed failure taxonomy - -- **Agent timeout on heavy / anti-bot sites** (the dominant exception mode). - Amazon, Apple, Booking, Allrecipes drove 16–48 steps and either hit the 900s - cap or burned most of it without producing `answer.txt`. These are the - stealth-required, heavy-DOM surfaces; the agent doesn't converge inside a - 900s budget. A real parity run should drop the multiplier (full 1800s) and/or - add a residential `proxy_id` at the env level. -- **Judge strictness on visually-unconfirmed answers.** `apple--15` and - `huggingface--2` produced a plausible textual answer but the last-k - screenshots didn't *show* the supporting page, so the WebJudge scored - `NOT SUCCESS` / fail-closed (`huggingface--2`'s verdict had no clean - `SUCCESS` token and defaulted to 0). Inherent to a screenshot+answer LLM - judge; `grading_details.json` keeps the raw verdict for triage. -- **One-off env error (`AddTestsDirError`).** A single trial failed during the - env's tests-dir setup. Not reproduced on the other 19; noted, not chased. -- **Teardown race on timed-out trials (cosmetic).** When a trial times out its - Kernel session is already gone, so the env's post-trial log download + session - stop log `Failed to upload agent logs` and `Error stopping Kernel session: 404 - browser not found`. The trial still records as errored; no reward is lost that - wasn't already lost to the timeout. Not an adapter bug, but noisy. - -The pipeline itself was clean on all 20: browser provisioned, kernel.json -overrides applied, agent drove + spilled `answer.txt` + `shots/`, verifier ran -the WebJudge in-VM (stdlib HTTPS, no pip) and wrote `reward.txt` + -`grading_details.json`. No generation or wiring bug surfaced. - -## Build-time validation that WAS done - -Generation is the deterministic, network-free half of this adapter and was exercised in -full (no live browser/judge needed): - -- **Generated all 643 task dirs** from the vendored dataset (`uv run python -m - webvoyager.main --output-dir .tasks`): 643 dirs, 643 unique registry names. -- **Every `task.toml` loads through harbor's real `TaskConfig.model_validate_toml`** - (`name`, `[verifier.env]`, timeouts all parse). -- **Every `task.toml` `[task].name` matches `ORG_NAME_PATTERN`** and contains no `..`. -- **Every `environment/kernel.json` parses** with an `http(s)` `start_url` and the pinned - `viewport 1280x1024` / `stealth:true`. -- **Every `tests/ground_truth.json` parses** and carries a non-empty `task`. -- **No `{placeholder}` leaks** into any generated `instruction.md` / `task.toml` / - `kernel.json` / `solve.sh` / `ground_truth.json`. -- **Generated `test.sh` / `solve.sh` pass `bash -n`** (incl. the form-feed edge case below). -- **Mocked unit tests green**: `uv run pytest` (23 tests) covers dataset load, id - slugification, reference indexing, template substitution, TOML/control-char escaping, - the judge's numeric shot sort + last-k selection + `SUCCESS`/`NOT SUCCESS` verdict parse - (Anthropic client stubbed). `uv run ruff check` clean. - -## Bugs found and fixed during validation - -1. **Multi-word site names broke registry validation.** `ORG_NAME_PATTERN` - (`harbor/src/harbor/constants.py`) is - `^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*$` — it **forbids spaces**. - Upstream ids like `Google Flights--0` / `BBC News--3` / `Wolfram Alpha--45` would have - produced names with spaces and failed registration for ~250 tasks across 5 sites - (Google Flights, Google Map, BBC News, Wolfram Alpha, Cambridge Dictionary). The design - doc's claim that "double-dash and single-dash are fine" missed the space. Fixed by - slugifying the site portion (`normalize_id`: lowercase + spaces→`-`) in both the dir - name and the `task.toml` name. The `--<n>` suffix is fine (no `..`). - -2. **A reference answer contained a raw form-feed.** `Wolfram Alpha--39`'s - `reference_answer` has a literal `\x0c` (a broken LaTeX `\frac` where `\f` became a - form-feed). TOML basic strings forbid raw control chars, so the naive - `replace('"').replace('\\')` escape produced an invalid `task.toml`. Fixed `_toml_escape` - to emit `\uXXXX` for every control char and collapse newlines to spaces. +| **pass rate** | **14/20 (Mean 0.700)** | +| graded SUCCESS (reward 1) | 14 | +| graded NOT SUCCESS (reward 0) | 5 | +| exceptions | 2 (1 `AgentTimeoutError` + 1 `AddTestsDirError`) | +| judge/adapter exceptions | 0 | +| runtime / cost | ~49 min, $13.54 (15.1M input tok, 14.4M cached; opus agent + opus judge) | + +`reward = 1` iff the WebJudge returns `SUCCESS`. amazon--6 hit the agent timeout but the verifier +still ran and scored it 0; espn--7's session was deleted before the verifier could run, so it has +no reward and is excluded from the 19-trial mean. + +| task | reward | note | +|---|---|---| +| allrecipes--0 | 1 | | +| allrecipes--12 | 1 | | +| apple--15 | 1 | ~110-turn episode but completed | +| arxiv--3 | 1 | | +| arxiv--20 | 1 | | +| bbc-news--3 | 1 | | +| booking--2 | 1 | anti-bot site, not blocked | +| coursera--4 | 1 | | +| coursera--19 | 1 | | +| espn--22 | 1 | | +| github--5 | 1 | | +| github--18 | 1 | | +| google-search--1 | 1 | | +| wolfram-alpha--0 | 1 | | +| apple--2 | 0 | screenshot-coverage (judge only saw the iPhone 14 Pro page) | +| cambridge-dictionary | 0 | anti-bot: Cloudflare "verify you are human" loop, never reached page | +| google-search--10 | 0 | screenshot-coverage (answer claimed Billboard #1, no chart in shots) | +| huggingface--2 | 0 | screenshot-coverage (named 3 models, only 1 visible in shots) | +| amazon--6 | 0 | `AgentTimeoutError` (1800s); partial faceted search, judge scored 0 correctly | +| espn--7 | — | `AddTestsDirError`: Kernel session deleted before verifier could attach; no reward | + +## Failure taxonomy + +- **adapter bug (1, FIXED — see below):** the verifier crashed on every trial in the first + validation pass. Root-caused and fixed before the 20-run, which then had **0** judge/adapter + exceptions. +- **site drift / anti-bot (1): cambridge-dictionary.** Cloudflare interstitial looped indefinitely + (new Ray IDs each cycle); the agent never reached the dictionary page and fell back to its own + knowledge. The opus judge correctly scored NOT SUCCESS (screenshots show only the challenge + page) — a genuine env failure indistinguishable from a real miss in a 0/1 reward; + `grading_details.json` keeps the raw verdict for triage. Booking and Amazon were *not* + hard-blocked (booking--2 passed; amazon--6 reached the catalog). +- **judge strictness / screenshot-coverage (3): apple--2, google-search--10, huggingface--2.** + Likely *false negatives*: the agent's text answer was plausibly correct, but the last-3 + screenshots didn't capture the evidence, so the opus judge — correctly applying WebVoyager's + "screenshot prevails" rule — couldn't verify and returned NOT SUCCESS. This is the `MAX_IMAGES` + tension flagged in the design doc (open question #1): cua's final frames are often a post-answer + state, not the deciding frame. `MAX_IMAGES` is a `[verifier.env]` knob (`WEBVOYAGER_MAX_IMAGES`) + so it's tunable per run without an adapter change; default left at 3 (the doc's choice) since the + smoke shows the tradeoff but doesn't establish a better fixed value — raising it could dilute the + judge on shorter tasks. +- **long episodes / agent timeout (1): amazon--6.** Multi-constraint faceted search (black + strollers $100–200, >20k reviews, >4★) drove a ~130-turn episode that hit the 1800s + `AgentTimeoutError`. The verifier still ran and scored 0 — the agent had only found a stroller + with ~7k reviews, so the 0 is correct, not a false negative. Several-simultaneous-filter tasks + reliably produce very long opus trajectories on the Kernel browser. +- **env / session lifetime (1): espn--7.** `AddTestsDirError` — the agent ran to the 30-min + timeout and the **Kernel browser session was deleted before the shared-session verifier could + upload `tests/` and run `test.sh`** (`kernel.BadRequestError: session has already been deleted` + → `Failed to add tests directory to environment`). No `reward.txt` was produced, so harbor counts + it as an errored trial (excluded from the 19-trial mean). This is a harbor/env-level interaction + (session TTL vs agent timeout), **not** an adapter or judge bug — the adapter controls neither + session lifetime nor the agent timeout, so there's nothing to fix in the task dirs. Worth flagging + upstream: a maxed-out episode can outlive its session and strand the shared-session verifier. + +## Adapter bug found and fixed (surfaced by the smoke) + +**The verifier crashed on every trial → `RewardFileNotFoundError`.** The first 2-task validation +pass returned Mean 0.000 with 2 `RewardFileNotFoundError`s. `verifier/test-stdout.txt` showed +`webjudge.py` raising `urllib.error.HTTPError: HTTP Error 400` from the Anthropic Messages call, so +no `reward.txt` was ever written. Two distinct problems, both fixed in +`task-template/tests/webjudge.py`: + +1. **`temperature` is rejected by `claude-opus-4-8`.** The API returned + `400 invalid_request_error: "temperature is deprecated for this model."` The judge sent + `temperature: 0` (carried over from upstream WebVoyager for determinism); newer Anthropic models + reject the param. Fix: `_call_anthropic` now retries once without `temperature` when a 400 + specifically cites it, so the same judge code works across model generations (older models that + accept `temperature` are unaffected). Verified live: opus-4-8 then returns a verdict. +2. **An HTTP error crashed the whole trial instead of scoring 0.** `urllib.urlopen` raises on any + non-2xx, which propagated out of `main()` and left no reward file — turning a transient judge + hiccup into a discarded trial. Fix: `main()` wraps the judge call, fails **closed to reward 0** + on any `HTTPError`/`OSError`, and records the error in `grading_details.json` (`"error": ...`). + A judge API blip is now a recorded 0 with diagnostics, never a `RewardFileNotFoundError`. + +After the fix, re-validation scored Mean 1.000 on the same 2 tasks, and the full 20-run had **0** +judge/adapter exceptions (the only 2 exceptions were env/timeout, above). Two unit tests were added +(`tests/test_webjudge.py`): the temperature-drop retry and the fail-closed-on-HTTP-error path +(25 tests total). + +## Pipeline confirmed green + +Browser provisions from a `pool_size=5` warm pool; the cua agent drives via the shared +`cua_harbor:CuaHarborAgent` + Node entrypoint; `answer.txt` + numerically-sorted +`shots/shot-<n>.png` land under `/logs/agent/`; the ported WebVoyager single-call multimodal judge +(`webjudge.py`, opus-4-8, stdlib HTTPS — the Kernel verifier VM has no pip) reads the last-k shots ++ answer and writes `/logs/verifier/reward.txt` + `grading_details.json`. No adapter changes to the +shared core. ## Deviations from the design doc (`map-webvoyager.md`) -- **Judge provider: Anthropic, not OpenAI.** `OPENAI_API_KEY` is absent this session and - the shared core standardizes on Anthropic for the live-web judges. `tests/webjudge.py` - ports WebVoyager's single multimodal call to the Anthropic Messages API (SYSTEM_PROMPT is - verbatim from upstream `evaluation/auto_eval.py`; last-k screenshots + verdict parse - unchanged). `[verifier.env]` carries `ANTHROPIC_API_KEY` + `JUDGE_MODEL` - (`claude-sonnet-4-5`) instead of `OPENAI_API_KEY` + `gpt-4o`. -- **Reuse the shared `cua_harbor` agent + Node entrypoint** (not the `simpleqa`/`-a cua` - layout in the doc). The agent and the screenshot/answer spill already exist on the - shared-core branch; the adapter only emits task dirs + the judge. -- **Answer/screenshot paths follow the shared core**: `/logs/agent/answer.txt` and - `/logs/agent/shots/shot-<n>.png`. The shots are **not** zero-padded, so the judge sorts - numerically (not lexically) to take the last k. -- **No `captureScreenshot` re-screenshot fallback in the verifier.** Keeping the verifier - to a single `pip install anthropic` (the doc's own stated rationale, §2a) is simpler than - pulling the Kernel SDK into the grade path. The shared-core sink reliably spills a shot - per step; if both answer and shots are empty the judge fails closed to 0. +- **Judge provider: Anthropic, not OpenAI** (`OPENAI_API_KEY` absent; shared core standardizes on + Anthropic for the live-web judges). SYSTEM_PROMPT verbatim from upstream `evaluation/auto_eval.py`; + last-k shots + `SUCCESS`/`NOT SUCCESS` verdict parse unchanged. `[verifier.env]` carries + `ANTHROPIC_API_KEY` + `JUDGE_MODEL` (default `claude-sonnet-4-5`) instead of `OPENAI_API_KEY` + + `gpt-4o`. +- **Judge model: `claude-opus-4-8`** for this smoke (task requirement), set via + `${WEBVOYAGER_JUDGE_MODEL:-...}`. +- **Verifier is pip-free**: `webjudge.py` POSTs the Messages API with `urllib.request` (stdlib), not + the `anthropic` SDK, because the Kernel verifier VM has Python 3 but no pip/ensurepip. +- **Reuse the shared `cua_harbor` agent + Node entrypoint** (not the `simpleqa`/`-a cua` layout in + the doc). Answer/screenshot paths: `/logs/agent/answer.txt`, `/logs/agent/shots/shot-<n>.png` + (not zero-padded → judge sorts numerically for the last k). +- **No `captureScreenshot` re-screenshot fallback in the verifier.** Kept the verifier + dependency-free per the doc's own rationale (§2a). Note: such a fallback would *not* have saved + espn--7, whose session was already deleted. - **Dataset vendored + pinned** under `src/webvoyager/data/` (commit `0915445`, checksums in `adapter_metadata.json`) for hermetic generation; `--refresh` re-fetches from upstream. -## Failure taxonomy predicted by the design doc — all confirmed by the smoke +## Build-time validation (deterministic, network-free half) + +All 643 task dirs generate; every `task.toml` loads through harbor's real +`TaskConfig.model_validate_toml` and matches `ORG_NAME_PATTERN` (no `..`); every `kernel.json` +parses with an `http(s)` `start_url` + pinned `viewport 1280x1024` + `stealth:true`; every +`ground_truth.json` carries a non-empty `task`; no `{placeholder}` leaks into any generated file; +`test.sh`/`solve.sh` pass `bash -n`. `uv run pytest` (25 tests) + `uv run ruff check` green. + +Two generation bugs were found and fixed earlier: -The design doc predicted env-vs-task ambiguity, site drift / anti-bot (Amazon, -Booking, Google as the block surface), judge disagreement, and no adapter bugs. -The live smoke bore all of these out — see "Observed failure taxonomy" above. -The one the doc under-weighted: on the heavy/anti-bot sites the agent more often -**times out mid-task** than it reaches a wrong answer, so those land as -`AgentTimeoutError` rather than `NOT SUCCESS`. Mitigations to apply for a parity -run: full 1800s budget, residential `proxy_id` at the env level, `temperature=0` -+ pinned `JUDGE_MODEL`. +1. **Multi-word site names broke registry validation.** `ORG_NAME_PATTERN` forbids spaces, so ids + like `Google Flights--0` / `BBC News--3` / `Wolfram Alpha--45` would fail registration for ~250 + tasks. Fixed by slugifying the site portion (`normalize_id`: lowercase + spaces→`-`) in both the + dir name and the `task.toml` name. +2. **A reference answer contained a raw form-feed.** `Wolfram Alpha--39`'s `reference_answer` has a + literal `\x0c`. TOML basic strings forbid raw control chars, so the naive escape produced an + invalid `task.toml`. Fixed `_toml_escape` to emit `\uXXXX` for every control char and collapse + newlines to spaces. From b4e7e162fabe2752d564d6b012fd2c32f2a8b277 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:45:01 +0000 Subject: [PATCH 06/11] webvoyager: raise MAX_IMAGES judge default 3 -> 15 to match canonical auto-eval The canonical WebVoyager auto-eval invocation (evaluation/run_eval.sh + README) runs the GPT-4V judge with --max_attached_imgs 15; our default of 3 was read from auto_eval.py's argparse default, which is never what produces the published numbers. With one screenshot spilled per agent step, the last-k window is the only place the deciding frame can land, so k=3 left correct answers unverifiable and produced screenshot-coverage false-negatives. Set the default to 15 in task.toml and webjudge.py (env override preserved) and fix the README/run-config notes that quoted the old default. A live re-run at k=15 recovers the SMOKE false-negatives (apple--2, huggingface--2 both 0 -> 1). Adds PARITY.md documenting the applied fix vs the deliberate Kernel adaptations left intact. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- benchmarks/adapters/webvoyager/PARITY.md | 258 ++++++++++++++++++ benchmarks/adapters/webvoyager/README.md | 2 +- .../adapters/webvoyager/run_webvoyager.yaml | 3 +- .../src/webvoyager/task-template/task.toml | 2 +- .../task-template/tests/webjudge.py | 2 +- 5 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 benchmarks/adapters/webvoyager/PARITY.md diff --git a/benchmarks/adapters/webvoyager/PARITY.md b/benchmarks/adapters/webvoyager/PARITY.md new file mode 100644 index 0000000..6deac32 --- /dev/null +++ b/benchmarks/adapters/webvoyager/PARITY.md @@ -0,0 +1,258 @@ +# WebVoyager adapter — parity vs upstream + +Line-by-line comparison of this adapter against the canonical WebVoyager source, +`github.com/MinorJerry/WebVoyager` (pinned commit `0915445`, the same SHA vendored under +`src/webvoyager/data/` and recorded in `adapter_metadata.json`). Focus: the things that change +**grading / results**, not cosmetics. Each row cites the upstream file it came from. + +Upstream files inspected: +- `evaluation/auto_eval.py` — the GPT-4V single-call judge (SYSTEM_PROMPT, last-k screenshots, + `SUCCESS` / `NOT SUCCESS` verdict parse). +- `evaluation/run_eval.sh` + `README.md` — the **canonical invocation** of the judge. +- `data/WebVoyager_data.jsonl` — task records (4 fields). +- `data/reference_answer.json` — human-eval reference answers (not the auto-judge input). +- `prompts.py` / `run.py` — the agent loop, for the `ANSWER; [...]` answer convention and the + `Now given a task: … Please interact with …` task framing that `auto_eval.py` parses back out. + +Our files: `src/webvoyager/adapter.py`, `task-template/{instruction.md,task.toml,environment/kernel.json, +solution/solve.sh,tests/test.sh,tests/webjudge.py}`. + +--- + +## TL;DR + +- **Judge prompt (SYSTEM_PROMPT), last-k screenshot selection, and the `SUCCESS`/`NOT SUCCESS` + verdict parse are faithful ports** — SYSTEM_PROMPT is byte-for-byte verbatim; last-k and verdict + logic are semantically identical. +- **One fidelity-bug, now FIXED: `MAX_IMAGES` default was 3, canonical WebVoyager auto-eval uses 15.** + Our design doc reasoned from `auto_eval.py`'s *argparse default* (`1`), but the value actually + used to produce the published numbers — in both `README.md` and `evaluation/run_eval.sh` — is + `--max_attached_imgs 15`. With cua spilling one screenshot per step, 3 vs 15 materially changes + how much evidence the judge sees; SMOKE.md's three false-negatives (apple--2, huggingface--2, + google-search--10) were all "deciding frame not in the last-3" screenshot-coverage misses, which + is exactly the symptom a too-small k produces. **Applied:** default raised to 15 in `task.toml` + and `webjudge.py` (env override kept). See "Applied vs skipped" below. +- Everything else that diverges is a **deliberate Kernel adaptation** (Anthropic judge instead of + OpenAI, whole-answer-file instead of the brittle `ANSWER;` regex, fail-closed `None→0`, stdlib + HTTP instead of the SDK) and should **not** be reverted. + +--- + +## Material differences + +### 1. `MAX_IMAGES` default 3 vs canonical 15 — **fidelity-bug (FIXED)** + +- **Upstream:** `evaluation/auto_eval.py` `argparse` default is `--max_attached_imgs 1`, **but that + default is never what runs.** The canonical auto-eval invocation, in `README.md` (line 181) and + `evaluation/run_eval.sh`, is: + ```bash + python -u auto_eval.py --api_key ... --process_dir ../results/examples --max_attached_imgs 15 + ``` + i.e. the judge sees the **last 15** screenshots. (`README.md` line 102's `--max_attached_imgs 3` + is for the **agent run** `run.py` — context clipping of the agent's own window — a different knob + on a different program. Do not conflate the two.) +- **Ours (after fix):** `task.toml` `[verifier.env] MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-15}"`; + `webjudge.py` `k = int(os.getenv("MAX_IMAGES", "15"))`. Default **15**, matching canonical; the + `WEBVOYAGER_MAX_IMAGES` env override is preserved for tuning. +- **Why it matters:** the judge is multimodal and "screenshot prevails" (SYSTEM_PROMPT). cua + captures one screenshot per agent step, so the last-k window is the only place the deciding frame + can live. At k=3 the deciding frame is frequently *not* attached on multi-step tasks → the judge + can't verify a correct answer and returns NOT SUCCESS. SMOKE.md's three screenshot-coverage + false-negatives are this failure mode. Raising k toward the canonical 15 brings our judge's + evidence window in line with the published benchmark and should recover those. +- **Change applied:** default set to **15** to match the canonical run + (`MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-15}"` in `task.toml`; default `"15"` in `webjudge.py`). + The env override stays tunable. The doc's open-question #1 ("1 vs 3") was framed against the wrong + upstream baseline; the real canonical baseline is 15. + +### 2. `<num>` in USER_PROMPT = configured-k vs actual-attached-count — **minor** + +- **Upstream:** builds the user text from the **configured** `img_num`, regardless of how many + files exist: `auto_eval.py` line 83 `user_prompt_tmp.replace('<num>', str(img_num))`. So if + `--max_attached_imgs 15` but only 6 screenshots exist, the text still says "15 screenshots at the + end:" while 6 images are attached. +- **Ours:** `USER_TMPL.format(..., n=len(shots))` — we say the **actual** number attached + (`webjudge.py` line 127). If 6 are attached we say "6 screenshot(s) at the end:". +- **Assessment:** ours is arguably *more correct* (the number matches reality), and the count is not + load-bearing for the verdict. Truthful is fine here. Keep ours; not a fidelity bug. (If we ever + want literal text parity, inject the configured `k`, but there's no grading reason to.) + +### 3. USER_PROMPT wording: "screenshots" vs "screenshot(s)" + trailing space — **minor** + +- **Upstream:** `USER_PROMPT = "TASK: <task>\nResult Response: <answer>\n<num> screenshots at the end: "` + (trailing space), then a **separate** text block `"Your verdict:\n"` appended after the images + (`auto_eval.py` lines 26–28, 92). +- **Ours:** `USER_TMPL = "TASK: {task}\nResult Response: {answer}\n{n} screenshot(s) at the end:"` + (no trailing space; "screenshot(s)"), then the same separate `{"type":"text","text":"Your verdict:\n"}` + block after the images (`webjudge.py` lines 51, 140). The block structure (text → images → + "Your verdict:\n") matches upstream exactly. +- **Assessment:** trivial whitespace/pluralization drift inside the prompt; no grading impact. Could + match verbatim ("screenshots", trailing space) for tidiness, but it's optional. + +### 4. Judge provider: Anthropic Messages API vs OpenAI GPT-4V — **intentional-keep** + +- **Upstream:** OpenAI `chat.completions` with `model=gpt-4-vision-preview` (default) / + `gpt-4o`, `image_url` data-URI payload, `max_tokens=1000, seed=42, temperature=0` + (`auto_eval.py` line 99). +- **Ours:** Anthropic Messages API (`webjudge.py`): `system`+`messages`, `image`/`base64` blocks, + `max_tokens=1000, temperature=0`, default model `claude-sonnet-4-5`, posted with stdlib + `urllib.request` (the Kernel verifier VM has no pip). `[verifier.env]` carries `ANTHROPIC_API_KEY` + + `JUDGE_MODEL` instead of `OPENAI_API_KEY`. +- **Assessment:** deliberate Kernel-wide standardization on the Anthropic judge for the live-web + adapters. The *prompt and decision logic are unchanged*, so the grading contract is preserved. + **Do not revert.** Documented in SMOKE.md "Deviations" and `adapter_metadata.json`. (Caveat below.) + +### 5. `seed=42` not set — **minor / not-applicable** + +- **Upstream:** passes `seed=42` to OpenAI for reproducibility (`auto_eval.py` line 99). +- **Ours:** no seed (the Anthropic Messages API has no `seed` parameter). We pin `temperature=0` + (dropped only if the model 400s on it — see #8). +- **Assessment:** can't be matched on Anthropic; `temperature=0` is the available determinism lever. + No action. Inherent to the provider swap (#4). + +### 6. Agent answer extraction: whole answer file vs `ANSWER[; ]+[...]` regex — **intentional-keep** + +- **Upstream:** the agent emits `Action: ANSWER; [content]` (`prompts.py` action format), and + `auto_eval.py` lines 56–62 require `'Action: ANSWER' in ans_info`, then extracts only the bracket + content via `pattern_ans = r"ANSWER[; ]+\[?(.[^\]]*)\]?"`. No `ANSWER` ⇒ returns `0`. +- **Ours:** the cua agent writes its final assistant text to `/logs/agent/answer.txt`; the judge + reads the **whole file** as `Result Response` (`webjudge.py` line 118). No `ANSWER;` convention, + no regex. +- **Assessment:** deliberate and **more robust** — we control both ends (the cua harness produces + the answer, not WebVoyager's prompt), so re-imposing the brittle `ANSWER;` parse would only add a + way to silently drop a present answer. The judge's `Result Response` slot gets the same semantic + content (the agent's final answer). **Keep.** (Documented in the design doc §1b.) + +### 7. Task content into the judge: `ground_truth["task"]` vs regex-from-logs — **intentional-keep (equivalent)** + +- **Upstream:** recovers the task text by regexing the agent's logged first message: + `pattern = r"Now given a task:(.+?)Please interact with"` over `interact_messages.json` + (`auto_eval.py` lines 47–53). That captured text **is** `task['ques']` (run.py line 313 builds the + message as `f"Now given a task: {task['ques']} Please interact with {web} ..."`). +- **Ours:** the adapter writes `ques` straight into `tests/ground_truth.json` `"task"` + (`adapter.py` line 157), and `webjudge.py` reads `ground_truth["task"]` (line 117). +- **Assessment:** same string by construction, sourced directly instead of round-tripped through a + log regex — strictly more reliable. **Keep.** + +### 8. `None` (abstain) verdict folded to 0; `temperature` auto-drop — **intentional-keep** + +- **Upstream:** `auto_eval_res = 0 if 'NOT SUCCESS' in res else 1; if 'SUCCESS' not in res: + auto_eval_res = None` (`auto_eval.py` lines 128–130). `None` (judge emitted neither marker) is + returned and **excluded** from aggregation. +- **Ours:** `reward = 0 if "NOT SUCCESS" in verdict else (1 if "SUCCESS" in verdict else 0)` + (`webjudge.py` line 161) — ambiguous **fails closed to 0**, with the raw verdict saved to + `grading_details.json` for audit. Plus a Kernel-specific retry: a 400 citing `temperature` retries + once without it (newer Anthropic models reject `temperature`), and any HTTP/OSError also fails + closed to 0 with an `error` note (`webjudge.py` `_call_anthropic` + `main` try/except). +- **Assessment:** Harbor's reward channel is a single float, so an abstain must become a number; + fail-closed + audit trail is the right single-reward encoding and the verdict-marker logic + otherwise matches upstream exactly (`NOT SUCCESS` wins over `SUCCESS`, same precedence). **Keep.** + - *Optional refinement (minor, not required):* the abstain rate is currently invisible in the + aggregate (folded into "fail"). If we later want upstream-style exclusion, emit + `reward.json {"reward":0,"abstain":1}` and drop abstains from the mean. The design doc flags this + as open-question #3; start fail-closed, revisit only if the abstain rate is material. Today's + `grading_details.json` already records `verdict_raw`, so abstains are recoverable post-hoc. + +### 9. Dataset → task mapping (4 fields) — **parity: exact** + +- **Upstream record** (`WebVoyager_data.jsonl`, 643 rows, 643 unique ids): exactly + `{web_name, id, ques, web}`. +- **Ours:** `WebVoyagerTask` reads all four — `web_name`, `id`→`source_id`, `ques`, `web`→`start_url` + (`adapter.py` lines 44–48). `ques`→instruction + `ground_truth.task`; `web`→`start_url` in + `kernel.json` + instruction; `id` normalized to the Harbor task name. The human-eval + `reference_answer.json` `ans`/`type` are carried into `[metadata]`/`ground_truth.json` for analysis + only and (correctly) **not** fed to the auto-judge — matching upstream, where `reference_answer.json` + is the human-eval track, not an `auto_eval.py` input. **No fidelity gap.** + +### 10. Task-selection / id normalization — **parity-preserving adaptation** + +- **Upstream:** `auto_eval.py` iterates `web` × `idx in range(0,46)`, dir `task<Web>--<idx>`; ids + carry spaces (e.g. `Google Flights--7`). +- **Ours:** `normalize_id` lowercases + replaces spaces with `-` (`adapter.py` lines 73–86), so + `Google Flights--7` → dir `webvoyager-google-flights--7`, name + `webvoyager/webvoyager__google-flights--7`. Forced by Harbor's `ORG_NAME_PATTERN` (no spaces). + `--task-ids` accepts **both** the raw upstream id and the normalized id (`adapter.py` lines 124–131), + so curated subsets work either way. Selecting all 643 tasks is the full benchmark; a curated + subset is the documented practice for the flaky live sites. +- **Assessment:** required for Harbor registry validity; does not change *which* task is which or its + content. **Keep.** + +--- + +## No-difference confirmations (faithful ports) + +- **SYSTEM_PROMPT** — byte-for-byte verbatim from `auto_eval.py` lines 10–25 (the "three primary + components … 'SUCCESS' or 'NOT SUCCESS'" prompt). Diffs clean against `webjudge.py` lines 34–49. +- **Last-k screenshot selection** — upstream sorts PNGs by numeric index and takes `matches[-img_num:]` + (`auto_eval.py` lines 68–71); ours sorts `shot-<n>.png` by numeric stem and takes `shots[-k:]` + (`webjudge.py` `_last_shots` / `_shot_key`). Equivalent (take the final k, numeric order). Unit + tests `test_last_k_takes_final_screenshots` / `test_shot_key_numeric_order` cover the ordering. +- **Verdict marker precedence** — `NOT SUCCESS` checked before `SUCCESS` in both, so a verdict + containing both resolves to 0. Covered by `test_verdict_parsing`. +- **`max_tokens=1000`** — same in both. +- **No exact-match against `reference_answer`** — neither upstream nor ours scores the answer text + against the stale human reference; both rely on the multimodal judge with "screenshot prevails." + Ours preserves this (the reference `ans` is metadata only). + +--- + +## Applied vs skipped (this change) + +**Applied — the one warranted fidelity-bug fix:** + +- **[fidelity-bug] `MAX_IMAGES` default 3 → 15** (§1). Matches the canonical + `evaluation/run_eval.sh` / `README.md` auto-eval invocation (`--max_attached_imgs 15`). Edited two + load-bearing places plus two docs that quoted the old default: + - `src/webvoyager/task-template/task.toml`: `MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-15}"` + - `src/webvoyager/task-template/tests/webjudge.py`: `k = int(os.getenv("MAX_IMAGES", "15"))` + - `README.md` `[verifier.env]` table + `run_webvoyager.yaml` comment: default now `15`, and the + stale "set `1` to match the paper" line corrected (canonical is 15, not 1). + The `WEBVOYAGER_MAX_IMAGES` env override is preserved. This is the only change that affects parity + with the published WebVoyager numbers. Build + lint (`uv run ruff check`) + mocked unit tests + (`uv run pytest`, 25) green; generation re-verified to render `MAX_IMAGES=15`. + + **Live re-run at k=15 (same judge as SMOKE, `claude-opus-4-8`) confirms the recovery.** Re-ran the + three screenshot-coverage false-negatives: + - `apple--2`: 0 → **1** (judge now sees `n_images=15` incl. both the iPhone 14 Pro and 15 Pro + tech-spec pages it needs to compare; at k=3 it "only saw the iPhone 14 Pro page"). + - `huggingface--2`: 0 → **1** (`n_images=8`; all three named translation models now visible; at + k=3 "only 1 visible in shots"). + - `google-search--10`: re-run in progress at time of writing (long episode, 30+ steps); the two + confirmed flips already validate the k=3→15 fix as the cause of the screenshot-coverage misses. + +**Skipped — intentional Kernel adaptations (not reverted, per the compare pass):** + +- Anthropic Messages judge instead of OpenAI GPT-4V (§4) — prompt + decision logic unchanged. +- Whole `answer.txt` as `Result Response` instead of the brittle `ANSWER[; ]+[...]` regex (§6). +- `ground_truth["task"]` instead of regexing `ques` out of agent logs (§7). +- `None`→0 fail-closed + `temperature` auto-drop + fail-closed-on-HTTP-error (§8). +- stdlib `urllib` POST instead of the `anthropic` SDK (no pip on the Kernel verifier VM). +- id slugification for `ORG_NAME_PATTERN` (§10). + +**Skipped — minor / no grading impact (left as-is):** + +- USER_PROMPT wording "screenshot(s)" vs verbatim "screenshots" + trailing space (§3) — cosmetic. +- `<num>` = actual-attached-count vs upstream's configured-k (§2) — ours is arguably more correct. +- `seed=42` (§5) — unavailable on the Anthropic API. +- Abstain-rate surfacing via `reward.json` (§8 refinement) — defer until a parity run shows a + material abstain rate; today's `grading_details.json` keeps `verdict_raw` for post-hoc recovery. + +## Do-NOT-revert (deliberate Kernel adaptations) + +- Anthropic judge instead of OpenAI GPT-4V (#4) — prompt + logic unchanged. +- Whole `answer.txt` instead of the `ANSWER[; ]+[...]` regex (#6). +- `ground_truth["task"]` instead of regexing `ques` from agent logs (#7). +- `None`→0 fail-closed + `temperature` auto-drop + fail-closed-on-HTTP-error (#8). +- stdlib `urllib` POST instead of the `anthropic` SDK (no pip on the verifier VM). +- id slugification for `ORG_NAME_PATTERN` (#10). + +--- + +## Caveat on judge-model parity (out of adapter scope) + +Provider parity (#4) preserves the *prompt and decision logic*, but absolute scores still depend on +the judge model. Upstream's ~85.3% human agreement was measured with **GPT-4V**; our default is +`claude-sonnet-4-5` (the SMOKE used `claude-opus-4-8`). That's the intended Kernel adaptation, but +when reporting a WebVoyager number, pin and report `JUDGE_MODEL` alongside it (already recorded in +`grading_details.json` + `parity_experiment` config) and treat cross-judge comparisons as a tolerance +band, not an exact match — consistent with the design doc's parity section. diff --git a/benchmarks/adapters/webvoyager/README.md b/benchmarks/adapters/webvoyager/README.md index 360daf4..511f128 100644 --- a/benchmarks/adapters/webvoyager/README.md +++ b/benchmarks/adapters/webvoyager/README.md @@ -67,4 +67,4 @@ and writes a single `0|1` reward to `/logs/verifier/reward.txt` plus a `grading_ |---|---|---| | `ANTHROPIC_API_KEY` | `${ANTHROPIC_API_KEY}` | resolved from the host at exec time | | `JUDGE_MODEL` | `claude-sonnet-4-5` | vision judge model (`WEBVOYAGER_JUDGE_MODEL` to override) | -| `MAX_IMAGES` | `3` | last-k screenshots sent to the judge (`WEBVOYAGER_MAX_IMAGES`; set `1` to match the paper) | +| `MAX_IMAGES` | `15` | last-k screenshots sent to the judge (`WEBVOYAGER_MAX_IMAGES`; `15` matches the canonical `auto_eval.py` invocation) | diff --git a/benchmarks/adapters/webvoyager/run_webvoyager.yaml b/benchmarks/adapters/webvoyager/run_webvoyager.yaml index d282897..4f58ef4 100644 --- a/benchmarks/adapters/webvoyager/run_webvoyager.yaml +++ b/benchmarks/adapters/webvoyager/run_webvoyager.yaml @@ -13,7 +13,8 @@ # The judge model is pinned via [verifier.env] in each task.toml # (JUDGE_MODEL=claude-sonnet-4-5, override with WEBVOYAGER_JUDGE_MODEL); the verifier # reads ANTHROPIC_API_KEY from the host. MAX_IMAGES (last-k screenshots sent to the -# judge) defaults to 3; override with WEBVOYAGER_MAX_IMAGES=1 to match the paper. +# judge) defaults to 15, matching the canonical auto_eval.py invocation; override with +# WEBVOYAGER_MAX_IMAGES. tasks_path: adapters/webvoyager/.tasks environment: kernel environment_kwargs: diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml index bbc8c24..15e5827 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml @@ -20,7 +20,7 @@ timeout_sec = 600.0 [verifier.env] ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" JUDGE_MODEL = "${WEBVOYAGER_JUDGE_MODEL:-claude-sonnet-4-5}" -MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-3}" +MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-15}" [agent] timeout_sec = 1800.0 diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py index 1b610ab..00d5284 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py @@ -116,7 +116,7 @@ def main() -> None: ground_truth = json.loads(GROUND_TRUTH.read_text()) task = ground_truth["task"] answer = ANSWER_FILE.read_text().strip() if ANSWER_FILE.exists() else "" - k = int(os.getenv("MAX_IMAGES", "3")) + k = int(os.getenv("MAX_IMAGES", "15")) shots = _last_shots(k) if not answer and not shots: From 729ec25d447346eded2e842b23d81d195b7f5906 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 21:42:46 +0000 Subject: [PATCH 07/11] webvoyager: convert the verifier judge to a pi-ai node bin Replace the hand-rolled stdlib-urllib Anthropic judge (webjudge.py) with a self-contained TypeScript bin under judge/ that calls the model through @earendil-works/pi-ai's completeSimple. pi-ai owns provider routing, env-var keys, o-series temperature/max_completion_tokens quirks, vision, and retries, so the manual provider client and temperature-drop retry are deleted rather than ported. Transport-only change: the SYSTEM_PROMPT, the last-k (MAX_IMAGES=15) screenshot selection, the SUCCESS/NOT SUCCESS verdict parse, and the claude-sonnet-4-5 default are carried over byte-identically. JUDGE_MODEL is now a pi-ai provider:name ref (bare name defaults to anthropic). pi-ai is bundled into a single judge.js via tsdown (inlineDynamicImports) so the verifier runs with no install on the Kernel VM. test.sh shells `node judge.js` with the same inputs; the adapter copies the built bundle into each task's tests/ and build-checks it. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- benchmarks/adapters/webvoyager/.gitignore | 4 + benchmarks/adapters/webvoyager/PARITY.md | 50 +- benchmarks/adapters/webvoyager/README.md | 31 +- benchmarks/adapters/webvoyager/SMOKE.md | 9 + .../adapters/webvoyager/adapter_metadata.json | 3 +- .../webvoyager/judge/package-lock.json | 3610 +++++++++++++++++ .../adapters/webvoyager/judge/package.json | 28 + .../webvoyager/judge/src/artifacts.ts | 51 + .../adapters/webvoyager/judge/src/judge.ts | 119 + .../adapters/webvoyager/judge/src/model.ts | 49 + .../adapters/webvoyager/judge/src/prompts.ts | 50 + .../adapters/webvoyager/judge/src/types.ts | 25 + .../adapters/webvoyager/judge/src/webjudge.ts | 28 + .../webvoyager/judge/test/artifacts.test.ts | 63 + .../webvoyager/judge/test/judge.test.ts | 110 + .../webvoyager/judge/test/prompts.test.ts | 37 + .../webvoyager/judge/test/webjudge.test.ts | 62 + .../adapters/webvoyager/judge/tsconfig.json | 17 + .../webvoyager/judge/tsdown.config.ts | 18 + .../webvoyager/judge/vitest.config.ts | 12 + .../webvoyager/src/webvoyager/adapter.py | 18 +- .../webvoyager/task-template/tests/test.sh | 24 +- .../task-template/tests/webjudge.py | 179 - .../adapters/webvoyager/tests/test_adapter.py | 2 +- .../webvoyager/tests/test_webjudge.py | 173 - 25 files changed, 4382 insertions(+), 390 deletions(-) create mode 100644 benchmarks/adapters/webvoyager/judge/package-lock.json create mode 100644 benchmarks/adapters/webvoyager/judge/package.json create mode 100644 benchmarks/adapters/webvoyager/judge/src/artifacts.ts create mode 100644 benchmarks/adapters/webvoyager/judge/src/judge.ts create mode 100644 benchmarks/adapters/webvoyager/judge/src/model.ts create mode 100644 benchmarks/adapters/webvoyager/judge/src/prompts.ts create mode 100644 benchmarks/adapters/webvoyager/judge/src/types.ts create mode 100644 benchmarks/adapters/webvoyager/judge/src/webjudge.ts create mode 100644 benchmarks/adapters/webvoyager/judge/test/artifacts.test.ts create mode 100644 benchmarks/adapters/webvoyager/judge/test/judge.test.ts create mode 100644 benchmarks/adapters/webvoyager/judge/test/prompts.test.ts create mode 100644 benchmarks/adapters/webvoyager/judge/test/webjudge.test.ts create mode 100644 benchmarks/adapters/webvoyager/judge/tsconfig.json create mode 100644 benchmarks/adapters/webvoyager/judge/tsdown.config.ts create mode 100644 benchmarks/adapters/webvoyager/judge/vitest.config.ts delete mode 100644 benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py delete mode 100644 benchmarks/adapters/webvoyager/tests/test_webjudge.py diff --git a/benchmarks/adapters/webvoyager/.gitignore b/benchmarks/adapters/webvoyager/.gitignore index 9b758b0..7831f0f 100644 --- a/benchmarks/adapters/webvoyager/.gitignore +++ b/benchmarks/adapters/webvoyager/.gitignore @@ -9,3 +9,7 @@ __pycache__/ .cache/ .pytest_cache/ .ruff_cache/ + +# Node judge bin build artifacts +judge/dist/ +judge/node_modules/ diff --git a/benchmarks/adapters/webvoyager/PARITY.md b/benchmarks/adapters/webvoyager/PARITY.md index 6deac32..52874fe 100644 --- a/benchmarks/adapters/webvoyager/PARITY.md +++ b/benchmarks/adapters/webvoyager/PARITY.md @@ -15,7 +15,20 @@ Upstream files inspected: `Now given a task: … Please interact with …` task framing that `auto_eval.py` parses back out. Our files: `src/webvoyager/adapter.py`, `task-template/{instruction.md,task.toml,environment/kernel.json, -solution/solve.sh,tests/test.sh,tests/webjudge.py}`. +solution/solve.sh,tests/test.sh}`, and the judge bin under `judge/src/` (built to `tests/judge.js`). + +> **Transport note.** The judge was reimplemented as a self-contained `node` bin +> (`judge/src/`, bundled to `judge.js`) that calls the model through +> [`@earendil-works/pi-ai`](https://www.npmjs.com/package/@earendil-works/pi-ai) +> instead of a hand-rolled `urllib`/Anthropic POST. This is a **transport-only** +> change: the SYSTEM_PROMPT, the last-k (MAX_IMAGES=15) screenshot selection, the +> `SUCCESS`/`NOT SUCCESS` verdict parse, and the `claude-sonnet-4-5` default are +> carried over byte-identically. The rows below cite the original `webjudge.py` +> lines for the logic; that same logic now lives in `judge/src/{prompts,artifacts, +> webjudge,judge}.ts`. pi-ai handles provider routing (a `provider:name` +> `JUDGE_MODEL` ref, bare name = `anthropic`), env-var keys, o-series quirks, +> vision, and retries, so the abandon-on-error / temperature-drop handling is +> pi-ai's rather than the old manual retry. --- @@ -33,8 +46,8 @@ solution/solve.sh,tests/test.sh,tests/webjudge.py}`. is exactly the symptom a too-small k produces. **Applied:** default raised to 15 in `task.toml` and `webjudge.py` (env override kept). See "Applied vs skipped" below. - Everything else that diverges is a **deliberate Kernel adaptation** (Anthropic judge instead of - OpenAI, whole-answer-file instead of the brittle `ANSWER;` regex, fail-closed `None→0`, stdlib - HTTP instead of the SDK) and should **not** be reverted. + OpenAI, whole-answer-file instead of the brittle `ANSWER;` regex, fail-closed `None→0`, a bundled + pi-ai `node` bin instead of the OpenAI SDK) and should **not** be reverted. --- @@ -94,10 +107,11 @@ solution/solve.sh,tests/test.sh,tests/webjudge.py}`. - **Upstream:** OpenAI `chat.completions` with `model=gpt-4-vision-preview` (default) / `gpt-4o`, `image_url` data-URI payload, `max_tokens=1000, seed=42, temperature=0` (`auto_eval.py` line 99). -- **Ours:** Anthropic Messages API (`webjudge.py`): `system`+`messages`, `image`/`base64` blocks, - `max_tokens=1000, temperature=0`, default model `claude-sonnet-4-5`, posted with stdlib - `urllib.request` (the Kernel verifier VM has no pip). `[verifier.env]` carries `ANTHROPIC_API_KEY` - + `JUDGE_MODEL` instead of `OPENAI_API_KEY`. +- **Ours:** the model is called through pi-ai's `completeSimple` (`judge/src/model.ts`): + `systemPrompt` + one user message of text/image blocks, `maxTokens=1000, temperature=0`, default + model `claude-sonnet-4-5`. pi-ai is bundled into `judge.js`, so the verifier needs no install on + the Kernel VM. `[verifier.env]` carries `ANTHROPIC_API_KEY` + `JUDGE_MODEL` instead of + `OPENAI_API_KEY`; `JUDGE_MODEL` is a pi-ai `provider:name` ref (bare name = `anthropic`). - **Assessment:** deliberate Kernel-wide standardization on the Anthropic judge for the live-web adapters. The *prompt and decision logic are unchanged*, so the grading contract is preserved. **Do not revert.** Documented in SMOKE.md "Deviations" and `adapter_metadata.json`. (Caveat below.) @@ -139,11 +153,12 @@ solution/solve.sh,tests/test.sh,tests/webjudge.py}`. - **Upstream:** `auto_eval_res = 0 if 'NOT SUCCESS' in res else 1; if 'SUCCESS' not in res: auto_eval_res = None` (`auto_eval.py` lines 128–130). `None` (judge emitted neither marker) is returned and **excluded** from aggregation. -- **Ours:** `reward = 0 if "NOT SUCCESS" in verdict else (1 if "SUCCESS" in verdict else 0)` - (`webjudge.py` line 161) — ambiguous **fails closed to 0**, with the raw verdict saved to - `grading_details.json` for audit. Plus a Kernel-specific retry: a 400 citing `temperature` retries - once without it (newer Anthropic models reject `temperature`), and any HTTP/OSError also fails - closed to 0 with an `error` note (`webjudge.py` `_call_anthropic` + `main` try/except). +- **Ours:** `parseReward`: `NOT SUCCESS` -> 0 else `SUCCESS` -> 1 else 0 (`judge/src/prompts.ts`) — + ambiguous **fails closed to 0**, with the raw verdict saved to `grading_details.json` for audit. + Any error from the judge call (model resolution, a 4xx/5xx, a transient network failure) also + fails closed to 0 with an `error` note (`judge/src/judge.ts` `run` try/catch). The old manual + `temperature`-drop retry is gone: pi-ai owns the o-series `temperature`/`max_completion_tokens` + quirks and client-side retries, so the judge no longer hand-rolls them. - **Assessment:** Harbor's reward channel is a single float, so an abstain must become a number; fail-closed + audit trail is the right single-reward encoding and the verdict-marker logic otherwise matches upstream exactly (`NOT SUCCESS` wins over `SUCCESS`, same precedence). **Keep.** @@ -204,7 +219,8 @@ solution/solve.sh,tests/test.sh,tests/webjudge.py}`. `evaluation/run_eval.sh` / `README.md` auto-eval invocation (`--max_attached_imgs 15`). Edited two load-bearing places plus two docs that quoted the old default: - `src/webvoyager/task-template/task.toml`: `MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-15}"` - - `src/webvoyager/task-template/tests/webjudge.py`: `k = int(os.getenv("MAX_IMAGES", "15"))` + - the judge default (now `judge/src/judge.ts` `--max-images` default `15`, plumbed via `test.sh`; + originally `webjudge.py`'s `k = int(os.getenv("MAX_IMAGES", "15"))`). - `README.md` `[verifier.env]` table + `run_webvoyager.yaml` comment: default now `15`, and the stale "set `1` to match the paper" line corrected (canonical is 15, not 1). The `WEBVOYAGER_MAX_IMAGES` env override is preserved. This is the only change that affects parity @@ -225,8 +241,8 @@ solution/solve.sh,tests/test.sh,tests/webjudge.py}`. - Anthropic Messages judge instead of OpenAI GPT-4V (§4) — prompt + decision logic unchanged. - Whole `answer.txt` as `Result Response` instead of the brittle `ANSWER[; ]+[...]` regex (§6). - `ground_truth["task"]` instead of regexing `ques` out of agent logs (§7). -- `None`→0 fail-closed + `temperature` auto-drop + fail-closed-on-HTTP-error (§8). -- stdlib `urllib` POST instead of the `anthropic` SDK (no pip on the Kernel verifier VM). +- `None`→0 fail-closed + fail-closed-on-judge-error; pi-ai owns the o-series `temperature` quirks (§8). +- bundled pi-ai `node` bin instead of the OpenAI SDK (self-contained; no install on the Kernel verifier VM). - id slugification for `ORG_NAME_PATTERN` (§10). **Skipped — minor / no grading impact (left as-is):** @@ -242,8 +258,8 @@ solution/solve.sh,tests/test.sh,tests/webjudge.py}`. - Anthropic judge instead of OpenAI GPT-4V (#4) — prompt + logic unchanged. - Whole `answer.txt` instead of the `ANSWER[; ]+[...]` regex (#6). - `ground_truth["task"]` instead of regexing `ques` from agent logs (#7). -- `None`→0 fail-closed + `temperature` auto-drop + fail-closed-on-HTTP-error (#8). -- stdlib `urllib` POST instead of the `anthropic` SDK (no pip on the verifier VM). +- `None`→0 fail-closed + fail-closed-on-judge-error; pi-ai owns the o-series `temperature` quirks (#8). +- bundled pi-ai `node` bin instead of the OpenAI SDK (self-contained; no install on the verifier VM). - id slugification for `ORG_NAME_PATTERN` (#10). --- diff --git a/benchmarks/adapters/webvoyager/README.md b/benchmarks/adapters/webvoyager/README.md index 511f128..6e7dc9b 100644 --- a/benchmarks/adapters/webvoyager/README.md +++ b/benchmarks/adapters/webvoyager/README.md @@ -13,8 +13,8 @@ webvoyager-<site>--<n>/ ├── instruction.md # the ques + a "state the answer in your last line" directive ├── environment/kernel.json # { start_url: <web>, stealth: true, viewport: 1280x1024 } ├── tests/ -│ ├── test.sh # python3 webjudge.py (stdlib-only; the Kernel VM has no pip) -│ ├── webjudge.py # WebVoyager single-call multimodal judge (Anthropic port) +│ ├── test.sh # runs `node judge.js` (the Kernel VM ships node + fetch) +│ ├── judge.js # bundled WebVoyager single-call multimodal judge (built from judge/) │ └── ground_truth.json # { task, web_name, start_url, reference_answer, reference_type } ├── solution/solve.sh # oracle plumbing: writes the reference answer to answer.txt └── task.toml # name, timeouts, [verifier.env] judge config @@ -26,6 +26,12 @@ re-fetches it from upstream. ## Generate +Build the judge bundle first — generation copies it into each task's `tests/`: + +```bash +(cd adapters/webvoyager/judge && npm install && npm run build) +``` + ```bash cd benchmarks SRC=adapters/webvoyager/src/webvoyager/main.py @@ -34,10 +40,11 @@ python3 $SRC --output-dir adapters/webvoyager/.tasks --limit 20 # first 20 python3 $SRC --output-dir adapters/webvoyager/.tasks --task-ids Allrecipes--0 Amazon--3 ``` -The adapter has no third-party deps, so it runs on bare `python3` (no `uv sync` for -generation). `uv sync`/`npm` are only needed for the `harbor run` below. +The Python generator has no third-party deps, so it runs on bare `python3` (no `uv sync`). +The judge bundle (`judge/dist/judge.js`) must be built once before generation; `uv sync` +is only needed for the `harbor run` below. -`.tasks/` is gitignored. +`.tasks/` and `judge/dist/` are gitignored. ## Run (Kernel env + cua) @@ -55,16 +62,18 @@ uv run harbor run \ ## Verifier WebVoyager's grader is a single multimodal call (task + last-k screenshots + answer -> -`SUCCESS`/`NOT SUCCESS`), ported to the Anthropic Messages API in `webjudge.py` -(SYSTEM_PROMPT is verbatim from upstream `evaluation/auto_eval.py`). It reads the agent -answer from `/logs/agent/answer.txt` and the spilled screenshots from -`/logs/agent/shots/shot-<n>.png` (both written by the shared cua agent / Node entrypoint), -and writes a single `0|1` reward to `/logs/verifier/reward.txt` plus a `grading_details.json`. +`SUCCESS`/`NOT SUCCESS`), a self-contained `node` bin under `judge/` that calls the model +through [`@earendil-works/pi-ai`](https://www.npmjs.com/package/@earendil-works/pi-ai) +(SYSTEM_PROMPT is verbatim from upstream `evaluation/auto_eval.py`). pi-ai is bundled into +`judge.js` so the verifier runs with no install. It reads the agent answer from +`/logs/agent/answer.txt` and the spilled screenshots from `/logs/agent/shots/shot-<n>.png` +(both written by the shared cua agent / Node entrypoint), and writes a single `0|1` reward +to `/logs/verifier/reward.txt` plus a `grading_details.json`. `[verifier.env]` knobs (per task, overridable from the host): | var | default | meaning | |---|---|---| | `ANTHROPIC_API_KEY` | `${ANTHROPIC_API_KEY}` | resolved from the host at exec time | -| `JUDGE_MODEL` | `claude-sonnet-4-5` | vision judge model (`WEBVOYAGER_JUDGE_MODEL` to override) | +| `JUDGE_MODEL` | `claude-sonnet-4-5` | vision judge model; a pi-ai `provider:name` ref (bare name = `anthropic`), e.g. `openai:o4-mini` (`WEBVOYAGER_JUDGE_MODEL` to override) | | `MAX_IMAGES` | `15` | last-k screenshots sent to the judge (`WEBVOYAGER_MAX_IMAGES`; `15` matches the canonical `auto_eval.py` invocation) | diff --git a/benchmarks/adapters/webvoyager/SMOKE.md b/benchmarks/adapters/webvoyager/SMOKE.md index 7baeb81..cb75c20 100644 --- a/benchmarks/adapters/webvoyager/SMOKE.md +++ b/benchmarks/adapters/webvoyager/SMOKE.md @@ -1,5 +1,14 @@ # WebVoyager adapter — live smoke +> **Transport update (post-smoke).** This run used the original Python judge +> (`webjudge.py`, stdlib `urllib` POST to the Anthropic Messages API). The judge +> has since been reimplemented as a self-contained `node` bin +> (`judge/src/`, bundled to `judge.js`) calling the model through +> `@earendil-works/pi-ai`. That is a transport-only change — the SYSTEM_PROMPT, +> last-k selection, `SUCCESS`/`NOT SUCCESS` parse, and `claude-sonnet-4-5` default +> are unchanged — so the findings below still hold, but references to +> `webjudge.py` / `urllib` describe the transport at smoke time, not today's. + Live 20-task smoke on the Kernel env with the cua agent. Goal: exercise the real pipeline end to end, learn failure modes, fix adapter bugs surfaced. Learning smoke, **not** a definitive WebVoyager number. diff --git a/benchmarks/adapters/webvoyager/adapter_metadata.json b/benchmarks/adapters/webvoyager/adapter_metadata.json index 58cfcc5..96ae41a 100644 --- a/benchmarks/adapters/webvoyager/adapter_metadata.json +++ b/benchmarks/adapters/webvoyager/adapter_metadata.json @@ -17,6 +17,7 @@ "type": "multimodal-llm-judge", "port_of": "WebVoyager evaluation/auto_eval.py (single-call GPT-4V judge)", "provider": "anthropic", - "default_model": "claude-sonnet-4-5" + "default_model": "claude-sonnet-4-5", + "transport": "@earendil-works/pi-ai (bundled node bin)" } } diff --git a/benchmarks/adapters/webvoyager/judge/package-lock.json b/benchmarks/adapters/webvoyager/judge/package-lock.json new file mode 100644 index 0000000..8905d88 --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/package-lock.json @@ -0,0 +1,3610 @@ +{ + "name": "webvoyager-webjudge", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "webvoyager-webjudge", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "0.79.1" + }, + "bin": { + "webvoyager-webjudge": "dist/judge.js" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "tsdown": "^0.22.2", + "typescript": "5.9.3", + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1048.0.tgz", + "integrity": "sha512-u+NT61JZEkRFtpL0CAw1N1dwxnaLgwVXQl/zjJxTGgLyS/jTIdg2SdoEoCTHxgDyCnqa1HEi9QOoE9/pYRNpOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/credential-provider-node": "^3.972.42", + "@aws-sdk/eventstream-handler-node": "^3.972.16", + "@aws-sdk/middleware-eventstream": "^3.972.12", + "@aws-sdk/middleware-websocket": "^3.972.19", + "@aws-sdk/token-providers": "3.1048.0", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/fetch-http-handler": "^5.4.2", + "@smithy/node-http-handler": "^4.7.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.974.23", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.23.tgz", + "integrity": "sha512-MiWR/uWjxjFXGzrE0Ghc5lWxUxzHsUWFhV+OX7M4cR9SrmrnZs6TXavnCWnzzdwJeFri34xQo81rvGNzK3c4BQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.13", + "@aws-sdk/xml-builder": "^3.972.31", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/core": "^3.24.6", + "@smithy/signature-v4": "^5.4.6", + "@smithy/types": "^4.14.3", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.49", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.49.tgz", + "integrity": "sha512-liB3yQNHCM9k/gu/w36XHMKPluT7HTlnGUhRbBGSISDQkcr/Sy1zsZabiuvQj8WG5yW573u9RehrBvvnIQ9OEQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.51", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.51.tgz", + "integrity": "sha512-XET0H2oofciJ5lMRWNIvRjAP7Q3wv2XT+JtJJEdhPWUMwe3TvQ9qcxonpu7vXmNngncvFpi4E2It+Tamas/naA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/fetch-http-handler": "^5.4.6", + "@smithy/node-http-handler": "^4.7.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http/node_modules/@smithy/node-http-handler": { + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.8.2.tgz", + "integrity": "sha512-wfl1uwrAqMH9/pi4kqBo5LBcFwrJLxuDLqL7p7qNcJIFcyZDUc6pzhYk4CYv+DP7fIUpQCZumwNnkhPKS52osQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.972.56", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.56.tgz", + "integrity": "sha512-IAmc61hbgQiHht9U3x0tnRwz0lzdwOwD/i9voRgdJrKamF+JtmrBOsW9GwB7mfFonNWOWL4qARWYrF8veEMe3w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/credential-provider-env": "^3.972.49", + "@aws-sdk/credential-provider-http": "^3.972.51", + "@aws-sdk/credential-provider-login": "^3.972.55", + "@aws-sdk/credential-provider-process": "^3.972.49", + "@aws-sdk/credential-provider-sso": "^3.972.55", + "@aws-sdk/credential-provider-web-identity": "^3.972.55", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/credential-provider-imds": "^4.3.7", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.55", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.55.tgz", + "integrity": "sha512-hBBkANo3cDn+h2qxxzER4a+J8JCO9o9Z/YYmU7iky6AcaarX5RRdRcHNC6SLdwY0vAXQygn6soUbDqPn3GghaA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.58", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.58.tgz", + "integrity": "sha512-OyCLVmSI7pZO8hxwNVX6pXhTVlJqRBTp+ijdEfJSUj0RyjHnF602OfAarOzGq6wkGodeFkYBt8MmJ6A6ycRgWw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.49", + "@aws-sdk/credential-provider-http": "^3.972.51", + "@aws-sdk/credential-provider-ini": "^3.972.56", + "@aws-sdk/credential-provider-process": "^3.972.49", + "@aws-sdk/credential-provider-sso": "^3.972.55", + "@aws-sdk/credential-provider-web-identity": "^3.972.55", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/credential-provider-imds": "^4.3.7", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.49", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.49.tgz", + "integrity": "sha512-C8h36lBuC/RnBSsjlO+dn6xZm3KbAl5vpJaVPAfQnMmz2/OISmKOc8XZcqMQgO2ADwBYNRMM6Kf3vz9G/TulMQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.972.55", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.55.tgz", + "integrity": "sha512-1FkOz74Ea5QGS9jtIoXp55T/IkSS3spv+nLTT07fRY/+T5xmEOqaYBVIaEmX4zTNvbV6g2lrtlaVKWEoNyJt3w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/token-providers": "3.1074.0", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso/node_modules/@aws-sdk/token-providers": { + "version": "3.1074.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1074.0.tgz", + "integrity": "sha512-pv80IzgGW4RnXWtft692chZOM9i6PhebVsLCcnaM4dBEPZva2fE6FXAHs76G7Rc7s3yGyX/68G0nZMrUy+Vmpg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.55", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.55.tgz", + "integrity": "sha512-g2BoECD1q01kTPByi56+VLVvdWDzMkKIcr77qixpqH0okw2t0U5CoPv+6S8v/D1Y2Wa6QKKtn6XAtDzP+Kfpvg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/eventstream-handler-node": { + "version": "3.972.22", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.22.tgz", + "integrity": "sha512-tqPJv0dz4+O0hWGm1a6YekcMZyPhDFs/zH73Von7icaVT5n0Jqvm86typ3jRrG+qoUdPhALOnboRLTmnWQTlYQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-eventstream": { + "version": "3.972.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.18.tgz", + "integrity": "sha512-OHpk8YoZi3yexPq8aFt1vN1IxA2zLKvsIR5GpWYylX/ve6kQmY7wxHNSFy/D3t2apMZ16rs76Co4dJWcDyIk3A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-websocket": { + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.31.tgz", + "integrity": "sha512-ps1rumU1LybSFHaW9dTDgkhCMJLVaedEY78kKSzUDDY+b9974/g6aiaYYA0U9WV0oL4CJCJrVWG+EZ/qr4or7g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/fetch-http-handler": "^5.4.6", + "@smithy/signature-v4": "^5.4.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.23", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.23.tgz", + "integrity": "sha512-gO93ZPsI2bxeFZD42f1/qjDw6FAZkNZcKRO94LIiT03fzOmcJ9e/tunxjVjA1Rl69ClmVJzz8H3G9CdKef10PA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/signature-v4-multi-region": "^3.996.35", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/fetch-http-handler": "^5.4.6", + "@smithy/node-http-handler": "^4.7.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients/node_modules/@smithy/node-http-handler": { + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.8.2.tgz", + "integrity": "sha512-wfl1uwrAqMH9/pi4kqBo5LBcFwrJLxuDLqL7p7qNcJIFcyZDUc6pzhYk4CYv+DP7fIUpQCZumwNnkhPKS52osQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.35", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.35.tgz", + "integrity": "sha512-6L/VWs+Wch2stHemCGTmUNqKLMzURxQDK5boNG3Jn3kAOp71meDUuS5sbObpEvFxHDq0uWeSLFDNSYsjNt+Dlg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.13", + "@smithy/signature-v4": "^5.4.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1048.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1048.0.tgz", + "integrity": "sha512-k0y/GcuesuSfWyUM0WamrGyeZmltRYaPbHO82UDA6mZ/doB+FOHKutikPAtSXMn/hDz970cF+iRuuiYO9VEbAA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.11", + "@aws-sdk/nested-clients": "^3.997.9", + "@aws-sdk/types": "^3.973.8", + "@smithy/core": "^3.24.2", + "@smithy/types": "^4.14.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.973.13", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.13.tgz", + "integrity": "sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.965.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.8.tgz", + "integrity": "sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.31.tgz", + "integrity": "sha512-SzE4Pgyl+hDF+BuyuzxUSpwnuUu9lJuO1YGgteG89/4Qv0+2IQiVQqdbPV32IozLvXWQChPQcdkk/sKvb1QHiQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", + "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@babel/generator": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-8.0.0.tgz", + "integrity": "sha512-NT9NrVwJsbSV6Y2FSstWa71EETOnzrjkL5/wX3D2mYHtKM+qvqB1DvR4D0Setb/gDBsHzRICifwEWMO8CnTF6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^8.0.0", + "@babel/types": "^8.0.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "@types/jsesc": "^2.5.0", + "jsesc": "^3.0.2" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.0.tgz", + "integrity": "sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.2.tgz", + "integrity": "sha512-9Fr9QeyCAyi1BR1jKZ6uYQ24EIhQUx5ReHfQU7drOE+TPOb+w11/dsqLkMOT2U29OdCT71XajrOT8xDc1C7orA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/parser": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.0.tgz", + "integrity": "sha512-aLxAE+imI9bCcyaPrUDjBv3uSkWieifjLe0kuFOZF0zli0L6GCsTmsePnTr55adbIAgYz2zhN1vnFimCBUYcRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^8.0.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.0.tgz", + "integrity": "sha512-K8ponJDxBwDHigkeFqaqT5wLGl4bTlwMafR8k7b5CPxr6Ww+UG9ls8Yx6Tcpboxu97eeGVEEyKcHmEyOwN1vSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^8.0.0", + "@babel/helper-validator-identifier": "^8.0.0" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/@earendil-works/pi-ai": { + "version": "0.79.1", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.79.1.tgz", + "integrity": "sha512-UnORwrcsTNLm4StEvoM8iEom0u87Te7BXEWxhec3iNXygWD6eEBosUoq9ddcveqtj/QpUZBMPWUu81cCtZxzkQ==", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "0.91.1", + "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@google/genai": "1.52.0", + "@mistralai/mistralai": "2.2.1", + "@smithy/node-http-handler": "4.7.3", + "http-proxy-agent": "7.0.2", + "https-proxy-agent": "7.0.6", + "openai": "6.26.0", + "partial-json": "0.1.7", + "typebox": "1.1.38" + }, + "bin": { + "pi-ai": "dist/cli.js" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@google/genai": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/@google/genai/-/genai-1.52.0.tgz", + "integrity": "sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "google-auth-library": "^10.3.0", + "p-retry": "^4.6.2", + "protobufjs": "^7.5.4", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@modelcontextprotocol/sdk": "^1.25.2" + }, + "peerDependenciesMeta": { + "@modelcontextprotocol/sdk": { + "optional": true + } + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mistralai/mistralai": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-2.2.1.tgz", + "integrity": "sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ==", + "license": "Apache-2.0", + "dependencies": { + "ws": "^8.18.0", + "zod": "^3.25.0 || ^4.0.0", + "zod-to-json-schema": "^3.25.0" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.137.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.137.0.tgz", + "integrity": "sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", + "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", + "license": "BSD-3-Clause" + }, + "node_modules/@quansync/fs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@quansync/fs/-/fs-1.0.0.tgz", + "integrity": "sha512-4TJ3DFtlf1L5LDMaM6CanJ/0lckGNtJcMjQ1NAV6zDmA0tEHKZtxNKin8EgPaVX1YzljbxckyT2tJrpQKAtngQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.3.tgz", + "integrity": "sha512-DT6Z3PhvioeHMvxo+xHc3KtqggrI7CCTXCmC2h/5zUlp5jVitv7XEy+9q5/7v8IolhlioawpMo8Kg0EEBy7J0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.3.tgz", + "integrity": "sha512-0NwgwsjM7LrsuVnXMK3koTpagBNOhloc/BNjKqZjv4V5zI5r13qx69uVhRx+o5Z0yy4Hzq+lpy7TAgUG/ocvrw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.3.tgz", + "integrity": "sha512-YtiBp4disu6V560loT6PjMdiRaWmVvDNrUunAalbiFx2ggeJwxdAsgZMcoGP17uyAsTwAj5V1niksxlHnVQ1Sw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.3.tgz", + "integrity": "sha512-yD3EkEdXk2LypPxnf/kSZHirarsI8gcPzc62SukhR9VJTyvV+F9Q/GxWNuCojc7sXyuVC4DxRGhdDK4X8VSsbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.3.tgz", + "integrity": "sha512-c+8vieQbsD7HNAHKIA34w0GJ9FedFFuJGD+7E6vz7Q3uqAIugL5p45fhlsj4UaAsHpcmlqugBWMhA0/j7o0sIg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.3.tgz", + "integrity": "sha512-50jD0uUwLvur7Zz9LHz17kaAdTPjn5wN93hEgjvmYFRZwiR7ZJYovTd5ipyWJDAnXKvZ+wgc+/Ika6dwSF5OcA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.3.tgz", + "integrity": "sha512-BO9+oPL8K9poZJBfYPsXNtYjPE5uM3qeehT3aFcW4LITOl+iSqhp0abzjR2nWBUNjIZeKXjAEWBZ64WjNoHd6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.3.tgz", + "integrity": "sha512-f3VpLB1vQ0Eo6ecr/6cekLnvYMFF4YBFoVGkfkvPLq1bAkbAwHYQPZKoAmG6OJyTcxxoC+AvezGx/S1obNC0Mw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.3.tgz", + "integrity": "sha512-AmurZ26Pqx/RI9N1gzEOCklkKXl927yjfXWUUS0O7Puh8ARM/Ob8qfrD3qnWksScdw6cSrW5PSHE9DyLu7+PtA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.3.tgz", + "integrity": "sha512-JJpqs8bRGITDOdbkNKnlojzBabbOHrqjSvDr0IVsZObE1lBcPjxItUEY9eWIDbxaJ3cGrXPWGfGkIxFijg/URg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.3.tgz", + "integrity": "sha512-rSJcdjPxzA/by/6/rYs+v+bXU7UjvnbUWz8MJb6kh6+knqB1dCrtHg0uu7C/4haqJvqdkYHQ5IGn+tCH9GLW/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.3.tgz", + "integrity": "sha512-hQ3/PYkDJICgevvyNcVrihVeqq7k1Pp3VZ9lY+dauAYUJKO+auqApvANhvR1An9BhmqYKvW2Mu1F9u4DXSMLxQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.3.tgz", + "integrity": "sha512-Elcv/BtML9lXrV6JuKITc/grN2kYV9gjsQpW8Jfw4ioK0TOkjBjye0nnyqQNy9STNaI20lXNaQBRrD5gSgR0Yg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.3.tgz", + "integrity": "sha512-2DrEfhluH9yhiaFApmsjsjwrSYbNcY1oFTzYSP1a535jDbV98zCFanA/96TBUd0iDFcxGmw9QRExwGCXz3U+/g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.3.tgz", + "integrity": "sha512-OL4OMk7UPXOeVGGd3qo5zJyPIljf4AFgk5QAkPPS+OoLuOOozhuaQGC18MxVTnw/06q93gShAJzlwnSCY9YtqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", + "integrity": "sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.2.tgz", + "integrity": "sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.2.tgz", + "integrity": "sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.2.tgz", + "integrity": "sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.2.tgz", + "integrity": "sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.2.tgz", + "integrity": "sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.2.tgz", + "integrity": "sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.2.tgz", + "integrity": "sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.2.tgz", + "integrity": "sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.2.tgz", + "integrity": "sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.2.tgz", + "integrity": "sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.2.tgz", + "integrity": "sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.2.tgz", + "integrity": "sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.2.tgz", + "integrity": "sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.2.tgz", + "integrity": "sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.2.tgz", + "integrity": "sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.2.tgz", + "integrity": "sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.2.tgz", + "integrity": "sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.2.tgz", + "integrity": "sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.2.tgz", + "integrity": "sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.2.tgz", + "integrity": "sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.2.tgz", + "integrity": "sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.2.tgz", + "integrity": "sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.2.tgz", + "integrity": "sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.2.tgz", + "integrity": "sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@smithy/core": { + "version": "3.26.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.26.0.tgz", + "integrity": "sha512-mLUktFAn+Pa2agl1J7VgtYNFWCX8/b4GMJSK1hCu4YCvtBfM6F8Os3EP4ry+DFFlXOf3wyvlgXhuUdFoy52D3g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.2.tgz", + "integrity": "sha512-18UMDMyrAbDcpmL1gLUA7ww0fRTcdCrSjSJOi2Sbld+tVjwD/pW+OAwjlScFLR7vvBnhZrIPQ7kVuTf1mnJLug==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.5.2.tgz", + "integrity": "sha512-Ei/UK/QMhq0rKaMqGPlOAkE2yS9DZeYmZdk1RAKc3vp3zxgleZHZyBLlZv8yLsxljX4svCRuMTD6u3LLIcU4Bg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.7.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.7.3.tgz", + "integrity": "sha512-/jPhevcTFPMVl6KNjbaI47iOg1zxC7IsnX4PQDGVZKMFceOXtB8IEYaB7a9VvkP/3oC60WzTeKocvSI7vLT0vA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.24.3", + "@smithy/types": "^4.14.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.5.2.tgz", + "integrity": "sha512-7xHpmPY4rt0IOmeAA8EfjgEH8isT+587TCdy9H6a7d4OMi5CQ0oEHhWllunvPu4j4Cq0vTFwdxXN/kABWPjdyA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.15.0.tgz", + "integrity": "sha512-Z5TAOxygoFvybJV3igo5SloFflSokHx2hu1eFA+DxDTcn+FtKxUSui+rbTRG1pAafMA888Z3MVvCWUuvCrTXjg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/jsesc": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@types/jsesc/-/jsesc-2.5.1.tgz", + "integrity": "sha512-9VN+6yxLOPLOav+7PwjZbxiID2bVaeq0ED4qSQmdQTdjnXJSaCVKTR58t15oqH1H5t8Ng2ZX1SabJVoN9Q34bw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.20.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.0.tgz", + "integrity": "sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "license": "MIT" + }, + "node_modules/@vitest/expect": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.6.tgz", + "integrity": "sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/spy": "3.2.6", + "@vitest/utils": "3.2.6", + "chai": "^5.2.0", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.6.tgz", + "integrity": "sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "3.2.6", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.17" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.6.tgz", + "integrity": "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.6.tgz", + "integrity": "sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "3.2.6", + "pathe": "^2.0.3", + "strip-literal": "^3.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.6.tgz", + "integrity": "sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.6", + "magic-string": "^0.30.17", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.6.tgz", + "integrity": "sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyspy": "^4.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.6.tgz", + "integrity": "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "3.2.6", + "loupe": "^3.1.4", + "tinyrainbow": "^2.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansis": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/ansis/-/ansis-4.3.1.tgz", + "integrity": "sha512-BJ8/l4R5LRE7hW9WdSuGYrLSHi2ynxeFpDFbH0K/CgNeY/tyhk+vO6TYxXC5r5CpUhNVX310xzPsN/H9lCdfOA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/ast-kit": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ast-kit/-/ast-kit-3.0.0.tgz", + "integrity": "sha512-8OG92q3R35qjC/4i6BLBMg8IB+fClWu/1PEwg2Z9Rn+BuNaiEgJzpzn+pxWOdHJWDCAwu2JP0wCDTozAM4QirQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^8.0.0", + "estree-walker": "^3.0.3", + "pathe": "^2.0.3" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/birpc": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/birpc/-/birpc-4.0.0.tgz", + "integrity": "sha512-LShSxJP0KTmd101b6DRyGBj57LZxSDYWKitQNW/mi8GRMvZb078Uf9+pveax1DrVL89vm7mWe+TovdI/UDOuPw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/cac": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cac/-/cac-7.0.0.tgz", + "integrity": "sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/chai": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", + "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "assertion-error": "^2.0.1", + "check-error": "^2.1.1", + "deep-eql": "^5.0.1", + "loupe": "^3.1.0", + "pathval": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/check-error": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", + "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-eql": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", + "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/defu": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", + "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/dts-resolver": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/dts-resolver/-/dts-resolver-3.0.0.tgz", + "integrity": "sha512-1T1f+z+4tl9XD+m+0HBgWoL/nm0bOIffyWaUuUSBlFg/86IWvfx+wjNaO/ybU0AJzG9/Mi5hBUgGV6zCmWEN7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "oxc-resolver": ">=11.0.0" + }, + "peerDependenciesMeta": { + "oxc-resolver": { + "optional": true + } + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/empathic": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/empathic/-/empathic-2.0.1.tgz", + "integrity": "sha512-YGRs8knHhKHVShLkFET/rWAU8kmHbOV5LwN938RHI0pljAJ1Gf6SzXsSmRaEzcXTtOOmVqJ5+WtQPL5uigY50Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gaxios": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.5.tgz", + "integrity": "sha512-5FZy72Rh8LhtjmvDrKkI+lVhrsQrVKVsItxMoDm5mNQE+xR0WVIIs+jzPSJgBvKVsLi24fZhXJIsNI0bihDzFg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/get-tsconfig": { + "version": "5.0.0-beta.5", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-5.0.0-beta.5.tgz", + "integrity": "sha512-/6gFNr0N04nob252sTQxyFLi3eKFRqIg1I87YcqAMT1i6SQrSF6KujUEQrtrjMV0H/eejTCltLdDSTEMzHbnsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "engines": { + "node": ">=20.20.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/google-auth-library": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.0.tgz", + "integrity": "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/hookable": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/hookable/-/hookable-6.1.1.tgz", + "integrity": "sha512-U9LYDy1CwhMCnprUfeAZWZGByVbhd54hwepegYTK7Pi5NvqEj63ifz5z+xukznehT7i6NIZRu89Ay1AZmRsLEQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/import-without-cache": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/import-without-cache/-/import-without-cache-0.4.0.tgz", + "integrity": "sha512-NkJQA7oZ4YHQhd2+H3BoRFKF3d/XNsiKpHZCQEMH9pDX27hQQLsTyOocyRgaIVtf8gHX3Nt3LPkR4e5EdtPAGQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/js-tokens": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", + "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/loupe": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", + "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/obug": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", + "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/openai": { + "version": "6.26.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.26.0.tgz", + "integrity": "sha512-zd23dbWTjiJ6sSAX6s0HrCZi41JwTA1bQVs0wLQPZ2/5o2gxOJA5wh7yOAUgwYybfhDXyhwlpeQf7Mlgx8EOCA==", + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/pathval": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", + "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.16" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.15", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", + "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/protobufjs": { + "version": "7.6.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.4.tgz", + "integrity": "sha512-RJJPTTpvFfHcWLkIa2JFWK4XvtSzS0yEWDmunqHXli1h3JlkbcQZXDZdcWxv+JK3Xsl5/UFDPZ0iGm7DAengYw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/quansync": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/quansync/-/quansync-1.0.0.tgz", + "integrity": "sha512-5xZacEEufv3HSTPQuchrvV6soaiACMFnq1H8wkVioctoH3TRha9Sz66lOxRwPK/qZj7HPiSveih9yAyh98gvqA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rolldown": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.3.tgz", + "integrity": "sha512-1F1eEtUBtFvcGm1HQ9TiUIUHPQG7mSAODrhIzjxoUEFuo8OcbrGLiVLkevNgj84TE4lnHvnumwFjhJO5Eu135g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.137.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.1.3", + "@rolldown/binding-darwin-arm64": "1.1.3", + "@rolldown/binding-darwin-x64": "1.1.3", + "@rolldown/binding-freebsd-x64": "1.1.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.3", + "@rolldown/binding-linux-arm64-gnu": "1.1.3", + "@rolldown/binding-linux-arm64-musl": "1.1.3", + "@rolldown/binding-linux-ppc64-gnu": "1.1.3", + "@rolldown/binding-linux-s390x-gnu": "1.1.3", + "@rolldown/binding-linux-x64-gnu": "1.1.3", + "@rolldown/binding-linux-x64-musl": "1.1.3", + "@rolldown/binding-openharmony-arm64": "1.1.3", + "@rolldown/binding-wasm32-wasi": "1.1.3", + "@rolldown/binding-win32-arm64-msvc": "1.1.3", + "@rolldown/binding-win32-x64-msvc": "1.1.3" + } + }, + "node_modules/rolldown-plugin-dts": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/rolldown-plugin-dts/-/rolldown-plugin-dts-0.26.0.tgz", + "integrity": "sha512-e+kEPtUiDES0htk5iqkSeF4EzAV7R+vugGB44iPDuw1Kw9E+WyL1VG7PaV0IIjGHLiacztMBcMTyrr8ON9CT1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/generator": "^8.0.0", + "@babel/helper-validator-identifier": "^8.0.0", + "@babel/parser": "^8.0.0", + "ast-kit": "^3.0.0", + "birpc": "^4.0.0", + "dts-resolver": "^3.0.0", + "get-tsconfig": "5.0.0-beta.5", + "obug": "^2.1.3" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@ts-macro/tsc": "^0.3.6", + "@typescript/native-preview": ">=7.0.0-dev.20260325.1", + "rolldown": "^1.0.0", + "typescript": "^5.0.0 || ^6.0.0", + "vue-tsc": "~3.2.0 || ~3.3.0" + }, + "peerDependenciesMeta": { + "@ts-macro/tsc": { + "optional": true + }, + "@typescript/native-preview": { + "optional": true + }, + "typescript": { + "optional": true + }, + "vue-tsc": { + "optional": true + } + } + }, + "node_modules/rollup": { + "version": "4.62.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz", + "integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.62.2", + "@rollup/rollup-android-arm64": "4.62.2", + "@rollup/rollup-darwin-arm64": "4.62.2", + "@rollup/rollup-darwin-x64": "4.62.2", + "@rollup/rollup-freebsd-arm64": "4.62.2", + "@rollup/rollup-freebsd-x64": "4.62.2", + "@rollup/rollup-linux-arm-gnueabihf": "4.62.2", + "@rollup/rollup-linux-arm-musleabihf": "4.62.2", + "@rollup/rollup-linux-arm64-gnu": "4.62.2", + "@rollup/rollup-linux-arm64-musl": "4.62.2", + "@rollup/rollup-linux-loong64-gnu": "4.62.2", + "@rollup/rollup-linux-loong64-musl": "4.62.2", + "@rollup/rollup-linux-ppc64-gnu": "4.62.2", + "@rollup/rollup-linux-ppc64-musl": "4.62.2", + "@rollup/rollup-linux-riscv64-gnu": "4.62.2", + "@rollup/rollup-linux-riscv64-musl": "4.62.2", + "@rollup/rollup-linux-s390x-gnu": "4.62.2", + "@rollup/rollup-linux-x64-gnu": "4.62.2", + "@rollup/rollup-linux-x64-musl": "4.62.2", + "@rollup/rollup-openbsd-x64": "4.62.2", + "@rollup/rollup-openharmony-arm64": "4.62.2", + "@rollup/rollup-win32-arm64-msvc": "4.62.2", + "@rollup/rollup-win32-ia32-msvc": "4.62.2", + "@rollup/rollup-win32-x64-gnu": "4.62.2", + "@rollup/rollup-win32-x64-msvc": "4.62.2", + "fsevents": "~2.3.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", + "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/strip-literal": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", + "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tokens": "^9.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinypool": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", + "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/tinyrainbow": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", + "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tinyspy": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", + "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tree-kill": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", + "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "dev": true, + "license": "MIT", + "bin": { + "tree-kill": "cli.js" + } + }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/tsdown": { + "version": "0.22.3", + "resolved": "https://registry.npmjs.org/tsdown/-/tsdown-0.22.3.tgz", + "integrity": "sha512-louqbfA8Qf//B9jTTL0FPtXTNpjCWv1VPkbcmQMph2pTpzs+LnB1tbe4tDDRVpo2BjF5SgUXaTZe45SxB8pWHg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansis": "^4.3.1", + "cac": "^7.0.0", + "defu": "^6.1.7", + "empathic": "^2.0.1", + "hookable": "^6.1.1", + "import-without-cache": "^0.4.0", + "obug": "^2.1.3", + "picomatch": "^4.0.4", + "rolldown": "~1.1.1", + "rolldown-plugin-dts": "^0.26.0", + "semver": "^7.8.4", + "tinyexec": "^1.2.4", + "tinyglobby": "^0.2.17", + "tree-kill": "^1.2.2", + "unconfig-core": "^7.5.0" + }, + "bin": { + "tsdown": "dist/run.mjs" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@arethetypeswrong/core": "^0.18.1", + "@tsdown/css": "0.22.3", + "@tsdown/exe": "0.22.3", + "@vitejs/devtools": "*", + "publint": "^0.3.8", + "tsx": "*", + "typescript": "^5.0.0 || ^6.0.0", + "unplugin-unused": "^0.5.0", + "unrun": "*" + }, + "peerDependenciesMeta": { + "@arethetypeswrong/core": { + "optional": true + }, + "@tsdown/css": { + "optional": true + }, + "@tsdown/exe": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "publint": { + "optional": true + }, + "tsx": { + "optional": true + }, + "typescript": { + "optional": true + }, + "unplugin-unused": { + "optional": true + }, + "unrun": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typebox": { + "version": "1.1.38", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.1.38.tgz", + "integrity": "sha512-pZ0aQPmMmXoUvSbeuWf/Hzsc+avNw/Zd6VeE8CFgkVGWyuHPJvqeJJDeJqLve+K70LvjYIoleGcoJHPT17cWoA==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unconfig-core": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/unconfig-core/-/unconfig-core-7.5.0.tgz", + "integrity": "sha512-Su3FauozOGP44ZmKdHy2oE6LPjk51M/TRRjHv2HNCWiDvfvCoxC2lno6jevMA91MYAdCdwP05QnWdWpSbncX/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@quansync/fs": "^1.0.0", + "quansync": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "license": "MIT" + }, + "node_modules/vite": { + "version": "7.3.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", + "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0 || ^0.28.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-node": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", + "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cac": "^6.7.14", + "debug": "^4.4.1", + "es-module-lexer": "^1.7.0", + "pathe": "^2.0.3", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + }, + "bin": { + "vite-node": "vite-node.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/vite-node/node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/vitest": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.6.tgz", + "integrity": "sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/chai": "^5.2.2", + "@vitest/expect": "3.2.6", + "@vitest/mocker": "3.2.6", + "@vitest/pretty-format": "^3.2.6", + "@vitest/runner": "3.2.6", + "@vitest/snapshot": "3.2.6", + "@vitest/spy": "3.2.6", + "@vitest/utils": "3.2.6", + "chai": "^5.2.0", + "debug": "^4.4.1", + "expect-type": "^1.2.1", + "magic-string": "^0.30.17", + "pathe": "^2.0.3", + "picomatch": "^4.0.2", + "std-env": "^3.9.0", + "tinybench": "^2.9.0", + "tinyexec": "^0.3.2", + "tinyglobby": "^0.2.14", + "tinypool": "^1.1.1", + "tinyrainbow": "^2.0.0", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", + "vite-node": "3.2.4", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@types/debug": "^4.1.12", + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "@vitest/browser": "3.2.6", + "@vitest/ui": "3.2.6", + "happy-dom": "*", + "jsdom": "*" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@types/debug": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + } + } + }, + "node_modules/vitest/node_modules/tinyexec": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", + "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/benchmarks/adapters/webvoyager/judge/package.json b/benchmarks/adapters/webvoyager/judge/package.json new file mode 100644 index 0000000..3b44e2a --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/package.json @@ -0,0 +1,28 @@ +{ + "name": "webvoyager-webjudge", + "version": "0.1.0", + "description": "Self-contained WebVoyager single-call multimodal judge bin for the Harbor adapter; runs in-VM via node and grades the agent's answer + last-k screenshots through pi-ai (default claude-sonnet-4-5)", + "license": "MIT", + "private": true, + "type": "module", + "bin": { + "webvoyager-webjudge": "./dist/judge.js" + }, + "scripts": { + "build": "tsdown && chmod +x dist/judge.js", + "typecheck": "tsc --noEmit", + "test": "vitest --run" + }, + "engines": { + "node": ">=22.0.0" + }, + "dependencies": { + "@earendil-works/pi-ai": "0.79.1" + }, + "devDependencies": { + "@types/node": "^22.10.0", + "tsdown": "^0.22.2", + "typescript": "5.9.3", + "vitest": "^3.2.4" + } +} diff --git a/benchmarks/adapters/webvoyager/judge/src/artifacts.ts b/benchmarks/adapters/webvoyager/judge/src/artifacts.ts new file mode 100644 index 0000000..c68fdc8 --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/src/artifacts.ts @@ -0,0 +1,51 @@ +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { extname, join } from "node:path"; +import type { GroundTruth } from "./types.ts"; + +/** A screenshot the cua agent spilled: bytes plus the media type the judge needs. */ +export interface Shot { + name: string; + base64: string; + mimeType: string; +} + +const MEDIA_BY_SUFFIX: Record<string, string> = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".webp": "image/webp", +}; + +/** Numeric sort key. Shots are `shot-<n>.png` (not zero-padded). */ +function shotKey(name: string): number { + const stem = name.slice(0, name.length - extname(name).length); + const match = stem.match(/(\d+)/); + return match ? Number(match[1]) : 0; +} + +export function loadGroundTruth(path: string): GroundTruth { + return JSON.parse(readFileSync(path, "utf8")) as GroundTruth; +} + +export function loadAnswer(path: string): string { + return existsSync(path) ? readFileSync(path, "utf8").trim() : ""; +} + +/** + * The last `k` screenshots in numeric order, read into base64. + * + * Mirrors upstream `auto_eval.py`: sort the image files by numeric index and + * take the final `k` (`matches[-img_num:]`). cua spills one screenshot per + * step, so the last-k window is where the deciding frame can live. + */ +export function lastShots(shotsDir: string, k: number): Shot[] { + if (!existsSync(shotsDir)) return []; + const names = readdirSync(shotsDir) + .filter((name) => extname(name).toLowerCase() in MEDIA_BY_SUFFIX) + .sort((a, b) => shotKey(a) - shotKey(b)); + return names.slice(-k).map((name) => ({ + name, + base64: readFileSync(join(shotsDir, name)).toString("base64"), + mimeType: MEDIA_BY_SUFFIX[extname(name).toLowerCase()], + })); +} diff --git a/benchmarks/adapters/webvoyager/judge/src/judge.ts b/benchmarks/adapters/webvoyager/judge/src/judge.ts new file mode 100644 index 0000000..76c4358 --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/src/judge.ts @@ -0,0 +1,119 @@ +/** + * WebVoyager single-call multimodal judge, verifier entrypoint for the Harbor + * adapter. + * + * Runs inside the Kernel browser VM (which ships `node` + global `fetch`) as a + * self-contained bundle — no `npm install` at verify time. Reads the agent's + * final answer (`/logs/agent/answer.txt`) and the last `MAX_IMAGES` screenshots + * it spilled (`/logs/agent/shots/shot-<n>.png`), sends them to a vision model + * with WebVoyager's verbatim SYSTEM_PROMPT via pi-ai, and writes a single 0/1 + * reward to `/logs/verifier/reward.txt` (`NOT SUCCESS` -> 0, `SUCCESS` -> 1, + * ambiguous -> fail-closed 0). + * + * A judge call that errors fails closed to 0 (recorded in grading_details), so a + * transient API hiccup never crashes the verifier into a missing-reward trial. + */ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { argv } from "node:process"; +import { pathToFileURL } from "node:url"; +import { lastShots, loadAnswer, loadGroundTruth } from "./artifacts.ts"; +import { judgeModel } from "./model.ts"; +import type { GradingDetails, JudgeModel } from "./types.ts"; +import { gradeWithWebJudge } from "./webjudge.ts"; + +export interface Args { + groundTruth: string; + answer: string; + shots: string; + judgeModel: string; + maxImages: number; + rewardOut: string; + detailsOut?: string; +} + +function parseArgs(argv: string[]): Args { + const flags = new Map<string, string>(); + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg.startsWith("--")) { + flags.set(arg.slice(2), argv[i + 1] ?? ""); + i += 1; + } + } + const required = (name: string): string => { + const value = flags.get(name); + if (!value) throw new Error(`missing required --${name}`); + return value; + }; + return { + groundTruth: flags.get("ground-truth") ?? "/tests/ground_truth.json", + answer: flags.get("answer") ?? "/logs/agent/answer.txt", + shots: flags.get("shots") ?? "/logs/agent/shots", + judgeModel: flags.get("judge-model") ?? "claude-sonnet-4-5", + maxImages: Number(flags.get("max-images") ?? "15"), + rewardOut: required("reward-out"), + detailsOut: flags.get("details-out"), + }; +} + +function writeReward(path: string, reward: 0 | 1): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, String(reward)); +} + +/** + * Read the artifacts, grade through `makeJudge()`, and write reward.txt (+ + * optional grading_details.json). Both model resolution and the judge call run + * inside the try, so a missing key or a transient API error fails closed to + * reward 0 with the error recorded in the details rather than crashing the + * verifier. Takes a factory so the file contract is testable without a live + * provider call. + */ +export async function run(args: Args, makeJudge: () => JudgeModel): Promise<void> { + const task = loadGroundTruth(args.groundTruth).task; + const answer = loadAnswer(args.answer); + const shots = lastShots(args.shots, args.maxImages); + + // No answer and no screenshots: nothing to judge, fail closed without details. + if (!answer && shots.length === 0) { + writeReward(args.rewardOut, 0); + return; + } + + let verdict = ""; + let reward: 0 | 1 = 0; + let error: string | null = null; + try { + ({ verdict, reward } = await gradeWithWebJudge({ task, answer, shots, judge: makeJudge() })); + } catch (err) { + error = err instanceof Error ? `${err.name}: ${err.message}` : String(err); + } + + writeReward(args.rewardOut, reward); + if (args.detailsOut) { + const details: GradingDetails = { + verdict_raw: verdict, + reward, + n_images: shots.length, + answer, + model: args.judgeModel, + error, + }; + mkdirSync(dirname(args.detailsOut), { recursive: true }); + writeFileSync(args.detailsOut, JSON.stringify(details, null, 2)); + } +} + +async function main(): Promise<void> { + const args = parseArgs(argv.slice(2)); + await run(args, () => judgeModel(args.judgeModel)); +} + +// Run only when invoked as the bin (`node judge.js`), not when imported by tests. +if (argv[1] && import.meta.url === pathToFileURL(argv[1]).href) { + main().catch((err) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); + process.exit(1); + }); +} diff --git a/benchmarks/adapters/webvoyager/judge/src/model.ts b/benchmarks/adapters/webvoyager/judge/src/model.ts new file mode 100644 index 0000000..72bf35b --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/src/model.ts @@ -0,0 +1,49 @@ +import { + type Api, + completeSimple, + getEnvApiKey, + getModel, + type KnownProvider, + type Model, +} from "@earendil-works/pi-ai"; +import type { JudgeContent, JudgeModel } from "./types.ts"; + +/** + * A {@link JudgeModel} backed by pi-ai's `completeSimple`. pi-ai handles provider + * routing, env-var API keys, o-series quirks (omitting `temperature`, using + * `max_completion_tokens`), vision, and retries — so the judge needs no + * hand-rolled provider client. Bundled into the verifier (no `npm install` at + * grade time); the Kernel VM ships `node` + global `fetch`. + * + * `JUDGE_MODEL` is a `provider:name` ref. The provider defaults to `anthropic` + * when no prefix is given, matching the upstream WebVoyager judge (and this + * adapter's `claude-sonnet-4-5` default), so a bare model name stays valid while + * `openai:o4-mini` etc. remains configurable. + */ + +// Upstream auto_eval.py caps the judge at max_tokens=1000. +const MAX_TOKENS = 1000; + +export function parseModelRef(ref: string): { provider: KnownProvider; name: string } { + const idx = ref.indexOf(":"); + if (idx === -1) return { provider: "anthropic", name: ref }; + return { provider: ref.slice(0, idx) as KnownProvider, name: ref.slice(idx + 1) }; +} + +export function judgeModel(ref: string): JudgeModel { + const { provider, name } = parseModelRef(ref); + // getModel is typed for literal provider/model ids; JUDGE_MODEL is a runtime + // string, so widen the way pi-ai's own consumers do. + const model = getModel(provider as never, name as never) as Model<Api>; + const apiKey = getEnvApiKey(provider); + return { + async complete(systemPrompt, content) { + const res = await completeSimple( + model, + { systemPrompt, messages: [{ role: "user", content, timestamp: Date.now() }] }, + { apiKey, temperature: 0, maxTokens: MAX_TOKENS }, + ); + return res.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join(""); + }, + }; +} diff --git a/benchmarks/adapters/webvoyager/judge/src/prompts.ts b/benchmarks/adapters/webvoyager/judge/src/prompts.ts new file mode 100644 index 0000000..903b744 --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/src/prompts.ts @@ -0,0 +1,50 @@ +/** + * WebVoyager judge prompts and verdict parse, ported verbatim from + * MinorJerry/WebVoyager `evaluation/auto_eval.py` (the single-call GPT-4V judge). + * + * SYSTEM_PROMPT is byte-for-byte from upstream; the `SUCCESS` / `NOT SUCCESS` + * markers the verdict parse keys on are reproduced exactly. This is the + * parity-load-bearing part — do not reword. + */ + +// Verbatim from MinorJerry/WebVoyager evaluation/auto_eval.py SYSTEM_PROMPT. +export const SYSTEM_PROMPT = `As an evaluator, you will be presented with three primary components to assist you in your role: + +1. Web Task Instruction: This is a clear and specific directive provided in natural language, detailing the online activity to be carried out. These requirements may include conducting searches, verifying information, comparing prices, checking availability, or any other action relevant to the specified web service (such as Amazon, Apple, ArXiv, BBC News, Booking etc). + +2. Result Screenshots: This is a visual representation of the screen showing the result or intermediate state of performing a web task. It serves as visual proof of the actions taken in response to the instruction. + +3. Result Response: This is a textual response obtained after the execution of the web task. It serves as textual result in response to the instruction. + +-- You DO NOT NEED to interact with web pages or perform actions such as booking flights or conducting searches on websites. +-- You SHOULD NOT make assumptions based on information not presented in the screenshot when comparing it to the instructions. +-- Your primary responsibility is to conduct a thorough assessment of the web task instruction against the outcome depicted in the screenshot and in the response, evaluating whether the actions taken align with the given instructions. +-- NOTE that the instruction may involve more than one task, for example, locating the garage and summarizing the review. Failing to complete either task, such as not providing a summary, should be considered unsuccessful. +-- NOTE that the screenshot is authentic, but the response provided by LLM is generated at the end of web browsing, and there may be discrepancies between the text and the screenshots. +-- Note the difference: 1) Result response may contradict the screenshot, then the content of the screenshot prevails, 2) The content in the Result response is not mentioned on the screenshot, choose to believe the content. + +You should elaborate on how you arrived at your final evaluation and then provide a definitive verdict on whether the task has been successfully accomplished, either as 'SUCCESS' or 'NOT SUCCESS'.`; + +export const USER_TMPL = "TASK: {task}\nResult Response: {answer}\n{n} screenshot(s) at the end:"; + +export const VERDICT_TEXT = "Your verdict:\n"; + +export function userText(task: string, answer: string, n: number): string { + // Use replacer functions: a string replacement would interpret `$&`, `$'`, + // etc. in the task/answer (WebVoyager instructions are free-form and contain + // `$`), corrupting the prompt. The function form returns the value literally. + return USER_TMPL.replace("{task}", () => task) + .replace("{answer}", () => answer) + .replace("{n}", () => String(n)); +} + +/** + * Upstream `auto_eval.py`: `0 if 'NOT SUCCESS' in res else (1 if 'SUCCESS' in res + * else None)`, with `None` folded to 0 (fail-closed) for Harbor's single-float + * reward. `NOT SUCCESS` wins over `SUCCESS` (checked first). + */ +export function parseReward(verdict: string): 0 | 1 { + if (verdict.includes("NOT SUCCESS")) return 0; + if (verdict.includes("SUCCESS")) return 1; + return 0; +} diff --git a/benchmarks/adapters/webvoyager/judge/src/types.ts b/benchmarks/adapters/webvoyager/judge/src/types.ts new file mode 100644 index 0000000..6382d4f --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/src/types.ts @@ -0,0 +1,25 @@ +/** Multimodal content passed to a {@link JudgeModel}: text and image blocks. */ +export type JudgeContent = Array< + | { type: "text"; text: string } + | { type: "image"; data: string; mimeType: string } +>; + +/** A model used by the judge. Implemented by a pi-ai call; mocked in tests. */ +export interface JudgeModel { + complete(systemPrompt: string, content: JudgeContent): Promise<string>; +} + +/** Grader input read from tests/ground_truth.json. */ +export interface GroundTruth { + task: string; +} + +/** What the judge writes to grading_details.json alongside the reward. */ +export interface GradingDetails { + verdict_raw: string; + reward: 0 | 1; + n_images: number; + answer: string; + model: string; + error: string | null; +} diff --git a/benchmarks/adapters/webvoyager/judge/src/webjudge.ts b/benchmarks/adapters/webvoyager/judge/src/webjudge.ts new file mode 100644 index 0000000..480cf9b --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/src/webjudge.ts @@ -0,0 +1,28 @@ +import * as P from "./prompts.ts"; +import type { Shot } from "./artifacts.ts"; +import type { JudgeContent, JudgeModel } from "./types.ts"; + +/** + * WebVoyager's single-call multimodal judge: one message with the task + answer + * text, the last-k screenshots, and a trailing "Your verdict:" block, scored + * against the verbatim SYSTEM_PROMPT. Block order (text -> images -> verdict) + * matches upstream `auto_eval.py`. + */ +export async function gradeWithWebJudge(args: { + task: string; + answer: string; + shots: Shot[]; + judge: JudgeModel; +}): Promise<{ verdict: string; reward: 0 | 1 }> { + const content: JudgeContent = [ + { type: "text", text: P.userText(args.task, args.answer, args.shots.length) }, + ...args.shots.map((shot) => ({ + type: "image" as const, + data: shot.base64, + mimeType: shot.mimeType, + })), + { type: "text", text: P.VERDICT_TEXT }, + ]; + const verdict = await args.judge.complete(P.SYSTEM_PROMPT, content); + return { verdict, reward: P.parseReward(verdict) }; +} diff --git a/benchmarks/adapters/webvoyager/judge/test/artifacts.test.ts b/benchmarks/adapters/webvoyager/judge/test/artifacts.test.ts new file mode 100644 index 0000000..a1dafba --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/test/artifacts.test.ts @@ -0,0 +1,63 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { lastShots, loadAnswer, loadGroundTruth } from "../src/artifacts.ts"; + +/** Lay out a shots dir with `shot-<n>.png` files (1x1 PNG bytes). */ +function writeShots(indices: number[]): string { + const dir = mkdtempSync(join(tmpdir(), "wv-shots-")); + mkdirSync(dir, { recursive: true }); + for (const n of indices) { + writeFileSync(join(dir, `shot-${n}.png`), Buffer.from([0x89, 0x50, 0x4e, 0x47, n % 256])); + } + return dir; +} + +describe("lastShots", () => { + it("orders shots by numeric index, not lexicographically", () => { + const dir = writeShots([1, 2, 10]); + expect(lastShots(dir, 10).map((s) => s.name)).toEqual([ + "shot-1.png", + "shot-2.png", + "shot-10.png", + ]); + }); + + it("takes the final k screenshots", () => { + const dir = writeShots([1, 2, 3, 4, 5]); + expect(lastShots(dir, 3).map((s) => s.name)).toEqual([ + "shot-3.png", + "shot-4.png", + "shot-5.png", + ]); + }); + + it("reads bytes into base64 with the png media type", () => { + const dir = writeShots([1]); + const [shot] = lastShots(dir, 1); + expect(shot.mimeType).toBe("image/png"); + expect(shot.base64).toBe(Buffer.from([0x89, 0x50, 0x4e, 0x47, 1]).toString("base64")); + }); + + it("returns nothing when the shots dir is absent", () => { + expect(lastShots(join(tmpdir(), "wv-missing-shots-xyz"), 15)).toEqual([]); + }); +}); + +describe("loadGroundTruth / loadAnswer", () => { + it("reads the task from ground_truth.json", () => { + const dir = mkdtempSync(join(tmpdir(), "wv-gt-")); + const path = join(dir, "ground_truth.json"); + writeFileSync(path, JSON.stringify({ task: "find a recipe", web_name: "Allrecipes" })); + expect(loadGroundTruth(path).task).toBe("find a recipe"); + }); + + it("trims the answer file and returns empty string when absent", () => { + const dir = mkdtempSync(join(tmpdir(), "wv-ans-")); + const path = join(dir, "answer.txt"); + writeFileSync(path, " the answer\n"); + expect(loadAnswer(path)).toBe("the answer"); + expect(loadAnswer(join(dir, "nope.txt"))).toBe(""); + }); +}); diff --git a/benchmarks/adapters/webvoyager/judge/test/judge.test.ts b/benchmarks/adapters/webvoyager/judge/test/judge.test.ts new file mode 100644 index 0000000..3ce829b --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/test/judge.test.ts @@ -0,0 +1,110 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { Args } from "../src/judge.ts"; +import { run } from "../src/judge.ts"; +import type { GradingDetails, JudgeContent, JudgeModel } from "../src/types.ts"; + +/** A /logs/agent + /tests layout, plus the verifier output paths run() writes. */ +function setup(opts: { answer: string | null; shotIndices: number[] }) { + const root = mkdtempSync(join(tmpdir(), "wv-judge-")); + const shotsDir = join(root, "agent", "shots"); + mkdirSync(shotsDir, { recursive: true }); + if (opts.answer !== null) { + writeFileSync(join(root, "agent", "answer.txt"), opts.answer); + } + for (const n of opts.shotIndices) { + writeFileSync(join(shotsDir, `shot-${n}.png`), Buffer.from([0x89, 0x50, 0x4e, 0x47, n % 256])); + } + writeFileSync(join(root, "ground_truth.json"), JSON.stringify({ task: "do a thing" })); + const args: Args = { + groundTruth: join(root, "ground_truth.json"), + answer: join(root, "agent", "answer.txt"), + shots: shotsDir, + judgeModel: "claude-sonnet-4-5", + maxImages: 15, + rewardOut: join(root, "verifier", "reward.txt"), + detailsOut: join(root, "verifier", "grading_details.json"), + }; + return { args }; +} + +/** A judge that records what it was asked and returns a canned verdict. */ +function captureJudge(verdict: string): { make: () => JudgeModel; calls: JudgeContent[] } { + const calls: JudgeContent[] = []; + const make = (): JudgeModel => ({ + async complete(_systemPrompt, content) { + calls.push(content); + return verdict; + }, + }); + return { make, calls }; +} + +function readDetails(args: Args): GradingDetails { + return JSON.parse(readFileSync(args.detailsOut!, "utf8")) as GradingDetails; +} + +describe("run", () => { + it.each([ + ["The agent did it. SUCCESS", "1"], + ["It failed. NOT SUCCESS", "0"], + ["ambiguous waffle", "0"], + ["clearly NOT SUCCESS even though SUCCESS appears", "0"], + ])("verdict %s writes reward %s and the raw verdict", async (verdict, expected) => { + const { args } = setup({ answer: "ans", shotIndices: [1] }); + await run(args, captureJudge(verdict).make); + expect(readFileSync(args.rewardOut, "utf8")).toBe(expected); + const details = readDetails(args); + expect(details.verdict_raw).toBe(verdict); + expect(details.n_images).toBe(1); + }); + + it("fails closed to 0 with no details when there is no answer and no shots", async () => { + const { args } = setup({ answer: "", shotIndices: [] }); + let called = false; + await run(args, () => { + called = true; + return { async complete() { return ""; } }; + }); + expect(called).toBe(false); + expect(readFileSync(args.rewardOut, "utf8")).toBe("0"); + expect(existsSync(args.detailsOut!)).toBe(false); + }); + + it("attaches the last-k images and the task + answer text", async () => { + const { args } = setup({ answer: "my answer", shotIndices: [1, 2] }); + args.maxImages = 2; + const { make, calls } = captureJudge("SUCCESS"); + await run(args, make); + const images = calls[0].filter((c) => c.type === "image"); + expect(images).toHaveLength(2); + expect(images[0]).toMatchObject({ mimeType: "image/png" }); + const first = calls[0][0]; + expect(first.type === "text" && first.text).toContain("do a thing"); + expect(first.type === "text" && first.text).toContain("my answer"); + }); + + it("fails closed to reward 0 and records the error when the judge throws", async () => { + const { args } = setup({ answer: "ans", shotIndices: [1] }); + await run(args, () => ({ + async complete() { + throw new Error("API error 529: overloaded"); + }, + })); + expect(readFileSync(args.rewardOut, "utf8")).toBe("0"); + const details = readDetails(args); + expect(details.reward).toBe(0); + expect(details.error).toContain("529"); + }); + + it("fails closed when model resolution throws (missing key / bad provider)", async () => { + const { args } = setup({ answer: "ans", shotIndices: [1] }); + await run(args, () => { + throw new Error("ANTHROPIC_API_KEY is required"); + }); + expect(readFileSync(args.rewardOut, "utf8")).toBe("0"); + expect(readDetails(args).error).toContain("ANTHROPIC_API_KEY"); + }); +}); diff --git a/benchmarks/adapters/webvoyager/judge/test/prompts.test.ts b/benchmarks/adapters/webvoyager/judge/test/prompts.test.ts new file mode 100644 index 0000000..3368ad3 --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/test/prompts.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { parseReward, SYSTEM_PROMPT, userText } from "../src/prompts.ts"; + +describe("WebVoyager prompts", () => { + it("SYSTEM_PROMPT is the verbatim WebVoyager evaluator prompt", () => { + expect(SYSTEM_PROMPT.startsWith("As an evaluator")).toBe(true); + expect(SYSTEM_PROMPT).toContain("'SUCCESS' or 'NOT SUCCESS'"); + }); + + it("userText fills task, answer, and screenshot count", () => { + expect(userText("do a thing", "my answer", 2)).toBe( + "TASK: do a thing\nResult Response: my answer\n2 screenshot(s) at the end:", + ); + }); + + it("userText keeps $-sequences literal (prices, regex specials)", () => { + expect(userText("hotels $1500-$2500", "cheapest $2,499", 3)).toBe( + "TASK: hotels $1500-$2500\nResult Response: cheapest $2,499\n3 screenshot(s) at the end:", + ); + // `$&`/`$'` are special in string-replacement values; they must survive verbatim. + expect(userText("price $& and $' end", "x", 1)).toBe( + "TASK: price $& and $' end\nResult Response: x\n1 screenshot(s) at the end:", + ); + }); +}); + +describe("parseReward", () => { + // Mirrors the Python webjudge verdict cases verbatim. + it.each([ + ["The agent did it. SUCCESS", 1], + ["It failed. NOT SUCCESS", 0], + ["ambiguous waffle", 0], // neither marker -> fail-closed + ["clearly NOT SUCCESS even though SUCCESS appears", 0], // NOT SUCCESS wins + ])("%s -> %d", (verdict, expected) => { + expect(parseReward(verdict)).toBe(expected); + }); +}); diff --git a/benchmarks/adapters/webvoyager/judge/test/webjudge.test.ts b/benchmarks/adapters/webvoyager/judge/test/webjudge.test.ts new file mode 100644 index 0000000..167587b --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/test/webjudge.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import type { Shot } from "../src/artifacts.ts"; +import { SYSTEM_PROMPT, VERDICT_TEXT } from "../src/prompts.ts"; +import type { JudgeContent, JudgeModel } from "../src/types.ts"; +import { gradeWithWebJudge } from "../src/webjudge.ts"; + +interface Call { + systemPrompt: string; + content: JudgeContent; +} + +function scriptedJudge(verdict: string): { judge: JudgeModel; calls: Call[] } { + const calls: Call[] = []; + const judge: JudgeModel = { + async complete(systemPrompt, content) { + calls.push({ systemPrompt, content }); + return verdict; + }, + }; + return { judge, calls }; +} + +const shots: Shot[] = [ + { name: "shot-1.png", base64: "AAAA", mimeType: "image/png" }, + { name: "shot-2.png", base64: "BBBB", mimeType: "image/png" }, +]; + +describe("gradeWithWebJudge", () => { + it("sends the system prompt, task+answer text, images, then the verdict block", async () => { + const { judge, calls } = scriptedJudge("Reasoning. SUCCESS"); + const { verdict, reward } = await gradeWithWebJudge({ + task: "do a thing", + answer: "my answer", + shots, + judge, + }); + + expect(reward).toBe(1); + expect(verdict).toBe("Reasoning. SUCCESS"); + expect(calls).toHaveLength(1); + expect(calls[0].systemPrompt).toBe(SYSTEM_PROMPT); + + const content = calls[0].content; + const images = content.filter((c) => c.type === "image"); + expect(images).toHaveLength(2); + expect(images[0]).toMatchObject({ data: "AAAA", mimeType: "image/png" }); + + const first = content[0]; + expect(first.type === "text" && first.text).toContain("do a thing"); + expect(first.type === "text" && first.text).toContain("my answer"); + expect(first.type === "text" && first.text).toContain("2 screenshot(s) at the end:"); + + const last = content.at(-1)!; + expect(last).toEqual({ type: "text", text: VERDICT_TEXT }); + }); + + it("fails closed to 0 on a NOT SUCCESS verdict", async () => { + const { judge } = scriptedJudge("It did not work. NOT SUCCESS"); + const { reward } = await gradeWithWebJudge({ task: "t", answer: "a", shots, judge }); + expect(reward).toBe(0); + }); +}); diff --git a/benchmarks/adapters/webvoyager/judge/tsconfig.json b/benchmarks/adapters/webvoyager/judge/tsconfig.json new file mode 100644 index 0000000..2ea517d --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "allowImportingTsExtensions": true, + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "noEmit": true + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/benchmarks/adapters/webvoyager/judge/tsdown.config.ts b/benchmarks/adapters/webvoyager/judge/tsdown.config.ts new file mode 100644 index 0000000..c7307a7 --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/tsdown.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from "tsdown"; + +// Bundle to a single self-contained ESM file so `node dist/judge.js` runs inside +// the Kernel verifier VM with no dependency install. pi-ai is bundled in +// (noExternal); it lazy-loads providers via dynamic import(), so +// inlineDynamicImports folds those chunks back into the one judge.js the +// adapter copies next to test.sh. +export default defineConfig({ + entry: ["src/judge.ts"], + format: ["esm"], + platform: "node", + dts: false, + sourcemap: false, + clean: true, + noExternal: [/.*/], + outputOptions: { inlineDynamicImports: true }, + outExtensions: () => ({ js: ".js" }), +}); diff --git a/benchmarks/adapters/webvoyager/judge/vitest.config.ts b/benchmarks/adapters/webvoyager/judge/vitest.config.ts new file mode 100644 index 0000000..6ec3464 --- /dev/null +++ b/benchmarks/adapters/webvoyager/judge/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + server: { + host: "127.0.0.1", + }, + test: { + globals: true, + environment: "node", + testTimeout: 30000, + }, +}); diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py b/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py index 548104f..b812629 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py +++ b/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py @@ -2,9 +2,9 @@ Each upstream record (``web_name``/``id``/``ques``/``web``) becomes one task dir: ``instruction.md`` (the ``ques`` plus an answer directive), ``environment/kernel.json`` -(``start_url`` = ``web`` + stealth + viewport), ``tests/`` (the ported WebVoyager -multimodal judge + a per-task ``ground_truth.json``), ``solution/solve.sh`` (writes the -reference answer for an oracle plumbing run), and ``task.toml``. +(``start_url`` = ``web`` + stealth + viewport), ``tests/`` (the bundled WebVoyager +multimodal judge ``judge.js`` + a per-task ``ground_truth.json``), ``solution/solve.sh`` +(writes the reference answer for an oracle plumbing run), and ``task.toml``. The dataset is vendored under ``data/`` so generation is hermetic and pinned to the upstream commit recorded in ``adapter_metadata.json``; ``--refresh`` re-fetches it. @@ -25,6 +25,11 @@ DATASET_FILE = DATA_DIR / "WebVoyager_data.jsonl" REFERENCE_FILE = DATA_DIR / "reference_answer.json" +# Bundled, self-contained WebJudge (pi-ai inlined); built by the judge package and +# copied into each task's tests/ so the verifier runs `node judge.js` with no install. +ADAPTER_ROOT = PACKAGE_DIR.parents[1] +JUDGE_BUNDLE = ADAPTER_ROOT / "judge" / "dist" / "judge.js" + RAW_BASE = "https://raw.githubusercontent.com/MinorJerry/WebVoyager/main/data" DATASET_URL = f"{RAW_BASE}/WebVoyager_data.jsonl" REFERENCE_URL = f"{RAW_BASE}/reference_answer.json" @@ -152,7 +157,7 @@ def _prepare_task(self, task: WebVoyagerTask, task_dir: Path) -> None: tests_dir = task_dir / "tests" tests_dir.mkdir(exist_ok=True) shutil.copy2(TEMPLATE_DIR / "tests/test.sh", tests_dir / "test.sh") - shutil.copy2(TEMPLATE_DIR / "tests/webjudge.py", tests_dir / "webjudge.py") + shutil.copy2(JUDGE_BUNDLE, tests_dir / "judge.js") (tests_dir / "ground_truth.json").write_text( json.dumps( { @@ -184,6 +189,11 @@ def _prepare_task(self, task: WebVoyagerTask, task_dir: Path) -> None: logger.debug(f"Wrote task {local_id}") def run(self) -> None: + if not JUDGE_BUNDLE.exists(): + raise FileNotFoundError( + f"WebJudge bundle not built at {JUDGE_BUNDLE}; run " + "`cd benchmarks/adapters/webvoyager/judge && npm install && npm run build`" + ) selected = self._select() self.output_dir.mkdir(parents=True, exist_ok=True) logger.info(f"Generating {len(selected)} WebVoyager tasks -> {self.output_dir}") diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh index 86b0ccd..63e1067 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/test.sh @@ -1,6 +1,22 @@ #!/usr/bin/env bash -set -euo pipefail +set -uo pipefail -# The Kernel verifier VM has Python 3 but no pip; webjudge.py talks to the -# Anthropic API with the standard library, so no install step is needed. -python3 /tests/webjudge.py +# WebVoyager single-call multimodal judge. Runs the self-contained judge bundle +# (uploaded alongside this script to /tests) inside the Kernel VM, grading the +# agent's answer + last-MAX_IMAGES screenshots under /logs/agent. The bundle +# writes the reward itself and falls back to 0 on any error; this wrapper writes +# 0 too if node never runs so the reward file is never left empty. +mkdir -p /logs/verifier + +node /tests/judge.js \ + --ground-truth /tests/ground_truth.json \ + --answer /logs/agent/answer.txt \ + --shots /logs/agent/shots \ + --judge-model "${JUDGE_MODEL:-claude-sonnet-4-5}" \ + --max-images "${MAX_IMAGES:-15}" \ + --reward-out /logs/verifier/reward.txt \ + --details-out /logs/verifier/grading_details.json + +if [ ! -s /logs/verifier/reward.txt ]; then + echo 0 > /logs/verifier/reward.txt +fi diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py deleted file mode 100644 index 00d5284..0000000 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/tests/webjudge.py +++ /dev/null @@ -1,179 +0,0 @@ -"""WebVoyager's single-call multimodal judge, ported to the Anthropic Messages API. - -Reads the agent's final answer (``/logs/agent/answer.txt``) and the last ``MAX_IMAGES`` -screenshots the cua agent spilled (``/logs/agent/shots/shot-<n>.png``), sends them to a -vision model with WebVoyager's verbatim SYSTEM_PROMPT, and writes a single 0/1 reward to -``/logs/verifier/reward.txt`` (verdict ``NOT SUCCESS`` -> 0, ``SUCCESS`` -> 1, ambiguous -> -fail-closed 0). Upstream ``evaluation/auto_eval.py`` used the OpenAI GPT-4V payload; the -prompt, last-k screenshots, and verdict parse are unchanged. - -The request is made with the standard library (``urllib.request``) rather than the -``anthropic`` SDK: the Kernel verifier VM has Python 3.10 but no ``pip``/``ensurepip``, so -nothing can be installed at grade time. A bare HTTPS POST to the Messages API needs no -dependencies. -""" - -import base64 -import json -import os -import re -import urllib.error -import urllib.request -from pathlib import Path - -AGENT_DIR = Path("/logs/agent") -ANSWER_FILE = AGENT_DIR / "answer.txt" -SHOTS_DIR = AGENT_DIR / "shots" -GROUND_TRUTH = Path("/tests/ground_truth.json") -VERIFIER_DIR = Path("/logs/verifier") - -ANTHROPIC_URL = "https://api.anthropic.com/v1/messages" -ANTHROPIC_VERSION = "2023-06-01" - -# Verbatim from MinorJerry/WebVoyager evaluation/auto_eval.py SYSTEM_PROMPT. -SYSTEM_PROMPT = """As an evaluator, you will be presented with three primary components to assist you in your role: - -1. Web Task Instruction: This is a clear and specific directive provided in natural language, detailing the online activity to be carried out. These requirements may include conducting searches, verifying information, comparing prices, checking availability, or any other action relevant to the specified web service (such as Amazon, Apple, ArXiv, BBC News, Booking etc). - -2. Result Screenshots: This is a visual representation of the screen showing the result or intermediate state of performing a web task. It serves as visual proof of the actions taken in response to the instruction. - -3. Result Response: This is a textual response obtained after the execution of the web task. It serves as textual result in response to the instruction. - --- You DO NOT NEED to interact with web pages or perform actions such as booking flights or conducting searches on websites. --- You SHOULD NOT make assumptions based on information not presented in the screenshot when comparing it to the instructions. --- Your primary responsibility is to conduct a thorough assessment of the web task instruction against the outcome depicted in the screenshot and in the response, evaluating whether the actions taken align with the given instructions. --- NOTE that the instruction may involve more than one task, for example, locating the garage and summarizing the review. Failing to complete either task, such as not providing a summary, should be considered unsuccessful. --- NOTE that the screenshot is authentic, but the response provided by LLM is generated at the end of web browsing, and there may be discrepancies between the text and the screenshots. --- Note the difference: 1) Result response may contradict the screenshot, then the content of the screenshot prevails, 2) The content in the Result response is not mentioned on the screenshot, choose to believe the content. - -You should elaborate on how you arrived at your final evaluation and then provide a definitive verdict on whether the task has been successfully accomplished, either as 'SUCCESS' or 'NOT SUCCESS'.""" - -USER_TMPL = "TASK: {task}\nResult Response: {answer}\n{n} screenshot(s) at the end:" - -_MEDIA_BY_SUFFIX = { - ".png": "image/png", - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - ".webp": "image/webp", -} -_SHOT_INDEX = re.compile(r"(\d+)") - - -def _shot_key(path: Path) -> int: - """Numeric sort key. Shots are ``shot-<n>.png`` (not zero-padded).""" - match = _SHOT_INDEX.search(path.stem) - return int(match.group(1)) if match else 0 - - -def _last_shots(k: int) -> list[Path]: - if not SHOTS_DIR.is_dir(): - return [] - shots = sorted( - (p for p in SHOTS_DIR.iterdir() if p.suffix.lower() in _MEDIA_BY_SUFFIX), - key=_shot_key, - ) - return shots[-k:] - - -def _post(payload: dict, api_key: str) -> str: - """POST one Messages payload and return the concatenated text content.""" - request = urllib.request.Request( - ANTHROPIC_URL, - data=json.dumps(payload).encode(), - headers={ - "x-api-key": api_key, - "anthropic-version": ANTHROPIC_VERSION, - "content-type": "application/json", - }, - method="POST", - ) - with urllib.request.urlopen(request) as response: # noqa: S310 (pinned API host) - body = json.loads(response.read().decode()) - return "".join(part.get("text", "") for part in body.get("content", [])) - - -def _call_anthropic(payload: dict, api_key: str) -> str: - """Call the judge, dropping ``temperature`` if the model rejects it. - - Newer Anthropic models (e.g. claude-opus-4-8) reject the ``temperature`` - parameter with a 400; on that specific error, retry once without it so the - same judge code works across model generations. - """ - try: - return _post(payload, api_key) - except urllib.error.HTTPError as exc: - detail = exc.read().decode(errors="replace") if exc.code == 400 else "" - if exc.code == 400 and "temperature" in detail and "temperature" in payload: - retry = {k: v for k, v in payload.items() if k != "temperature"} - return _post(retry, api_key) - raise - - -def main() -> None: - VERIFIER_DIR.mkdir(parents=True, exist_ok=True) - reward_path = VERIFIER_DIR / "reward.txt" - - ground_truth = json.loads(GROUND_TRUTH.read_text()) - task = ground_truth["task"] - answer = ANSWER_FILE.read_text().strip() if ANSWER_FILE.exists() else "" - k = int(os.getenv("MAX_IMAGES", "15")) - shots = _last_shots(k) - - if not answer and not shots: - reward_path.write_text("0") - return - - blocks: list[dict] = [ - {"type": "text", "text": USER_TMPL.format(task=task, answer=answer, n=len(shots))} - ] - for shot in shots: - blocks.append( - { - "type": "image", - "source": { - "type": "base64", - "media_type": _MEDIA_BY_SUFFIX[shot.suffix.lower()], - "data": base64.b64encode(shot.read_bytes()).decode(), - }, - } - ) - blocks.append({"type": "text", "text": "Your verdict:\n"}) - - model = os.getenv("JUDGE_MODEL", "claude-sonnet-4-5") - payload = { - "model": model, - "max_tokens": 1000, - "temperature": 0, - "system": SYSTEM_PROMPT, - "messages": [{"role": "user", "content": blocks}], - } - - error = None - try: - verdict = _call_anthropic(payload, os.getenv("ANTHROPIC_API_KEY", "")) - except urllib.error.HTTPError as exc: - verdict, error = "", f"HTTP {exc.code}: {exc.read().decode(errors='replace')[:500]}" - except OSError as exc: # connection reset / DNS / timeout - verdict, error = "", f"{type(exc).__name__}: {exc}" - - # A judge call that errors fails closed to 0 (recorded in grading_details), so a - # transient API hiccup never crashes the verifier into a missing-reward trial. - reward = 0 if "NOT SUCCESS" in verdict else (1 if "SUCCESS" in verdict else 0) - reward_path.write_text(str(reward)) - (VERIFIER_DIR / "grading_details.json").write_text( - json.dumps( - { - "verdict_raw": verdict, - "reward": reward, - "n_images": len(shots), - "answer": answer, - "model": model, - "error": error, - }, - indent=2, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/adapters/webvoyager/tests/test_adapter.py b/benchmarks/adapters/webvoyager/tests/test_adapter.py index 131bbca..efc03ed 100644 --- a/benchmarks/adapters/webvoyager/tests/test_adapter.py +++ b/benchmarks/adapters/webvoyager/tests/test_adapter.py @@ -67,7 +67,7 @@ def test_run_emits_expected_files(adapter: WebVoyagerAdapter) -> None: assert (task_dir / "instruction.md").is_file() assert (task_dir / "environment" / "kernel.json").is_file() assert (task_dir / "tests" / "test.sh").is_file() - assert (task_dir / "tests" / "webjudge.py").is_file() + assert (task_dir / "tests" / "judge.js").is_file() assert (task_dir / "tests" / "ground_truth.json").is_file() assert (task_dir / "solution" / "solve.sh").is_file() assert (task_dir / "task.toml").is_file() diff --git a/benchmarks/adapters/webvoyager/tests/test_webjudge.py b/benchmarks/adapters/webvoyager/tests/test_webjudge.py deleted file mode 100644 index c3a8686..0000000 --- a/benchmarks/adapters/webvoyager/tests/test_webjudge.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Mocked unit tests for the ported WebVoyager judge (no live Anthropic call).""" - -from __future__ import annotations - -import importlib.util -import io -import json -import urllib.error -from pathlib import Path - -import pytest - -TEMPLATE_TESTS = ( - Path(__file__).resolve().parents[1] / "src" / "webvoyager" / "task-template" / "tests" -) - - -def _load_webjudge(monkeypatch, agent_dir: Path, tests_dir: Path, captured: dict): - """Import webjudge.py with the HTTP call stubbed and I/O paths redirected. - - ``_call_anthropic`` is replaced so no network request is made; the request - payload it would have sent is recorded in ``captured`` for assertions, and the - canned verdict comes from ``captured['_verdict']``. - """ - spec = importlib.util.spec_from_file_location( - "wv_webjudge_under_test", TEMPLATE_TESTS / "webjudge.py" - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - def fake_call(payload: dict, api_key: str) -> str: - captured.update(payload) - captured["_api_key"] = api_key - return captured.get("_verdict", "Reasoning... SUCCESS") - - monkeypatch.setattr(module, "_call_anthropic", fake_call) - monkeypatch.setattr(module, "AGENT_DIR", agent_dir) - monkeypatch.setattr(module, "ANSWER_FILE", agent_dir / "answer.txt") - monkeypatch.setattr(module, "SHOTS_DIR", agent_dir / "shots") - monkeypatch.setattr(module, "GROUND_TRUTH", tests_dir / "ground_truth.json") - monkeypatch.setattr(module, "VERIFIER_DIR", tests_dir / "verifier") - return module - - -def _setup_dirs(tmp_path: Path, *, answer: str, shot_indices: list[int]): - agent_dir = tmp_path / "agent" - shots = agent_dir / "shots" - shots.mkdir(parents=True) - if answer is not None: - (agent_dir / "answer.txt").write_text(answer) - for n in shot_indices: - # 1x1 PNG bytes; content is irrelevant for the mocked judge. - (shots / f"shot-{n}.png").write_bytes(b"\x89PNG\r\n\x1a\n" + bytes([n % 256])) - tests_dir = tmp_path / "tests" - tests_dir.mkdir() - (tests_dir / "ground_truth.json").write_text(json.dumps({"task": "do a thing"})) - return agent_dir, tests_dir - - -def test_shot_key_numeric_order(monkeypatch, tmp_path: Path): - agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1, 2, 10]) - module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) - ordered = module._last_shots(10) - assert [p.name for p in ordered] == ["shot-1.png", "shot-2.png", "shot-10.png"] - - -def test_last_k_takes_final_screenshots(monkeypatch, tmp_path: Path): - agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1, 2, 3, 4, 5]) - module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) - assert [p.name for p in module._last_shots(3)] == [ - "shot-3.png", - "shot-4.png", - "shot-5.png", - ] - - -@pytest.mark.parametrize( - "verdict,expected", - [ - ("The agent did it. SUCCESS", "1"), - ("It failed. NOT SUCCESS", "0"), - ("ambiguous waffle", "0"), # neither marker -> fail-closed - ("clearly NOT SUCCESS even though SUCCESS appears", "0"), # NOT SUCCESS wins - ], -) -def test_verdict_parsing(monkeypatch, tmp_path: Path, verdict: str, expected: str): - captured = {"_verdict": verdict} - agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1]) - module = _load_webjudge(monkeypatch, agent_dir, tests_dir, captured) - monkeypatch.setenv("ANTHROPIC_API_KEY", "test") - module.main() - assert (tests_dir / "verifier" / "reward.txt").read_text() == expected - details = json.loads((tests_dir / "verifier" / "grading_details.json").read_text()) - assert details["verdict_raw"] == verdict - assert details["n_images"] == 1 - - -def test_empty_answer_and_no_shots_fail_closed(monkeypatch, tmp_path: Path): - agent_dir, tests_dir = _setup_dirs(tmp_path, answer="", shot_indices=[]) - module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) - module.main() - assert (tests_dir / "verifier" / "reward.txt").read_text() == "0" - assert not (tests_dir / "verifier" / "grading_details.json").exists() - - -def test_payload_attaches_images_and_system_prompt(monkeypatch, tmp_path: Path): - captured: dict = {"_verdict": "SUCCESS"} - agent_dir, tests_dir = _setup_dirs(tmp_path, answer="my answer", shot_indices=[1, 2]) - module = _load_webjudge(monkeypatch, agent_dir, tests_dir, captured) - monkeypatch.setenv("ANTHROPIC_API_KEY", "test") - monkeypatch.setenv("MAX_IMAGES", "2") - module.main() - blocks = captured["messages"][0]["content"] - images = [b for b in blocks if b["type"] == "image"] - assert len(images) == 2 - assert images[0]["source"]["media_type"] == "image/png" - assert "do a thing" in blocks[0]["text"] - assert "my answer" in blocks[0]["text"] - assert captured["system"].startswith("As an evaluator") - assert captured["temperature"] == 0 - - -def _http_error(code: int, body: str) -> urllib.error.HTTPError: - return urllib.error.HTTPError( - url=MODULE_URL, code=code, msg="err", hdrs=None, fp=io.BytesIO(body.encode()) - ) - - -MODULE_URL = "https://api.anthropic.com/v1/messages" - - -def _import_webjudge(): - spec = importlib.util.spec_from_file_location( - "wv_webjudge_retry", TEMPLATE_TESTS / "webjudge.py" - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def test_call_anthropic_drops_temperature_on_400(monkeypatch): - """A 400 citing temperature retries once without it (newer models reject it).""" - module = _import_webjudge() - calls: list[dict] = [] - - def fake_post(payload: dict, api_key: str) -> str: - calls.append(payload) - if "temperature" in payload: - raise _http_error(400, '{"error":{"message":"temperature is deprecated"}}') - return "SUCCESS" - - monkeypatch.setattr(module, "_post", fake_post) - out = module._call_anthropic({"model": "m", "temperature": 0}, "k") - assert out == "SUCCESS" - assert len(calls) == 2 - assert "temperature" not in calls[1] - - -def test_main_fails_closed_on_http_error(monkeypatch, tmp_path: Path): - """A judge HTTP error writes reward 0 + an error note, never crashes the trial.""" - agent_dir, tests_dir = _setup_dirs(tmp_path, answer="ans", shot_indices=[1]) - module = _load_webjudge(monkeypatch, agent_dir, tests_dir, {}) - - def boom(payload: dict, api_key: str) -> str: - raise _http_error(529, '{"error":{"message":"overloaded"}}') - - monkeypatch.setattr(module, "_call_anthropic", boom) - monkeypatch.setenv("ANTHROPIC_API_KEY", "test") - module.main() - assert (tests_dir / "verifier" / "reward.txt").read_text() == "0" - details = json.loads((tests_dir / "verifier" / "grading_details.json").read_text()) - assert details["reward"] == 0 - assert "529" in details["error"] From 0e859b2a02c8feb4b30ddecb4326ca48439251bf Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:08:21 +0000 Subject: [PATCH 08/11] webvoyager: send o-series reasoning effort, drop temperature in the pi-ai judge Same o-series handling as the online-mind2web judge: o4-mini/o3 reject `temperature` and a `none` reasoning effort. Gate `reasoning: medium` + no temperature to OpenAI reasoning models; the claude-sonnet-4-5 default keeps `temperature: 0`. Also drops the docstring's incorrect claim that pi-ai omits temperature for o-series. Verified live with both claude-sonnet-4-5 and openai:o4-mini. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- .../adapters/webvoyager/judge/src/model.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/benchmarks/adapters/webvoyager/judge/src/model.ts b/benchmarks/adapters/webvoyager/judge/src/model.ts index 72bf35b..31df23f 100644 --- a/benchmarks/adapters/webvoyager/judge/src/model.ts +++ b/benchmarks/adapters/webvoyager/judge/src/model.ts @@ -10,10 +10,10 @@ import type { JudgeContent, JudgeModel } from "./types.ts"; /** * A {@link JudgeModel} backed by pi-ai's `completeSimple`. pi-ai handles provider - * routing, env-var API keys, o-series quirks (omitting `temperature`, using - * `max_completion_tokens`), vision, and retries — so the judge needs no - * hand-rolled provider client. Bundled into the verifier (no `npm install` at - * grade time); the Kernel VM ships `node` + global `fetch`. + * routing, env-var API keys, vision, and retries — so the judge needs no + * hand-rolled provider client. OpenAI reasoning backbones need `temperature` + * and reasoning effort handled explicitly (see below). Bundled into the + * verifier (no `npm install` at grade time); the Kernel VM ships `node`. * * `JUDGE_MODEL` is a `provider:name` ref. The provider defaults to `anthropic` * when no prefix is given, matching the upstream WebVoyager judge (and this @@ -36,12 +36,21 @@ export function judgeModel(ref: string): JudgeModel { // string, so widen the way pi-ai's own consumers do. const model = getModel(provider as never, name as never) as Model<Api>; const apiKey = getEnvApiKey(provider); + // OpenAI reasoning backbones (o4-mini, o3, …) reject `temperature` and a + // reasoning effort of "none" (pi-ai's default when unset); they require + // low/medium/high. Other backbones — including this adapter's + // claude-sonnet-4-5 default — keep deterministic scoring (temperature 0). + const baseOptions = { apiKey, maxTokens: MAX_TOKENS }; + const options = + model.reasoning && provider === "openai" + ? { ...baseOptions, reasoning: "medium" as const } + : { ...baseOptions, temperature: 0 }; return { async complete(systemPrompt, content) { const res = await completeSimple( model, { systemPrompt, messages: [{ role: "user", content, timestamp: Date.now() }] }, - { apiKey, temperature: 0, maxTokens: MAX_TOKENS }, + options, ); return res.content.flatMap((c) => (c.type === "text" ? [c.text] : [])).join(""); }, From cad3863763d4f6be88c53695151bd21ef9f8667a Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 27 Jun 2026 22:41:42 +0000 Subject: [PATCH 09/11] webvoyager: validate --max-images and reject negative --limit A 0/negative/non-numeric --max-images made lastShots' slice(-k) attach every screenshot instead of the last k, risking judge token blowups and spurious 0 rewards; parse it to a positive integer, falling back to the 15 default otherwise. A negative --limit likewise made tasks[:limit] drop tasks off the end instead of taking the first N, so reject it. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- .../adapters/webvoyager/judge/src/judge.ts | 15 ++++++++++++++- .../webvoyager/judge/test/judge.test.ts | 19 ++++++++++++++++++- .../webvoyager/src/webvoyager/adapter.py | 4 ++++ .../adapters/webvoyager/tests/test_adapter.py | 6 ++++++ 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/benchmarks/adapters/webvoyager/judge/src/judge.ts b/benchmarks/adapters/webvoyager/judge/src/judge.ts index 76c4358..72b4aae 100644 --- a/benchmarks/adapters/webvoyager/judge/src/judge.ts +++ b/benchmarks/adapters/webvoyager/judge/src/judge.ts @@ -32,6 +32,19 @@ export interface Args { detailsOut?: string; } +const DEFAULT_MAX_IMAGES = 15; + +/** + * Parse `--max-images` into a positive integer last-k window. A 0, negative, or + * non-numeric value (e.g. a bad `WEBVOYAGER_MAX_IMAGES` env override) would make + * `lastShots`' `slice(-k)` attach *all* screenshots — blowing the judge's token + * budget — so anything that isn't a positive integer falls back to the default. + */ +export function parseMaxImages(raw: string | undefined): number { + const value = Number(raw ?? DEFAULT_MAX_IMAGES); + return Number.isInteger(value) && value > 0 ? value : DEFAULT_MAX_IMAGES; +} + function parseArgs(argv: string[]): Args { const flags = new Map<string, string>(); for (let i = 0; i < argv.length; i += 1) { @@ -51,7 +64,7 @@ function parseArgs(argv: string[]): Args { answer: flags.get("answer") ?? "/logs/agent/answer.txt", shots: flags.get("shots") ?? "/logs/agent/shots", judgeModel: flags.get("judge-model") ?? "claude-sonnet-4-5", - maxImages: Number(flags.get("max-images") ?? "15"), + maxImages: parseMaxImages(flags.get("max-images")), rewardOut: required("reward-out"), detailsOut: flags.get("details-out"), }; diff --git a/benchmarks/adapters/webvoyager/judge/test/judge.test.ts b/benchmarks/adapters/webvoyager/judge/test/judge.test.ts index 3ce829b..1a4fa47 100644 --- a/benchmarks/adapters/webvoyager/judge/test/judge.test.ts +++ b/benchmarks/adapters/webvoyager/judge/test/judge.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import type { Args } from "../src/judge.ts"; -import { run } from "../src/judge.ts"; +import { parseMaxImages, run } from "../src/judge.ts"; import type { GradingDetails, JudgeContent, JudgeModel } from "../src/types.ts"; /** A /logs/agent + /tests layout, plus the verifier output paths run() writes. */ @@ -108,3 +108,20 @@ describe("run", () => { expect(readDetails(args).error).toContain("ANTHROPIC_API_KEY"); }); }); + +describe("parseMaxImages", () => { + it("keeps a valid positive integer", () => { + expect(parseMaxImages("3")).toBe(3); + expect(parseMaxImages("15")).toBe(15); + }); + + it("defaults to 15 when unset", () => { + expect(parseMaxImages(undefined)).toBe(15); + }); + + // A 0/negative/non-numeric last-k makes slice(-k) attach ALL screenshots, so + // anything that isn't a positive integer must fall back to the default. + it.each(["0", "-5", "abc", "", "2.5"])("falls back to 15 for invalid %o", (raw) => { + expect(parseMaxImages(raw)).toBe(15); + }); +}); diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py b/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py index b812629..af0f5e9 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py +++ b/benchmarks/adapters/webvoyager/src/webvoyager/adapter.py @@ -99,6 +99,10 @@ def __init__( refresh: bool = False, **kwargs: object, ): + if limit is not None and limit < 0: + # tasks[:limit] with a negative limit drops tasks off the *end* instead + # of taking the first N, so reject it rather than silently mis-selecting. + raise ValueError(f"limit must be non-negative, got {limit}") self.output_dir = Path(output_dir) self.limit = limit self.overwrite = overwrite diff --git a/benchmarks/adapters/webvoyager/tests/test_adapter.py b/benchmarks/adapters/webvoyager/tests/test_adapter.py index efc03ed..6098ea4 100644 --- a/benchmarks/adapters/webvoyager/tests/test_adapter.py +++ b/benchmarks/adapters/webvoyager/tests/test_adapter.py @@ -132,6 +132,12 @@ def test_limit_and_task_ids_select(tmp_path: Path) -> None: assert ids == {"Amazon--3", "Apple--1"} +def test_negative_limit_rejected(tmp_path: Path) -> None: + # tasks[:limit] with a negative limit would drop tasks off the end, so it must error. + with pytest.raises(ValueError, match="non-negative"): + WebVoyagerAdapter(output_dir=tmp_path / "out", limit=-1) + + def test_overwrite_false_skips_existing(adapter: WebVoyagerAdapter) -> None: adapter.run() target = adapter.output_dir / "webvoyager-allrecipes--0" / "instruction.md" From b8d8bba976fd6fea94a42dac755be526b1ff85a3 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 28 Jun 2026 13:12:18 +0000 Subject: [PATCH 10/11] webvoyager: fix default --output-dir to adapters/webvoyager/.tasks The default output dir was computed with parents[3], which resolves to benchmarks/adapters/.tasks instead of the documented adapters/webvoyager/.tasks. Use parents[2] so the default lands in the adapter package root, matching the README and --help text. Add a test covering the resolved default. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- benchmarks/adapters/webvoyager/src/webvoyager/main.py | 4 +++- benchmarks/adapters/webvoyager/tests/test_adapter.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/main.py b/benchmarks/adapters/webvoyager/src/webvoyager/main.py index 0df9d22..31e6e1a 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/main.py +++ b/benchmarks/adapters/webvoyager/src/webvoyager/main.py @@ -18,7 +18,9 @@ def _default_output_dir() -> Path: - return Path(__file__).resolve().parents[3] / ".tasks" + # main.py is at adapters/webvoyager/src/webvoyager/main.py; parents[2] is the + # adapters/webvoyager package root, so the default lands in adapters/webvoyager/.tasks. + return Path(__file__).resolve().parents[2] / ".tasks" def _parse_args() -> argparse.Namespace: diff --git a/benchmarks/adapters/webvoyager/tests/test_adapter.py b/benchmarks/adapters/webvoyager/tests/test_adapter.py index 6098ea4..98df38c 100644 --- a/benchmarks/adapters/webvoyager/tests/test_adapter.py +++ b/benchmarks/adapters/webvoyager/tests/test_adapter.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(SRC)) from webvoyager.adapter import WebVoyagerAdapter, _index_reference, _toml_escape # noqa: E402 +from webvoyager.main import _default_output_dir # noqa: E402 @pytest.fixture @@ -146,6 +147,14 @@ def test_overwrite_false_skips_existing(adapter: WebVoyagerAdapter) -> None: assert target.read_text() == "SENTINEL" +def test_default_output_dir_is_adapter_dot_tasks() -> None: + # Default must land in adapters/webvoyager/.tasks, as the README + --help document. + default = _default_output_dir() + assert default.name == ".tasks" + assert default.parent.name == "webvoyager" + assert default.parent.parent.name == "adapters" + + def test_toml_escape_quotes_and_backslashes() -> None: assert _toml_escape('a"b\\c') == 'a\\"b\\\\c' assert "\n" not in _toml_escape("line1\nline2") From a15dafe5fe336d49fdeb3bb1f49c0aece65c46fe Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sun, 28 Jun 2026 13:28:08 +0000 Subject: [PATCH 11/11] webvoyager: fail closed on artifact-read errors and plumb OPENAI_API_KEY The judge bin read ground_truth.json / answer.txt / the screenshots before the try/catch, so a missing or corrupt artifact threw out of the bin and left reward.txt unwritten (only test.sh's empty-file fallback wrote 0). Move the artifact reads inside the try so any read/parse error fails closed to reward 0 with the error in grading_details, matching the online-mind2web judge. Also forward OPENAI_API_KEY in [verifier.env] so the documented JUDGE_MODEL=openai:o4-mini path has its key on the verifier VM. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --- benchmarks/adapters/webvoyager/README.md | 3 +- .../adapters/webvoyager/judge/src/judge.ts | 79 ++++++++++++------- .../webvoyager/judge/test/judge.test.ts | 24 ++++++ .../src/webvoyager/task-template/task.toml | 5 ++ .../adapters/webvoyager/tests/test_adapter.py | 2 + 5 files changed, 83 insertions(+), 30 deletions(-) diff --git a/benchmarks/adapters/webvoyager/README.md b/benchmarks/adapters/webvoyager/README.md index 6e7dc9b..ed7b0c2 100644 --- a/benchmarks/adapters/webvoyager/README.md +++ b/benchmarks/adapters/webvoyager/README.md @@ -74,6 +74,7 @@ to `/logs/verifier/reward.txt` plus a `grading_details.json`. | var | default | meaning | |---|---|---| -| `ANTHROPIC_API_KEY` | `${ANTHROPIC_API_KEY}` | resolved from the host at exec time | +| `ANTHROPIC_API_KEY` | `${ANTHROPIC_API_KEY}` | resolved from the host at exec time; used by the default `claude-sonnet-4-5` judge | +| `OPENAI_API_KEY` | `${OPENAI_API_KEY}` | resolved from the host at exec time; used when `JUDGE_MODEL` is an `openai:` ref (e.g. `openai:o4-mini`) | | `JUDGE_MODEL` | `claude-sonnet-4-5` | vision judge model; a pi-ai `provider:name` ref (bare name = `anthropic`), e.g. `openai:o4-mini` (`WEBVOYAGER_JUDGE_MODEL` to override) | | `MAX_IMAGES` | `15` | last-k screenshots sent to the judge (`WEBVOYAGER_MAX_IMAGES`; `15` matches the canonical `auto_eval.py` invocation) | diff --git a/benchmarks/adapters/webvoyager/judge/src/judge.ts b/benchmarks/adapters/webvoyager/judge/src/judge.ts index 72b4aae..e543125 100644 --- a/benchmarks/adapters/webvoyager/judge/src/judge.ts +++ b/benchmarks/adapters/webvoyager/judge/src/judge.ts @@ -10,8 +10,10 @@ * reward to `/logs/verifier/reward.txt` (`NOT SUCCESS` -> 0, `SUCCESS` -> 1, * ambiguous -> fail-closed 0). * - * A judge call that errors fails closed to 0 (recorded in grading_details), so a - * transient API hiccup never crashes the verifier into a missing-reward trial. + * Any failure the bin owns — a missing/corrupt artifact, model resolution, or + * the judge call — fails closed to 0 (recorded in grading_details), so a missing + * ground_truth.json or a transient API hiccup never crashes the verifier into a + * missing-reward trial. */ import { mkdirSync, writeFileSync } from "node:fs"; import { dirname } from "node:path"; @@ -75,47 +77,66 @@ function writeReward(path: string, reward: 0 | 1): void { writeFileSync(path, String(reward)); } +function formatError(err: unknown): string { + return err instanceof Error ? `${err.name}: ${err.message}` : String(err); +} + +function writeDetails(args: Args, details: GradingDetails): void { + if (!args.detailsOut) return; + mkdirSync(dirname(args.detailsOut), { recursive: true }); + writeFileSync(args.detailsOut, JSON.stringify(details, null, 2)); +} + /** * Read the artifacts, grade through `makeJudge()`, and write reward.txt (+ - * optional grading_details.json). Both model resolution and the judge call run - * inside the try, so a missing key or a transient API error fails closed to - * reward 0 with the error recorded in the details rather than crashing the - * verifier. Takes a factory so the file contract is testable without a live + * optional grading_details.json). Every step the bin owns — reading + * ground_truth.json / answer.txt / the screenshots, model resolution, and the + * judge call — runs inside the try, so a missing/corrupt artifact, a missing + * key, or a transient API error fails closed to reward 0 with the error recorded + * in the details rather than throwing out of the bin and leaving the reward file + * empty. Takes a factory so the file contract is testable without a live * provider call. */ export async function run(args: Args, makeJudge: () => JudgeModel): Promise<void> { - const task = loadGroundTruth(args.groundTruth).task; - const answer = loadAnswer(args.answer); - const shots = lastShots(args.shots, args.maxImages); - - // No answer and no screenshots: nothing to judge, fail closed without details. - if (!answer && shots.length === 0) { - writeReward(args.rewardOut, 0); - return; - } - + let answer = ""; + let nImages = 0; let verdict = ""; let reward: 0 | 1 = 0; - let error: string | null = null; try { + const task = loadGroundTruth(args.groundTruth).task; + answer = loadAnswer(args.answer); + const shots = lastShots(args.shots, args.maxImages); + nImages = shots.length; + + // No answer and no screenshots: nothing to judge, fail closed without details. + if (!answer && shots.length === 0) { + writeReward(args.rewardOut, 0); + return; + } + ({ verdict, reward } = await gradeWithWebJudge({ task, answer, shots, judge: makeJudge() })); } catch (err) { - error = err instanceof Error ? `${err.name}: ${err.message}` : String(err); - } - - writeReward(args.rewardOut, reward); - if (args.detailsOut) { - const details: GradingDetails = { + writeReward(args.rewardOut, 0); + writeDetails(args, { verdict_raw: verdict, - reward, - n_images: shots.length, + reward: 0, + n_images: nImages, answer, model: args.judgeModel, - error, - }; - mkdirSync(dirname(args.detailsOut), { recursive: true }); - writeFileSync(args.detailsOut, JSON.stringify(details, null, 2)); + error: formatError(err), + }); + return; } + + writeReward(args.rewardOut, reward); + writeDetails(args, { + verdict_raw: verdict, + reward, + n_images: nImages, + answer, + model: args.judgeModel, + error: null, + }); } async function main(): Promise<void> { diff --git a/benchmarks/adapters/webvoyager/judge/test/judge.test.ts b/benchmarks/adapters/webvoyager/judge/test/judge.test.ts index 1a4fa47..2131e7e 100644 --- a/benchmarks/adapters/webvoyager/judge/test/judge.test.ts +++ b/benchmarks/adapters/webvoyager/judge/test/judge.test.ts @@ -107,6 +107,30 @@ describe("run", () => { expect(readFileSync(args.rewardOut, "utf8")).toBe("0"); expect(readDetails(args).error).toContain("ANTHROPIC_API_KEY"); }); + + it("fails closed to reward 0 when ground_truth.json is missing", async () => { + const { args } = setup({ answer: "ans", shotIndices: [1] }); + args.groundTruth = join(args.shots, "does-not-exist.json"); + let called = false; + await expect( + run(args, () => { + called = true; + return { async complete() { return "SUCCESS"; } }; + }), + ).resolves.toBeUndefined(); + expect(called).toBe(false); + expect(readFileSync(args.rewardOut, "utf8")).toBe("0"); + expect(readDetails(args).error).toBeTruthy(); + }); + + it("fails closed to reward 0 when ground_truth.json is corrupt", async () => { + const { args } = setup({ answer: "ans", shotIndices: [1] }); + writeFileSync(args.groundTruth, "{ not valid json"); + await run(args, () => ({ async complete() { return "SUCCESS"; } })); + expect(readFileSync(args.rewardOut, "utf8")).toBe("0"); + expect(readDetails(args).reward).toBe(0); + expect(readDetails(args).error).toBeTruthy(); + }); }); describe("parseMaxImages", () => { diff --git a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml index 15e5827..18d6935 100644 --- a/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml +++ b/benchmarks/adapters/webvoyager/src/webvoyager/task-template/task.toml @@ -18,7 +18,12 @@ reference_type = "{reference_type}" timeout_sec = 600.0 [verifier.env] +# JUDGE_MODEL is a pi-ai provider:name ref (bare name = anthropic). The judge +# resolves the key for whichever provider it names, so both are forwarded: +# the default claude-sonnet-4-5 uses ANTHROPIC_API_KEY; an openai:o4-mini +# override uses OPENAI_API_KEY. ANTHROPIC_API_KEY = "${ANTHROPIC_API_KEY}" +OPENAI_API_KEY = "${OPENAI_API_KEY}" JUDGE_MODEL = "${WEBVOYAGER_JUDGE_MODEL:-claude-sonnet-4-5}" MAX_IMAGES = "${WEBVOYAGER_MAX_IMAGES:-15}" diff --git a/benchmarks/adapters/webvoyager/tests/test_adapter.py b/benchmarks/adapters/webvoyager/tests/test_adapter.py index 98df38c..1563a15 100644 --- a/benchmarks/adapters/webvoyager/tests/test_adapter.py +++ b/benchmarks/adapters/webvoyager/tests/test_adapter.py @@ -102,6 +102,8 @@ def test_task_toml_valid_and_named(adapter: WebVoyagerAdapter) -> None: assert "allrecipes" in parsed["task"]["keywords"] assert parsed["metadata"]["start_url"] == "https://www.allrecipes.com/" assert parsed["verifier"]["env"]["ANTHROPIC_API_KEY"] == "${ANTHROPIC_API_KEY}" + # JUDGE_MODEL=openai:o4-mini needs OPENAI_API_KEY on the verifier VM. + assert parsed["verifier"]["env"]["OPENAI_API_KEY"] == "${OPENAI_API_KEY}" def test_ground_truth_carries_task(adapter: WebVoyagerAdapter) -> None: