diff --git a/taskcluster/docs/kinds.md b/taskcluster/docs/kinds.md index a8058becd6e2a..34b9cc930ebb5 100644 --- a/taskcluster/docs/kinds.md +++ b/taskcluster/docs/kinds.md @@ -125,6 +125,12 @@ unit tests, source-code analysis, or measurement work. While source-test tasks r a source checkout, it is still possible for them to depend on a build artifact, though often they do not. +## try-status + +Find the try push of a pull request's branch and generate one task per task of +that push, each reporting the outcome of the task it monitors on the pull +request. + ## code-review Publish issues found by source-test tasks on Phabricator. diff --git a/taskcluster/gecko_taskgraph/target_tasks.py b/taskcluster/gecko_taskgraph/target_tasks.py index 33d9a80111230..1c031116f310b 100644 --- a/taskcluster/gecko_taskgraph/target_tasks.py +++ b/taskcluster/gecko_taskgraph/target_tasks.py @@ -633,7 +633,17 @@ def filter(task): return False - return [l for l in filtered_for_project if filter(full_task_graph[l])] + selected = [l for l in filtered_for_project if filter(full_task_graph[l])] + + level = int(parameters["level"]) + # Make sure to always schedule, but on PR only + selected += [ + label + for label, task in full_task_graph.tasks.items() + if task.kind == "try-status" and label not in selected and level < 3 + ] + + return selected @register_target_task("graphics_tasks") diff --git a/taskcluster/gecko_taskgraph/transforms/try_status.py b/taskcluster/gecko_taskgraph/transforms/try_status.py new file mode 100644 index 0000000000000..cb26deaf1b2e1 --- /dev/null +++ b/taskcluster/gecko_taskgraph/transforms/try_status.py @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Hand the branch of the current push to the try status generator. + +The generator looks that branch up in the try repository at runtime, so the only +thing it needs from the decision task is where this push came from, and the right +to put the report tasks it creates on this push. +""" + +from taskgraph.transforms.base import TransformSequence + +from gecko_taskgraph.transforms.task import ( + TREEHERDER_ROUTE_ROOT, + get_branch_rev, + get_treeherder_project, +) + +transforms = TransformSequence() + + +@transforms.add +def add_push_coordinates(config, jobs): + # The decision task only holds the route of the push it is running for, so + # the scope has to name that one route rather than the whole namespace. Built + # the same way as the route itself, in gecko_taskgraph.transforms.task. + route = f"{TREEHERDER_ROUTE_ROOT}.v2.{get_treeherder_project(config)}.{get_branch_rev(config)}" + + for job in jobs: + env = job.setdefault("worker", {}).setdefault("env", {}) + env["TRY_STATUS_HEAD_REF"] = config.params["head_ref"] or "" + # The commit the try push has to sit on for it to be this pull request. + env["TRY_STATUS_HEAD_REV"] = config.params["head_rev"] + env["TRY_STATUS_TRUST_DOMAIN"] = config.graph_config["trust-domain"] + + job.setdefault("scopes", []).append(f"queue:route:{route}") + yield job diff --git a/taskcluster/kinds/try-status/kind.yml b/taskcluster/kinds/try-status/kind.yml new file mode 100644 index 0000000000000..e7d7f6ec3e56a --- /dev/null +++ b/taskcluster/kinds/try-status/kind.yml @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - gecko_taskgraph.transforms.try_status:transforms + - gecko_taskgraph.transforms.job:transforms + - gecko_taskgraph.transforms.task:transforms + +tasks: + gen: + label: try-status-gen + description: >- + Find the try push of this branch and generate one task per task of + that push, to report its outcome on this pull request + + # Selected by name in target_tasks_enterprise_firefox_with_tests, since + # it reports on the try push of the branch rather than on the graph it + # belongs to and so has no business going through the project filter. + run-on-projects: [] + + # `code-review` earns the `checks` route in + # gecko_taskgraph.transforms.task, which is what puts a task on the pull + # request. It applies to this task so that a branch with no try push is + # reported, and the generated tasks carry the same route. + attributes: + code-review: true + + worker-type: t-linux-arm64-docker + worker: + docker-image: {in-tree: debian12-base} + # Long enough to wait for the try decision task to run. + max-run-time: 3600 + taskcluster-proxy: true + env: + TRY_STATUS_TRY_REMOTE: https://github.com/mozilla/enterprise-firefox-try + TRY_STATUS_TRY_PROJECT: enterprise-firefox-try + TRY_STATUS_TREEHERDER_URL: https://treeherder.mozilla.org + TRY_STATUS_REPORT_SCRIPT: taskcluster/scripts/try-status-report.py + artifacts: + - name: public/try-status + path: /builds/worker/artifacts + type: directory + + # Creating the report tasks, in this task group, with the routes that + # surface them on the pull request. No secret is involved: nothing here + # authenticates to GitHub, Taskcluster reports on its own tasks. The + # treeherder route is added by the transform, which is where the exact + # route of this push is known. + scopes: + - queue:create-task:highest:enterprise-t/t-linux-arm64-docker + - queue:scheduler-id:enterprise-level-{level} + - queue:route:checks + + treeherder: + kind: other + symbol: try-status + tier: 1 + platform: gecko-decision/opt + + run: + using: run-task + checkout: true + cwd: '{checkout}' + command: python3 taskcluster/scripts/try-status-gen.py diff --git a/taskcluster/scripts/try-status-gen.py b/taskcluster/scripts/try-status-gen.py new file mode 100644 index 0000000000000..7861900108063 --- /dev/null +++ b/taskcluster/scripts/try-status-gen.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Generate one report task per task of the try push of this branch. + +Finds the branch of this pull request in the try repository, resolves its first +commit to the decision task indexed under +.v2..revision..taskgraph.decision, waits for +that decision task, and reads the task graph it published. Every task in there +gets a try-status-report task in this task group, depending on the completion of +the task it monitors. + +Nothing here authenticates to GitHub. The generated tasks carry the `checks` +route, the one `code-review` tasks are given, and Taskcluster reports them on the +pull request by itself. + +Standard library only, the base image has neither jq nor curl. +""" + +import base64 +import json +import os +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + +USER_AGENT = "enterprise-try-status (+https://github.com/mozilla/enterprise-firefox)" + +HTTP_ATTEMPTS = 3 +# How long to wait for the try push to show up. A push to try lands its branch +# first, and only gets indexed once its decision task has completed, so both are +# waited on under this one deadline. +PUSH_TIMEOUT = 900 +PUSH_INTERVAL = 30 +# How long to wait for the try decision task to resolve. +DECISION_TIMEOUT = 1800 +DECISION_INTERVAL = 30 +# Refuse to flood a pull request beyond this. +MAX_REPORTS = 500 +# Prefixes that keep a report distinguishable from the job it mirrors, both in +# the task label and on the treeherder row it lands on. +REPORT_PREFIX = "try-status-report-" +PLATFORM_PREFIX = "try-" +# The kind holds create-task at `highest`, which is the priority the repository +# role grants, and that satisfies creating a task at any priority. These are +# cheap and wait on a dependency anyway, so let's make sure that the run as soon +# as possible +REPORT_PRIORITY = "highest" +# Days before a report task gives up waiting, and before its artifacts expire. +REPORT_DEADLINE_DAYS = 3 +REPORT_EXPIRES_DAYS = 28 + + +def log(message): + print(message, flush=True) + + +def request(url, method="GET", body=None, raw=False): + headers = {"User-Agent": USER_AGENT} + data = None + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + + last_error = None + for attempt in range(1, HTTP_ATTEMPTS + 1): + message = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(message, timeout=120) as response: + payload = response.read() + if raw: + return payload + return json.loads(payload) if payload else {} + except urllib.error.HTTPError as error: + if 400 <= error.code < 500 and error.code != 429: + raise + last_error = error + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + last_error = error + if attempt < HTTP_ATTEMPTS: + time.sleep(2**attempt) + raise RuntimeError( + f"{method} {url} failed after {HTTP_ATTEMPTS} attempts: {last_error}" + ) + + +def slugid(): + return base64.urlsafe_b64encode(uuid.uuid4().bytes).rstrip(b"=").decode() + + +def stamp(days=0, seconds=0): + moment = time.gmtime(time.time() + days * 86400 + seconds) + return time.strftime("%Y-%m-%dT%H:%M:%S.000Z", moment) + + +def branch_refs(remote, branch): + """The commits `git ls-remote` reports for this branch, as [(sha, ref)]. + + A pattern is matched against the tail of each ref, so the bare branch name + of the pull request also finds the refs/heads/user// copies + that `mach try` pushes. + """ + listing = subprocess.run( + ["git", "ls-remote", "--heads", remote, branch], + check=True, + capture_output=True, + text=True, + timeout=120, + ).stdout + return [ + (sha, ref) + for sha, _, ref in (line.partition("\t") for line in listing.splitlines()) + if ref.startswith("refs/heads/") and ref.endswith(f"/{branch}") + ] + + +def second_commit(workdir, remote, sha): + """The parent commit of `sha`. + + A try push is a branch with one commit on top holding what to run, so the + commit under the tip is the one the developer pushed. `tree:0` keeps this to + the commit objects, which is a fraction of a second rather than a clone. + """ + + def git(*args): + return subprocess.run( + ["git", "-C", workdir, *args], + check=True, + capture_output=True, + text=True, + timeout=300, + ).stdout.strip() + + try: + git("fetch", "-q", "--depth=2", "--filter=tree:0", remote, sha) + return git("rev-parse", f"{sha}^") + except subprocess.CalledProcessError as error: + log(f"Could not read the commit under {sha[:12]}: {error.stderr.strip()}") + return None + + +def wait_for_try_push(root_url, trust_domain, project, remote, branch, head_rev): + """Wait for the try push of this pull request, and its decision task. + + A branch of this name is not enough: `mach try` pushes the branch with one + commit on top, so the commit under the tip has to be the head of the pull + request, or the push is of some other revision and reporting it here would + be a lie. Several users can also hold a branch of the same name, and this is + what tells them apart. + + Pushing to try is not instant from here either: the branch appears first, + and the index entry only exists once the decision task has completed. + """ + workdir = tempfile.mkdtemp(prefix="try-status-") + subprocess.run(["git", "init", "-q", workdir], check=True, timeout=60) + + deadline = time.monotonic() + PUSH_TIMEOUT + parents = {} + while True: + refs = branch_refs(remote, branch) + waiting = f"no branch named '{branch}' yet" + + for sha, ref in refs: + if sha not in parents: + parents[sha] = second_commit(workdir, remote, sha) + parent = parents[sha] + if parent != head_rev: + log( + f" {ref} -> {sha[:12]} sits on {(parent or '?')[:12]}, not {head_rev[:12]}" + ) + waiting = f"'{branch}' is pushed, but from another revision" + continue + + log(f" {ref} -> {sha[:12]} sits on {head_rev[:12]}") + decision_id = decision_task_of(root_url, trust_domain, project, sha) + if decision_id: + return sha, decision_id + waiting = ( + f"{sha[:12]} is not indexed yet, its decision task may still be running" + ) + + if time.monotonic() >= deadline: + log(f"Gave up after {PUSH_TIMEOUT}s: {waiting}") + return None, None + log(f"{waiting}, retrying in {PUSH_INTERVAL}s") + time.sleep(PUSH_INTERVAL) + + +def decision_task_of(root_url, trust_domain, project, revision): + namespace = f"{trust_domain}.v2.{project}.revision.{revision}.taskgraph.decision" + log(f"Looking up {namespace}") + try: + return request(f"{root_url}/api/index/v1/task/{namespace}")["taskId"] + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + + +def wait_for(root_url, task_id): + """Wait until a task stops running, and return its state.""" + deadline = time.monotonic() + DECISION_TIMEOUT + while True: + status = request(f"{root_url}/api/queue/v1/task/{task_id}/status")["status"] + state = status["state"] + if state not in ("unscheduled", "pending", "running"): + return state + if time.monotonic() >= deadline: + return state + log(f"Decision task is {state}, waiting {DECISION_INTERVAL}s") + time.sleep(DECISION_INTERVAL) + + +def published_task_graph(root_url, task_id): + url = f"{root_url}/api/queue/v1/task/{task_id}/artifacts/public%2Ftask-graph.json" + return json.loads(request(url, raw=True)) + + +def report_task( + template, monitored, monitored_id, script, treeherder_url, project, revision +): + """Build the task that monitors one task of the try push. + + It keeps the treeherder placement of the task it monitors, so that it reads + the same way, with the platform prefixed: these land on the pull request push + next to the jobs of that pull request, and the two have to be told apart. + """ + label = monitored.get("label") or monitored["task"]["metadata"]["name"] + name = f"{REPORT_PREFIX}{label}" + task_url = f"{treeherder_url}/jobs?{urllib.parse.urlencode({'repo': project, 'revision': revision})}" + + treeherder = dict(monitored["task"].get("extra", {}).get("treeherder", {})) + machine = dict(treeherder.get("machine", {})) + machine["platform"] = "{}{}".format( + PLATFORM_PREFIX, machine.get("platform", "other") + ) + treeherder["machine"] = machine + try_task_url = f"{task_url}&selectedTaskRun={monitored_id}.0" + + # Deliberately not inheriting the environment of this task: that one is set + # up for run-task and a checkout, and the worker provides + # TASKCLUSTER_ROOT_URL by itself. + environment = { + "TRY_STATUS_TASK_ID": monitored_id, + "TRY_STATUS_TASK_LABEL": label, + "TRY_STATUS_TASK_URL": try_task_url, + "MOZ_UPLOAD_DIR": "/builds/worker/artifacts", + } + + # The image of an in-tree docker task is an artifact of the task that built + # it, which the worker mounts, and it will not mount an artifact of a task it + # does not depend on. + dependencies = [monitored_id] + image = template["payload"]["image"] + if isinstance(image, dict) and image.get("taskId"): + dependencies.append(image["taskId"]) + + routes = ["checks"] + [ + route + for route in template.get("routes", []) + if route.startswith("tc-treeherder.") + ] + + return { + "taskGroupId": template["taskGroupId"], + "schedulerId": template["schedulerId"], + "projectId": template.get("projectId", "none"), + "provisionerId": template["provisionerId"], + "workerType": template["workerType"], + "priority": REPORT_PRIORITY, + # The point of the whole thing: hold this task until the task it + # monitors has resolved, whatever it resolved to. + "dependencies": dependencies, + "requires": "all-resolved", + "created": stamp(), + "deadline": stamp(days=REPORT_DEADLINE_DAYS), + "expires": stamp(days=REPORT_EXPIRES_DAYS), + "scopes": [], + "routes": routes, + "payload": { + "image": template["payload"]["image"], + "maxRunTime": 1800, + "env": environment, + "command": [ + "/bin/bash", + "-cx", + f"echo {base64.b64encode(script.encode()).decode()} | base64 -d > /tmp/try-status-report.py && " + "python3 /tmp/try-status-report.py", + ], + "artifacts": { + "public/try-status": { + "type": "directory", + "path": "/builds/worker/artifacts", + "expires": stamp(days=REPORT_EXPIRES_DAYS), + } + }, + }, + "metadata": { + "name": name, + "description": f"Outcome of `{label}` on the try push of this branch ([Treeherder job]({try_task_url}))", + "owner": template["metadata"]["owner"], + "source": template["metadata"]["source"], + }, + "tags": {"kind": "try-status-report", "label": name}, + "extra": {"treeherder": treeherder, "try-status": {"taskId": monitored_id}}, + } + + +def artifact(name, content): + directory = os.environ.get("MOZ_UPLOAD_DIR", "/builds/worker/artifacts") + os.makedirs(directory, exist_ok=True) + path = os.path.join(directory, name) + with open(path, "w", encoding="utf-8") as handle: + handle.write(content) + log(f"Wrote {path}") + + +def main(): + root_url = os.environ["TASKCLUSTER_ROOT_URL"].rstrip("/") + proxy_url = os.environ["TASKCLUSTER_PROXY_URL"].rstrip("/") + trust_domain = os.environ["TRY_STATUS_TRUST_DOMAIN"] + project = os.environ["TRY_STATUS_TRY_PROJECT"] + remote = os.environ["TRY_STATUS_TRY_REMOTE"] + treeherder_url = os.environ["TRY_STATUS_TREEHERDER_URL"].rstrip("/") + branch = os.environ.get("TRY_STATUS_HEAD_REF", "").removeprefix("refs/heads/") + head_rev = os.environ["TRY_STATUS_HEAD_REV"] + + with open(os.environ["TRY_STATUS_REPORT_SCRIPT"], encoding="utf-8") as handle: + script = handle.read() + + template = request(f"{root_url}/api/queue/v1/task/{os.environ['TASK_ID']}") + + log(f"Looking for a try push of '{branch}' on {head_rev[:12]} in {remote}") + revision, decision_id = wait_for_try_push( + root_url, trust_domain, project, remote, branch, head_rev + ) + if decision_id is None: + log(f"::error::no {project} push found for branch '{branch}'") + log("Push it with `./mach try` and update this pull request.") + return 1 + + log(f"Try decision task is {decision_id}") + state = wait_for(root_url, decision_id) + if state in ("unscheduled", "pending", "running"): + log(f"::error::gave up waiting for the try decision task, it is {state}") + return 1 + if state != "completed": + log(f"::error::the try decision task is {state}, so it published no graph") + return 1 + + graph = published_task_graph(root_url, decision_id) + log(f"The try push generated {len(graph)} task(s)") + + created = [] + for monitored_id, monitored in sorted(graph.items()): + if len(created) >= MAX_REPORTS: + log(f"Stopping at {MAX_REPORTS} report tasks, {len(graph)} were found") + break + definition = report_task( + template, monitored, monitored_id, script, treeherder_url, project, revision + ) + task_id = slugid() + request(f"{proxy_url}/queue/v1/task/{task_id}", method="PUT", body=definition) + log(f" {definition['metadata']['name']} -> {task_id} on {monitored_id}") + created.append({ + "taskId": task_id, + "monitors": monitored_id, + "label": definition["metadata"]["name"], + }) + + artifact( + "generated-tasks.json", + json.dumps( + { + "branch": branch, + "revision": revision, + "decisionTaskId": decision_id, + "taskGroupUrl": f"{root_url}/tasks/groups/{decision_id}", + "treeherderUrl": "{}/jobs?{}".format( + treeherder_url, + urllib.parse.urlencode({"repo": project, "revision": revision}), + ), + "tasks": created, + }, + indent=2, + ), + ) + log(f"Generated {len(created)} report task(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taskcluster/scripts/try-status-report.py b/taskcluster/scripts/try-status-report.py new file mode 100644 index 0000000000000..bb73f4db2d23c --- /dev/null +++ b/taskcluster/scripts/try-status-report.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Replicate the outcome of one task of the try push of this branch. + +Generated by try-status-gen, one per task of the try push, and run by the queue +once the task it monitors has resolved: the whole wait is a dependency, so there +is nothing to poll here. It copies the status and the logs of that task, points +at it with an artifact, and exits with its outcome. + +This file is read by try-status-gen and embedded in the tasks it generates, so +it runs without a checkout and must stay standard library only. +""" + +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +USER_AGENT = "enterprise-try-status (+https://github.com/mozilla/enterprise-firefox)" + +ARTIFACTS = os.environ.get("MOZ_UPLOAD_DIR", "/builds/worker/artifacts") +# Logs of a try task run to tens of megabytes, and this is a copy of one, so the +# tail is dropped rather than moving the whole thing for every task of a push. +MAX_LOG_BYTES = 20 * 1024 * 1024 +CHUNK = 1024 * 1024 +# `live.log` redirects to `live_backing.log` and `certified.log` repeats it for +# chain of trust, so copying either would move the same bytes twice. +SKIPPED_LOGS = frozenset({"public/logs/live.log", "public/logs/certified.log"}) +# Resolutions that say nothing about the change being tested. +INFRA_REASONS = frozenset({ + "claim-expired", + "deadline-exceeded", + "internal-error", + "intermittent-task", + "resource-unavailable", + "worker-shutdown", +}) + + +def log(message): + print(message, flush=True) + + +def get(url): + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + return urllib.request.urlopen(request, timeout=120) + + +def get_json(url): + with get(url) as response: + return json.load(response) + + +def write(name, content): + os.makedirs(ARTIFACTS, exist_ok=True) + with open(os.path.join(ARTIFACTS, name), "w", encoding="utf-8") as handle: + handle.write(content) + + +def copy_logs(root_url, task_id, run_id): + """Bring the logs of the monitored task over as artifacts of this one.""" + url = f"{root_url}/api/queue/v1/task/{task_id}/runs/{run_id}/artifacts" + for entry in get_json(url).get("artifacts", []): + name = entry["name"] + if not name.startswith("public/logs/") or name in SKIPPED_LOGS: + continue + target = os.path.join(ARTIFACTS, os.path.basename(name)) + source = "{}/api/queue/v1/task/{}/runs/{}/artifacts/{}".format( + root_url, task_id, run_id, urllib.parse.quote(name, safe="") + ) + copied = 0 + try: + with get(source) as response, open(target, "wb") as handle: + while copied < MAX_LOG_BYTES: + chunk = response.read(min(CHUNK, MAX_LOG_BYTES - copied)) + if not chunk: + break + handle.write(chunk) + copied += len(chunk) + if response.read(1): + handle.write( + f"\n[truncated at {MAX_LOG_BYTES} bytes, see {task_id}]\n".encode() + ) + log(f"{name} was truncated at {MAX_LOG_BYTES} bytes") + except urllib.error.HTTPError as error: + log(f"Could not copy {name}: {error}") + continue + log( + f"Copied {name} ({copied} bytes) to public/try-status/{os.path.basename(name)}" + ) + + +def main(): + root_url = os.environ["TASKCLUSTER_ROOT_URL"].rstrip("/") + task_id = os.environ["TRY_STATUS_TASK_ID"] + label = os.environ.get("TRY_STATUS_TASK_LABEL", task_id) + task_url = os.environ.get("TRY_STATUS_TASK_URL", "") + + status = get_json(f"{root_url}/api/queue/v1/task/{task_id}/status")["status"] + state = status["state"] + runs = status.get("runs") or [] + run_id = runs[-1]["runId"] if runs else 0 + reason = runs[-1].get("reasonResolved") if runs else None + + inspector = f"{root_url}/tasks/{task_id}/runs/{run_id}" + log(f"{label} is {state} ({reason})") + log(f" task {inspector}") + log(f" treeherder {task_url}") + + write( + "original-task.json", + json.dumps( + { + "label": label, + "taskId": task_id, + "runId": run_id, + "state": state, + "reasonResolved": reason, + "taskUrl": inspector, + "treeherderUrl": task_url, + }, + indent=2, + ), + ) + write( + "original-task.html", + "" + f"{label}" + f"

{label} on enterprise-firefox-try" + f"

the same task on Treeherder\n", + ) + + copy_logs(root_url, task_id, run_id) + + if state == "completed": + return 0 + if reason == "superseded": + log("Superseded on try, nothing to report") + return 0 + if reason in INFRA_REASONS: + log( + "Infrastructure failure on try, not a result for this branch, report as failure" + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main())