diff --git a/.github/workflows/auto-start-ci.yml b/.github/workflows/auto-start-ci.yml index 34488eeed6d7..0a69ff636f25 100644 --- a/.github/workflows/auto-start-ci.yml +++ b/.github/workflows/auto-start-ci.yml @@ -1,3 +1,6 @@ +# This action uses the following secrets: +# JENKINS_USER: GitHub user whose Jenkins token is defined below +# JENKINS_TOKEN: Jenkins token, to be used to start CI name: Auto Start CI on: @@ -36,11 +39,13 @@ jobs: -t '{{ range . }}{{ .number }} {{ end }}' \ --limit 5)" >> "$GITHUB_OUTPUT" env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} start-ci: permissions: + checks: read contents: read pull-requests: write + statuses: read needs: get-prs-for-ci if: needs.get-prs-for-ci.outputs.numbers != '' runs-on: ubuntu-slim @@ -59,10 +64,10 @@ jobs: ncu-config set token "$GH_TOKEN" ncu-config set jenkins_token "$JENKINS_TOKEN" ncu-config set owner "$GITHUB_REPOSITORY_OWNER" - ncu-config set repo "$(echo "$GITHUB_REPOSITORY" | cut -d/ -f2)" + ncu-config set repo "${GITHUB_REPOSITORY#*/}" env: USERNAME: ${{ secrets.JENKINS_USER }} - GH_TOKEN: ${{ secrets.GH_USER_TOKEN }} + GH_TOKEN: ${{ github.token }} JENKINS_TOKEN: ${{ secrets.JENKINS_TOKEN }} - name: Start the CI @@ -70,5 +75,4 @@ jobs: curl -fsSL "https://github.com/${GITHUB_REPOSITORY}/raw/${GITHUB_SHA}/tools/actions/start-ci.sh" \ | sh -s -- ${{ needs.get-prs-for-ci.outputs.numbers }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/commit-queue.yml b/.github/workflows/commit-queue.yml index a4d61132deea..89c22d50048b 100644 --- a/.github/workflows/commit-queue.yml +++ b/.github/workflows/commit-queue.yml @@ -54,7 +54,7 @@ jobs: jq -r -s 'reduce .[] as $pr ([]; if index($pr) then . else . + [$pr] end) | join(" ")') echo "candidates=$candidates" >> "$GITHUB_OUTPUT" env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} commitQueue: needs: get_candidate_prs if: needs.get_candidate_prs.outputs.candidates != '' @@ -65,18 +65,14 @@ jobs: statuses: read runs-on: ubuntu-slim steps: - # Install dependencies - name: Install Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: ${{ env.NODE_VERSION }} + - name: Install @node-core/utils run: npm install -g @node-core/utils - - name: Set variables - run: | - echo "REPOSITORY=$(echo "$GITHUB_REPOSITORY" | cut -d/ -f2)" >> "$GITHUB_ENV" - - name: Configure @node-core/utils run: | # Keep the config outside the workspace so checkout does not remove it. @@ -85,7 +81,7 @@ jobs: ncu-config --global set username "$USERNAME" ncu-config --global set token "$GH_TOKEN" ncu-config --global set jenkins_token "$JENKINS_TOKEN" - ncu-config --global set repo "${REPOSITORY}" + ncu-config --global set repo "${GITHUB_REPOSITORY#*/}" ncu-config --global set owner "${GITHUB_REPOSITORY_OWNER}" env: USERNAME: ${{ secrets.JENKINS_USER }} @@ -104,8 +100,6 @@ jobs: metadata="${RUNNER_TEMP}/metadata-${pr}.json" output="${RUNNER_TEMP}/metadata-${pr}.txt" if git node metadata "$pr" \ - --owner "$GITHUB_REPOSITORY_OWNER" \ - --repo "$REPOSITORY" \ --readme "$readme" \ --json > "$metadata" 2> "$output"; then metadata_status=0 @@ -173,7 +167,9 @@ jobs: - name: Start the Commit Queue if: steps.get_mergeable_prs.outputs.numbers != '' run: | + git config --local user.email "github-bot@iojs.org" + git config --local user.name "Node.js GitHub Bot" ncu-config set token "$GH_TOKEN" - ./tools/actions/commit-queue.sh "${GITHUB_REPOSITORY_OWNER}" "${REPOSITORY}" ${{ steps.get_mergeable_prs.outputs.numbers }} + ./tools/actions/commit-queue.sh ${{ steps.get_mergeable_prs.outputs.numbers }} env: GH_TOKEN: ${{ secrets.GH_USER_TOKEN }} diff --git a/doc/contributing/commit-queue.md b/doc/contributing/commit-queue.md index 08f01b32d4ee..4b0647852c98 100644 --- a/doc/contributing/commit-queue.md +++ b/doc/contributing/commit-queue.md @@ -11,48 +11,14 @@ blocked on a deferrable condition, currently wait time, the queue leaves the label in place and retries later. Other failures continue to the existing landing and failure-reporting path. -This document gives an overview of how the Commit Queue works, as well as -implementation details, reasoning for design choices, and current limitations. - -## Overview - -From a high-level, the Commit Queue works as follows: - -1. Collaborators will add `commit-queue` label to pull requests they want the - queue to land. The label can be added before the pull request has completed - its wait time, or before requested CI has finished. Required approvals must - already be in place. The commit queue does not request CI on its own. -2. On each scheduled run, the queue builds a candidate list from open pull - requests with the `commit-queue` label and without the `blocked` label. The - workflow uses a five-minute cron, but GitHub Actions scheduled workflows are - not guaranteed to run exactly every five minutes. For each candidate, the - queue will: - 1. In the landing job, install and configure `@node-core/utils`, then run a - metadata-only readiness check without checking out the repository - 2. If the metadata check exits with a deferrable readiness code, meaning - the PR is only blocked on wait time, keep the `commit-queue` label and - skip this PR until a later queue run - 3. Check if the PR also has a `request-ci` label (if it has, skip this PR - since it's pending a CI run) - 4. Check whether GitHub checks are still running (if they are, skip this PR) - 5. Remove the `commit-queue` label and run `git node land` - 6. If it fails: - 1. Add the `commit-queue-failed` label to the PR - 2. Leave a comment on the PR with the output from `git node land` - 3. Abort the `git node land` session. If the abort succeeds, continue to - the next PR; otherwise, stop the queue in an unknown state - 7. If it succeeds: - 1. Push or merge the changes into nodejs/node - 2. Leave a comment on the PR with `Landed in ...` - 3. Close the PR - 4. Go to next PR in the queue - To make the Commit Queue squash all the commits of a pull request into the first one, add the `commit-queue-squash` label. To make the Commit Queue land a pull request containing several commits, add the `commit-queue-rebase` label. When using this option, make sure that all commits are self-contained, meaning every commit should pass all tests. +The implementation is in `commit-queue.yml` and `commit-queue.sh`. + ## Current limitations The Commit Queue feature is still in early stages, and as such it might not @@ -62,118 +28,9 @@ of the commit queue: 1. All commits in a pull request must either be following commit message guidelines or be a valid [`fixup!`](https://git-scm.com/docs/git-commit#Documentation/git-commit.txt---fixupamendrewordltcommitgt) commit that will be correctly handled by the [`--autosquash`](https://git-scm.com/docs/git-rebase#Documentation/git-rebase.txt---autosquash) - option -2. A CI must have run and succeeded since the last change on the PR -3. A collaborator must have approved the PR since the last change -4. Only Jenkins CI and GitHub Actions are checked (V8 CI and CITGM are ignored) + option. +2. A CI must have run and succeeded since the last change on the PR. +3. A collaborator must have approved the PR since the last change. +4. Only Jenkins CI and GitHub Actions are checked (V8 CI and CITGM are ignored). 5. The PR must target the `main` branch (PRs opened against other branches, such - as backport PRs, are ignored) - -## Implementation - -The [action](../../.github/workflows/commit-queue.yml) runs on scheduled events. -It uses a five-minute cron because that is the smallest interval accepted by -GitHub Actions. Scheduled workflows are not guaranteed to run exactly at that -cadence and might take longer between runs. - -The workflow also uses a concurrency group so only one commit queue run can be -active at a time. If a scheduled run starts while a previous run is still -running, GitHub Actions keeps at most one pending run for the same concurrency -group. A newer pending run replaces an older pending run. - -Using the scheduler is preferable over using pull\_request\_target for two -reasons: - -1. if two Commit Queue Actions execution overlap, there's a high-risk that - the last one to finish will fail because the local branch will be out of - sync with the remote after the first Action pushes. `issue_comment` event - has the same limitation. -2. `pull_request_target` will only run if the Action exists on the base commit - of a pull request, and it will run the Action version present on that - commit, meaning we wouldn't be able to use it for already opened PRs - without rebasing them first. - -The workflow starts with a small candidate job that uses GitHub CLI to fetch -pull requests with the `commit-queue` label. It first fetches the same -age-based and fast-track buckets the queue used before accepting early queue -requests, then fetches the broader queue and de-duplicates the result. This -keeps not-yet-ready PRs from crowding out PRs that the previous query would -have selected if GitHub paginates or caps a query result. - -If there are candidate PRs, the landing job installs and configures -`@node-core/utils` once with a personal token and a Jenkins token from -[@nodejs-github-bot](https://github.com/nodejs/github-bot). It then downloads -the workflow commit's README without checking out the repository and runs -`git node metadata --readme --json` for each candidate. This uses the same -`@node-core/utils` PR readiness checks as `git node land`, but does not clone, -fetch, or merge the PR. The filter consumes the structured metadata result -and its exit code instead of matching human-readable output: - -* exit code `0`: the PR is ready and is passed to - [`commit-queue.sh`](../../tools/actions/commit-queue.sh) -* exit codes `20`-`29`: the PR is not ready for a deferrable metadata reason, - currently wait time, so it keeps the `commit-queue` label and is retried - later -* exit codes `40`-`49`: the PR has a hard or mixed metadata readiness failure - and is passed to [`commit-queue.sh`](../../tools/actions/commit-queue.sh) - -The `20`-`29` exit code range is reserved by `@node-core/utils` for deferrable -metadata readiness states, and `40`-`49` is reserved for hard metadata failure -states. Unknown filter failures fail the workflow before starting the landing -script and leave PR labels unchanged so the queue can retry on a later -scheduled run. PRs passed through with exit code `40`-`49` continue through -`commit-queue.sh`. The workflow checks out the repository only when at least -one PR remains after filtering. The script still applies its existing -`request-ci` and pending-check deferrals before removing the queue label and -reporting a hard failure. - -> The personal token needs permission for public repositories and to read -> profiles. It is used by `@node-core/utils` and by the landing job for -> checkout, label and comment updates, merging, and pushing. Jenkins token is -> required to check CI status. - -`commit-queue.sh` receives the following positional arguments: - -1. The repository owner -2. The repository name -3. Every positional argument starting at this one will be a pull request ID of - a pull request with commit-queue set. - -The script will iterate over the pull requests. GitHub CLI is used to check if -the PR is waiting for CI to start (`request-ci` label) or still has pending -GitHub checks. The PR is skipped if CI is pending. No other CI validation is -done here since `git node land` will fail if the last CI failed. - -The script removes the `commit-queue` label, then runs `git node land`, -forwarding stdout and stderr to a file. PRs that are only blocked on wait time -should have already been filtered by the metadata check. If a hard readiness -failure appears between the metadata filter and `git node land`, the landing -job adds a `commit-queue-failed` label to the PR, leaves a comment with the -output of `git node land`, and then aborts the landing session. If the abort -fails, the queue stops instead of continuing in an unknown state. - -Fast-tracked PRs use the metadata check before checkout and the landing script. -If the fast-track request has not yet received enough collaborator thumbs-up, -the queue keeps the `commit-queue` label and retries until either the -fast-track request is approved or the PR becomes landable through the regular -wait-time rules. The commit queue does not create the fast-track request -comment; that is handled when the `fast-track` label is added. If that comment -is missing, the queue reports the failure instead of keeping the PR queued. - -If no errors happen during `git node land`, the script either pushes the direct -rebase landing to `main` or uses GitHub's squash merge API for single-commit and -fixup landings. It then leaves a `Landed in ...` comment in the PR. GitHub -closes PRs merged through the merge API automatically; for direct pushes, the -script closes the PR. Iteration continues until all PRs have done the steps -above. - -## Reverting broken commits - -Reverting broken commits is done manually by collaborators, just like when -commits are landed manually via `git node land`. An easy way to revert is a -good feature for the project, but is not explicitly required for the Commit -Queue to work because the Action lands PRs just like collaborators do today. If -once we start using the Commit Queue we notice that the number of required -reverts increases drastically, we can pause the queue until a Revert Queue is -implemented, but until then we can enable the Commit Queue and then work on a -Revert Queue as a follow-up. + as backport PRs, are ignored). diff --git a/tools/actions/commit-queue.sh b/tools/actions/commit-queue.sh index b6e62139d626..3802a44857f2 100755 --- a/tools/actions/commit-queue.sh +++ b/tools/actions/commit-queue.sh @@ -2,39 +2,31 @@ set -xe -OWNER=$1 -REPOSITORY=$2 -shift 2 - UPSTREAM=origin DEFAULT_BRANCH=main COMMIT_QUEUE_LABEL="commit-queue" COMMIT_QUEUE_FAILED_LABEL="commit-queue-failed" +cqurl="${GITHUB_SERVER_URL:?}/${GITHUB_REPOSITORY:?}/actions/runs/${GITHUB_RUN_ID:?}" + commit_queue_failed() { pr=$1 - gh pr edit "$pr" --add-label "${COMMIT_QUEUE_FAILED_LABEL}" --remove-label "${COMMIT_QUEUE_LABEL}" + gh -R "$GITHUB_REPOSITORY" pr edit "$pr" --add-label "${COMMIT_QUEUE_FAILED_LABEL}" --remove-label "${COMMIT_QUEUE_LABEL}" - # shellcheck disable=SC2154 - cqurl="${GITHUB_SERVER_URL}/${OWNER}/${REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" body="
Commit Queue failed
$(sed -e 's/&/\&/g' -e 's//\>/g' output)
$cqurl
" echo "$body" - gh pr comment "$pr" --body "$body" + gh -R "$GITHUB_REPOSITORY" pr comment "$pr" --body "$body" rm output } -# TODO(mmarchini): should this be set with whoever added the label for each PR? -git config --local user.email "github-bot@iojs.org" -git config --local user.name "Node.js GitHub Bot" - SHOULD_ABORT= for pr in "$@"; do - gh pr view "$pr" --json labels --jq ".labels" > labels.json + gh -R "$GITHUB_REPOSITORY" pr view "$pr" --json labels --jq ".labels" > labels.json # Skip PR if CI was requested if jq -e 'map(.name) | index("request-ci")' < labels.json; then echo "pr ${pr} skipped, waiting for CI to start" @@ -42,7 +34,7 @@ for pr in "$@"; do fi # Skip PR if CI is still running - if gh pr checks "$pr" | grep -q "\spending\s"; then + if gh -R "$GITHUB_REPOSITORY" pr checks "$pr" | grep -q "\spending\s"; then echo "pr ${pr} skipped, CI still running" continue fi @@ -98,7 +90,7 @@ for pr in "$@"; do --arg body "${commit_body}" \ --arg head "${commit_head}" \ '{merge_method:"squash",commit_title:$title,commit_message:$body,sha:$head}' |\ - gh api -X PUT "repos/${OWNER}/${REPOSITORY}/pulls/${pr}/merge" --input -\ + gh api -X PUT "repos/${GITHUB_REPOSITORY}/pulls/${pr}/merge" --input -\ --jq 'if .merged then .sha else halt_error end' )"; then commit_queue_failed "$pr" @@ -108,12 +100,12 @@ for pr in "$@"; do rm output - gh pr comment "$pr" --body "Landed in $commits" + gh -R "$GITHUB_REPOSITORY" pr comment "$pr" --body "Landed in $commits" - [ -z "$MULTIPLE_COMMIT_POLICY" ] && gh pr close "$pr" + [ -z "$MULTIPLE_COMMIT_POLICY" ] && gh -R "$GITHUB_REPOSITORY" pr close "$pr" # Delete the commit queue label (but ignore errors, it's no big deal if a closed PR still has the label) - gh pr edit "$pr" --remove-label "$COMMIT_QUEUE_LABEL" || true + gh -R "$GITHUB_REPOSITORY" pr edit "$pr" --remove-label "$COMMIT_QUEUE_LABEL" || true done rm -f labels.json diff --git a/tools/actions/start-ci.sh b/tools/actions/start-ci.sh index 4d4fadf958a9..d4d19b92082d 100755 --- a/tools/actions/start-ci.sh +++ b/tools/actions/start-ci.sh @@ -4,9 +4,10 @@ set -xe REQUEST_CI_LABEL="request-ci" REQUEST_CI_FAILED_LABEL="request-ci-failed" +cqurl="${GITHUB_SERVER_URL:?}/${GITHUB_REPOSITORY:?}/actions/runs/${GITHUB_RUN_ID:?}" for pr in "$@"; do - gh pr edit "$pr" --remove-label "$REQUEST_CI_LABEL" + gh -R "$GITHUB_REPOSITORY" pr edit "$pr" --remove-label "$REQUEST_CI_LABEL" ci_started=yes rm -f output; @@ -15,14 +16,12 @@ for pr in "$@"; do if [ "$ci_started" = "no" ]; then # Do we need to reset? - gh pr edit "$pr" --add-label "$REQUEST_CI_FAILED_LABEL" + gh -R "$GITHUB_REPOSITORY" pr edit "$pr" --add-label "$REQUEST_CI_FAILED_LABEL" - # shellcheck disable=SC2154 - cqurl="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" body="
Failed to start CI
$(cat output)
$cqurl
" echo "$body" - gh pr comment "$pr" --body "$body" + gh -R "$GITHUB_REPOSITORY" pr comment "$pr" --body "$body" rm output fi