GH Rate Limit Fix

Author: UltraProdigy

README.md

@@ -259,6 +259,8 @@ Both inputs select repositories directly, independently of the configured sync s
 
 Override mode stops before any changes if the receiving repository has a label named `.` or `..`, because those names cannot be safely addressed through the label API's URL path.
 
+Transfers pause at least one second between label writes to reduce GitHub secondary rate limits. When GitHub rejects a request due to rate limiting, the workflow logs the wait and retries up to five times, honoring `Retry-After` and exhausted primary-limit reset headers. Retry waits start at one minute and grow exponentially; GitHub's headers can require a longer pause. A transfer creating 233 labels takes roughly four minutes plus API response time and any rate-limit waits. Permission, validation, and ambiguous network/server errors still stop the run. Rerun a partially completed transfer with the same inputs to finish the remaining changes.
+
 The workflow uses the Org-Label-Sync changelog layout in the GitHub Actions run summary, showing the source and receiving repositories, test and override settings, starting label counts, and created, updated, deleted, and retained counts. Retained labels are existing receiving labels left unchanged. Preview changelogs are marked as test-mode output. If the transfer fails partway through, the summary records completed changes and the failure; rerunning continues from the current label state.
 
 ### Config-Reset

scripts/lib/github-request.mjs

@@ -0,0 +1,64 @@
+import { setTimeout as sleepTimer } from "node:timers/promises";
+
+const mutationMethods = new Set(["POST", "PUT", "PATCH", "DELETE"]);
+const maxRateLimitRetries = 5;
+
+function retryAfterMilliseconds(value, now) {
+  if (!value?.trim()) return 0;
+  const seconds = Number(value);
+  if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000;
+  const date = Date.parse(value);
+  return Number.isFinite(date) ? Math.max(0, date - now) : 0;
+}
+
+// Keep one client per transfer and call it serially so all label mutations share
+// the same pacing window. Only explicit rate-limit rejections are safe to retry.
+export function createGithubRequest(token, { sleep = sleepTimer, now = Date.now } = {}) {
+  let nextWriteAt = 0;
+
+  return async (method, apiPath, body) => {
+    const isMutation = mutationMethods.has(method);
+    for (let retry = 0; ; retry += 1) {
+      const pacingDelay = nextWriteAt - now();
+      if (isMutation && pacingDelay > 0) await sleep(pacingDelay);
+
+      const response = await fetch(`https://api.github.com${apiPath}`, {
+        method,
+        headers: {
+          Accept: "application/vnd.github+json",
+          Authorization: `Bearer ${token}`,
+          "User-Agent": "label-sync",
+          "X-GitHub-Api-Version": "2022-11-28",
+        },
+        body: body ? JSON.stringify(body) : undefined,
+      });
+      if (isMutation) nextWriteAt = now() + 1000;
+      if (response.ok) return response.status === 204 ? null : response.json();
+
+      const message = await response.text();
+      const primaryLimitExhausted = response.headers.get("x-ratelimit-remaining") === "0";
+      const rateLimited = response.status === 429 || (response.status === 403 && (
+        primaryLimitExhausted
+        || response.headers.has("retry-after")
+        || /rate limit|abuse detection/i.test(message)
+      ));
+      if (!rateLimited || retry === maxRateLimitRetries) {
+        const exhausted = rateLimited ? ` (rate limit persisted after ${maxRateLimitRetries} retries)` : "";
+        throw new Error(`${method} ${apiPath} failed with ${response.status}: ${message}${exhausted}`);
+      }
+
+      const currentTime = now();
+      const resetSeconds = Number(response.headers.get("x-ratelimit-reset"));
+      const resetDelay = primaryLimitExhausted && Number.isFinite(resetSeconds) && resetSeconds > 0
+        ? Math.max(0, resetSeconds * 1000 - currentTime + 1000)
+        : 0;
+      const waitMilliseconds = Math.max(
+        60000 * (2 ** retry),
+        retryAfterMilliseconds(response.headers.get("retry-after"), currentTime),
+        resetDelay,
+      );
+      console.warn(`GitHub rate limit on ${method} ${apiPath}. Waiting ${Math.ceil(waitMilliseconds / 1000)}s before retry ${retry + 1}/${maxRateLimitRetries}.`);
+      await sleep(waitMilliseconds);
+    }
+  };
+}

scripts/transfer-labels.mjs

@@ -4,6 +4,7 @@ import { assert, labelsExactlyMatch, normalizeDescription, normalizeName } from
 import { validateLabels } from "./lib/config-validation.mjs";
 import { renderLabelSyncSection, writeChangelog } from "./lib/changelog-utils.mjs";
 import { formatRepositoryLink, getRepositorySkipReason, parseTokenPermissions } from "./lib/repository-selection.mjs";
+import { createGithubRequest } from "./lib/github-request.mjs";
 
 function resolveRepository(value, organization, inputName) {
   const name = typeof value === "string" ? value.trim() : "";
@@ -20,27 +21,10 @@ function resolveRepository(value, organization, inputName) {
   return fullName;
 }
 
-async function githubRequest(token, method, apiPath, body) {
-  const response = await fetch(`https://api.github.com${apiPath}`, {
-    method,
-    headers: {
-      Accept: "application/vnd.github+json",
-      Authorization: `Bearer ${token}`,
-      "User-Agent": "label-sync",
-      "X-GitHub-Api-Version": "2022-11-28",
-    },
-    body: body ? JSON.stringify(body) : undefined,
-  });
-  if (!response.ok) {
-    throw new Error(`${method} ${apiPath} failed with ${response.status}: ${await response.text()}`);
-  }
-  return response.status === 204 ? null : response.json();
-}
-
-async function getAllLabels(token, repository) {
+async function getAllLabels(githubRequest, repository) {
   const labels = [];
   for (let page = 1; ; page += 1) {
-    const batch = await githubRequest(token, "GET", `/repos/${repository}/labels?per_page=100&page=${page}`);
+    const batch = await githubRequest("GET", `/repos/${repository}/labels?per_page=100&page=${page}`);
     assert(Array.isArray(batch), `Invalid label response for ${repository}.`);
     labels.push(...batch);
     if (batch.length < 100) {
@@ -60,6 +44,7 @@ export async function transferLabels({
   dryRun = false,
   overrideExisting = false,
   tokenPermissions = null,
+  githubRequest = createGithubRequest(token),
 }) {
   const result = {
     repository: "",
@@ -84,8 +69,8 @@ export async function transferLabels({
     targetName = resolveRepository(targetRepository, organization, "Receiving repository");
     assert(sourceName.toLowerCase() !== targetName.toLowerCase(), "Source and receiving repositories must be different repositories.");
     assert(token, "LABEL_SYNC_TOKEN is required.");
-    const source = await githubRequest(token, "GET", `/repos/${sourceName}`);
-    const target = await githubRequest(token, "GET", `/repos/${targetName}`);
+    const source = await githubRequest("GET", `/repos/${sourceName}`);
+    const target = await githubRequest("GET", `/repos/${targetName}`);
     assert(source.id && target.id, "GitHub did not return valid repository IDs.");
     assert(source.id !== target.id, "Source and receiving repositories must be different repositories.");
     // Use canonical names after resolving renamed/transferred repository aliases.
@@ -100,9 +85,9 @@ export async function transferLabels({
     }
 
     // Read and validate both complete label sets before making any changes.
-    const sourceLabels = await getAllLabels(token, sourceName);
+    const sourceLabels = await getAllLabels(githubRequest, sourceName);
     sourceCount = sourceLabels.length;
-    const targetLabels = await getAllLabels(token, targetName);
+    const targetLabels = await getAllLabels(githubRequest, targetName);
     initialTargetCount = targetLabels.length;
     if (overrideExisting) {
       for (const label of targetLabels) {
@@ -126,14 +111,14 @@ export async function transferLabels({
       const existing = targetByName.get(normalizeName(desired.name));
       if (!existing) {
         if (!dryRun) {
-          await githubRequest(token, "POST", `/repos/${targetName}/labels`, desired);
+          await githubRequest("POST", `/repos/${targetName}/labels`, desired);
         }
         result.createdLabels.push(desired);
         result.hasChanges = true;
         console.log(`  + ${desired.name}`);
       } else if (overrideExisting && !labelsExactlyMatch(existing, desired)) {
         if (!dryRun) {
-          await githubRequest(token, "PATCH", `/repos/${targetName}/labels/${encodeURIComponent(existing.name)}`, {
+          await githubRequest("PATCH", `/repos/${targetName}/labels/${encodeURIComponent(existing.name)}`, {
             new_name: desired.name,
             color: desired.color,
             description: desired.description,
@@ -151,7 +136,7 @@ export async function transferLabels({
       for (const existing of targetLabels) {
         if (sourceNames.has(normalizeName(existing.name))) continue;
         if (!dryRun) {
-          await githubRequest(token, "DELETE", `/repos/${targetName}/labels/${encodeURIComponent(existing.name)}`);
+          await githubRequest("DELETE", `/repos/${targetName}/labels/${encodeURIComponent(existing.name)}`);
         }
         result.deletedConfiguredLabels.push(existing);
         result.hasChanges = true;

test/github-request.test.mjs

@@ -0,0 +1,101 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { createGithubRequest } from "../scripts/lib/github-request.mjs";
+
+function setup(t, responses) {
+  let time = 1_700_000_000_000;
+  const waits = [];
+  const requests = [];
+  const request = createGithubRequest("test-token", {
+    now: () => time,
+    sleep: async (milliseconds) => {
+      waits.push(milliseconds);
+      time += milliseconds;
+    },
+  });
+  t.mock.method(globalThis, "fetch", async (url, options) => {
+    requests.push({ url, ...options, time });
+    assert.ok(responses.length, "Unexpected extra request");
+    return responses.shift();
+  });
+  return { request, waits, requests };
+}
+
+test("paces successive label writes by one second without slowing reads", async (t) => {
+  const fixture = setup(t, [Response.json([]), Response.json({}), Response.json([]), Response.json({}), new Response(null, { status: 204 })]);
+  await fixture.request("GET", "/repos/example/repo/labels");
+  await fixture.request("POST", "/repos/example/repo/labels", { name: "Bug" });
+  await fixture.request("GET", "/repos/example/repo/labels");
+  await fixture.request("PATCH", "/repos/example/repo/labels/Bug", { color: "000000" });
+  assert.equal(await fixture.request("DELETE", "/repos/example/repo/labels/Old"), null);
+  assert.deepEqual(fixture.waits, [1000, 1000]);
+  assert.deepEqual(fixture.requests.map(({ time }) => time - 1_700_000_000_000), [0, 0, 0, 1000, 2000]);
+});
+
+test("secondary 403 retries the rejected write with the same URL, body and auth", async (t) => {
+  const fixture = setup(t, [
+    Response.json({ message: "You have exceeded a secondary rate limit. Please wait a few minutes before you try again." }, { status: 403 }),
+    Response.json({ name: "Bug" }),
+  ]);
+  assert.deepEqual(await fixture.request("POST", "/repos/example/repo/labels", { name: "Bug" }), { name: "Bug" });
+  assert.deepEqual(fixture.waits, [60000]);
+  assert.equal(fixture.requests.length, 2);
+  for (const request of fixture.requests) {
+    assert.equal(request.method, "POST");
+    assert.equal(request.url, "https://api.github.com/repos/example/repo/labels");
+    assert.equal(request.body, '{"name":"Bug"}');
+    assert.equal(request.headers.Authorization, "Bearer test-token");
+  }
+});
+
+for (const [name, headers, expectedWait] of [
+  ["Retry-After seconds", { "retry-after": "180" }, 180000],
+  ["Retry-After HTTP date", { "retry-after": "Tue, 14 Nov 2023 22:16:20 GMT" }, 180000],
+  ["primary reset time", { "x-ratelimit-remaining": "0", "x-ratelimit-reset": "1700000180" }, 181000],
+  ["both headers", { "retry-after": "90", "x-ratelimit-remaining": "0", "x-ratelimit-reset": "1700000180" }, 181000],
+  ["invalid headers", { "retry-after": "invalid", "x-ratelimit-remaining": "0", "x-ratelimit-reset": "invalid" }, 60000],
+]) {
+  test(`rate-limit recovery honors ${name}`, async (t) => {
+    const fixture = setup(t, [new Response("Throttled", { status: 429, headers }), Response.json([])]);
+    await fixture.request("GET", "/repos/example/repo/labels");
+    assert.deepEqual(fixture.waits, [expectedWait]);
+    assert.equal(fixture.requests.length, 2);
+  });
+}
+
+test("primary-limit 403 is retried even without a secondary-limit message", async (t) => {
+  const fixture = setup(t, [
+    new Response("Forbidden", { status: 403, headers: { "x-ratelimit-remaining": "0", "x-ratelimit-reset": "1700000180" } }),
+    Response.json([]),
+  ]);
+  await fixture.request("GET", "/repos/example/repo/labels");
+  assert.deepEqual(fixture.waits, [181000]);
+});
+
+test("persistent rate limits stop after five retries with exponential backoff", async (t) => {
+  const fixture = setup(t, Array.from({ length: 6 }, () => new Response("Secondary rate limit", { status: 403 })));
+  await assert.rejects(fixture.request("GET", "/repos/example/repo/labels"), /403.*after 5 retries/);
+  assert.deepEqual(fixture.waits, [60000, 120000, 240000, 480000, 960000]);
+  assert.equal(fixture.requests.length, 6);
+});
+
+for (const status of [401, 403, 404, 422, 500]) {
+  test(`ordinary ${status} errors fail immediately without retrying writes`, async (t) => {
+    const fixture = setup(t, [new Response("Request failed", { status, headers: { "x-ratelimit-remaining": "100" } })]);
+    await assert.rejects(fixture.request("POST", "/repos/example/repo/labels", { name: "Bug" }), new RegExp(`${status}: Request failed`));
+    assert.equal(fixture.requests.length, 1);
+    assert.deepEqual(fixture.waits, []);
+  });
+}
+
+test("network failures are not retried because a write may already have been applied", async (t) => {
+  const fixture = setup(t, []);
+  let attempts = 0;
+  t.mock.method(globalThis, "fetch", async () => {
+    attempts += 1;
+    throw new Error("Connection lost");
+  });
+  await assert.rejects(fixture.request("POST", "/repos/example/repo/labels", { name: "Bug" }), /Connection lost/);
+  assert.equal(attempts, 1);
+  assert.deepEqual(fixture.waits, []);
+});

test/transfer-labels.test.mjs

@@ -6,6 +6,7 @@ import path from "node:path";
 import test from "node:test";
 import { promisify } from "node:util";
 import { transferLabels } from "../scripts/transfer-labels.mjs";
+import { createGithubRequest } from "../scripts/lib/github-request.mjs";
 
 const label = (name, color = "abcdef", description = "") => ({ name, color, description });
 
@@ -27,6 +28,7 @@ async function setup(t, {
   source = [label("Bug", "ff0000", "Source description"), label("New / 🚀", "123456", null)],
   target = [label("bug", "000000", "Keep this"), label("Extra")],
   failRequest = () => false,
+  responseForRequest = () => null,
   sourceId = 1,
   targetId = 2,
   archived = false,
@@ -42,17 +44,22 @@ async function setup(t, {
     else process.env.GITHUB_STEP_SUMMARY = previousSummary;
   });
   const requests = [];
+  const waits = [];
+  let time = 1_700_000_000_000;
   t.mock.method(globalThis, "fetch", async (url, options) => {
     const parsed = new URL(url);
     const request = {
       method: options.method,
       path: parsed.pathname,
       query: parsed.searchParams,
       body: options.body ? JSON.parse(options.body) : undefined,
+      time,
     };
     requests.push(request);
     assert.equal(parsed.origin, "https://api.github.com");
     assert.equal(options.headers.Authorization, "Bearer test-token");
+    const customResponse = responseForRequest(request);
+    if (customResponse) return customResponse;
     if (failRequest(request)) return new Response("Simulated failure", { status: 403 });
     if (request.method === "GET") {
       if (request.path === "/repos/example/source") {
@@ -73,11 +80,16 @@ async function setup(t, {
   });
   return {
     requests,
+    waits,
     writes: () => requests.filter(({ method }) => method !== "GET"),
     summary: () => fs.readFile(summaryPath, "utf8"),
     run: (options = {}) => transferLabels({
       token: "test-token", organization: "example",
       sourceRepository: "source", targetRepository: "target", ...options,
+      githubRequest: createGithubRequest("test-token", {
+        now: () => time,
+        sleep: async (milliseconds) => { waits.push(milliseconds); time += milliseconds; },
+      }),
     }),
   };
 }
@@ -118,6 +130,7 @@ for (const overrideExisting of [false, true]) {
     const fixture = await setup(t);
     await fixture.run({ dryRun: true, overrideExisting });
     assert.deepEqual(fixture.writes(), []);
+    assert.deepEqual(fixture.waits, []);
     const summary = await fixture.summary();
     assert.match(summary, /# Transfer-Labels Fake Changelog/);
     assert.match(summary, /\*\*Test Mode:\*\* True/);
@@ -139,6 +152,60 @@ test("reads every page from both repositories before transferring", async (t) =>
   assert.match(await fixture.summary(), /\*\*Source Labels:\*\* 101/);
 });
 
+test("a 233-label transfer pauses on secondary limits and reports each copied label once", async (t) => {
+  let throttled = false;
+  const fixture = await setup(t, {
+    source: Array.from({ length: 233 }, (_, index) => label(`Label ${index}`)),
+    target: [],
+    responseForRequest: ({ method, body }) => {
+      if (method === "POST" && body.name === "Label 120" && !throttled) {
+        throttled = true;
+        return Response.json({ message: "You have exceeded a secondary rate limit." }, { status: 403 });
+      }
+      return null;
+    },
+  });
+  const result = await fixture.run();
+  assert.equal(result.createdLabels.length, 233);
+  const writes = fixture.writes();
+  assert.equal(writes.length, 234);
+  assert.equal(new Set(writes.map(({ body }) => body.name)).size, 233);
+  for (let index = 1; index < writes.length; index += 1) {
+    assert.ok(writes[index].time - writes[index - 1].time >= 1000, "Every write must be paced");
+  }
+  assert.deepEqual(fixture.waits.filter((milliseconds) => milliseconds >= 60000), [60000]);
+  const summary = await fixture.summary();
+  assert.match(summary, /\*\*Created Labels:\*\* 233/);
+  assert.doesNotMatch(summary, /## Workflow Failure/);
+  assert.equal((summary.match(/Created `Label 120`/g) ?? []).length, 1);
+});
+
+test("rerunning a partial 233-label transfer only creates the remaining labels", async (t) => {
+  const source = Array.from({ length: 233 }, (_, index) => label(`Label ${index}`));
+  const fixture = await setup(t, { source, target: source.slice(0, 150) });
+  await fixture.run();
+  assert.equal(fixture.writes().length, 83);
+  assert.ok(fixture.writes().every(({ method, body }) => method === "POST" && Number(body.name.slice(6)) >= 150));
+  assert.match(await fixture.summary(), /\*\*Created Labels:\*\* 83/);
+});
+
+test("exhausted rate-limit retries preserve the partial changelog and prevent deletions", async (t) => {
+  const fixture = await setup(t, {
+    source: [label("First"), label("Blocked")],
+    target: [label("Extra")],
+    responseForRequest: ({ method, body }) => method === "POST" && body.name === "Blocked"
+      ? new Response("Secondary rate limit", { status: 403 }) : null,
+  });
+  await assert.rejects(fixture.run({ overrideExisting: true }), /after 5 retries/);
+  assert.equal(fixture.writes().length, 7);
+  assert.ok(fixture.writes().every(({ method }) => method === "POST"));
+  const summary = await fixture.summary();
+  assert.match(summary, /\*\*Created Labels:\*\* 1/);
+  assert.match(summary, /\*\*Deleted Labels:\*\* 0/);
+  assert.match(summary, /## Workflow Failure/);
+  assert.doesNotMatch(summary, /Created `Blocked`/);
+});
+
 test("override is a no-op when the receiving repository already matches", async (t) => {
   const fixture = await setup(t, { source: [label("Bug")], target: [label("Bug")] });
   await fixture.run({ overrideExisting: true });