diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 701f6c1d0..a05de5db4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,37 +186,17 @@ jobs: # Three legs that partition the suite into disjoint sets (check names # `Test (base)`, `Test (storage-azure)`, `Test (storage-minio)` are # required under branch protection): - # - base: the full suite EXCEPT the storage manifest - # (vitest.base.config.ts) — the bulk, run once. + # - base: the full suite EXCEPT the storage manifest — the + # bulk, run once. # - storage-azure: the storage manifest against Azurite. # - storage-minio: the storage manifest against MinIO. - # base ∪ storage-* == full suite; the storage manifest is the single - # source of truth for both the base `exclude` and the storage `include`. - # Each `test:storage-*` script sets STORAGE_PROVIDER itself, which picks - # the testcontainer in globalSetup.ts. - include: - - leg: base - test_script: test:base - coverage_artifact: coverage-report-base - - leg: storage-azure - test_script: test:storage-azure - coverage_artifact: coverage-report-storage-azure - - leg: storage-minio - test_script: test:storage-minio - coverage_artifact: coverage-report-storage-minio - env: - # Provider-specific vars below satisfy the conditional validation in - # apps/api/src/config/environment.ts. STORAGE_PROVIDER itself is set by - # each test script (azure_blob_storage for test:base and - # test:storage-azure, minio for test:storage-minio); base sets it so its - # app.ready() boot never depends on the storage container starting. - # Real connection details come from the testcontainer. - AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1 - AZURE_STORAGE_CONTAINER_NAME: test-files - MINIO_ENDPOINT: http://localhost:9000 - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin - MINIO_BUCKET: test-files + # `leg` is the Vitest project name in apps/api/vitest.config.ts + # (test.projects); base ∪ storage-* == full suite. The storage manifest + # is the single source of truth for both the base `exclude` and the + # storage `include`. Each project selects its provider (and the + # matching STORAGE_PROVIDER in its test.env) from its name in + # globalSetup.ts — no env is needed here. + leg: [base, storage-azure, storage-minio] # The storage legs are quick (18 files each); the base leg is the long pole. # Gating every step is where docs-only PRs save the most. All three legs # still report their required "Test (...)" checks. @@ -242,25 +222,35 @@ jobs: if: needs.changes.outputs.code == 'true' - name: Run tests if: needs.changes.outputs.code == 'true' - # Indirection via env (never interpolate matrix values into `run:` - # directly — that is a template-injection finding for zizmor). + # Runs one Vitest project and emits a blob report (coverage embedded). + # LEG is passed via env (never interpolate matrix values into `run:` + # directly — that is a template-injection finding for zizmor); the blob + # filename embeds LEG so the three legs never collide in the coverage job. env: - TEST_SCRIPT: ${{ matrix.test_script }} - run: pnpm "$TEST_SCRIPT" + LEG: ${{ matrix.leg }} + run: pnpm test:ci - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() && needs.changes.outputs.code == 'true' with: - name: ${{ matrix.coverage_artifact }} - path: apps/api/coverage/ + name: blob-report-${{ matrix.leg }} + path: apps/api/.vitest-reports/ + # `.vitest-reports` is a dotfile dir; without this, upload-artifact + # (default include-hidden-files: false) skips its contents and the + # coverage job's download finds nothing. + include-hidden-files: true retention-days: 7 # Enforces the apps/api coverage gate. Unlike the other heavy jobs, this one - # `needs: test` because it consumes the three legs' coverage artifacts — the - # legs partition the suite, so coverage must be merged across them before any - # threshold means anything (a single leg never sees the whole codebase). When - # a test leg fails there is nothing to gate and the PR is already blocked, so - # this job is simply skipped. On docs-only PRs every step is gated off (like - # the other jobs) and the required "Coverage" check still reports success. + # `needs: test` because it consumes the three legs' blob reports — the legs + # partition the suite, so coverage must be merged across them before any + # threshold means anything (a single leg never sees the whole codebase). Vitest + # merges the blobs natively (`--merge-reports`) and applies the gate + # (90% for lines, statements, functions, and branches, passed by flag in the + # test:coverage:merge script); there is no external merge script. When a test + # leg fails there is nothing to gate + # and the PR is already blocked, so this job is simply skipped. On docs-only PRs + # every step is gated off (like the other jobs) and the required "Coverage" + # check still reports success. coverage: needs: [check-draft, changes, test] name: Coverage @@ -279,16 +269,29 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile if: needs.changes.outputs.code == 'true' - # Pull every leg's coverage artifact (coverage-report-base, - # coverage-report-storage-azure, coverage-report-storage-minio) into a - # dir each, so check-coverage.mjs can merge them. + # Pull every leg's blob report into ONE flat dir. `merge-multiple: true` is + # required: `vitest --merge-reports` reads the dir non-recursively and + # rejects subfolders. Each leg's blob is uniquely named (blob-.json) + # so they never collide. - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 if: needs.changes.outputs.code == 'true' with: - pattern: coverage-report-* - path: coverage-artifacts - - run: node scripts/check-coverage.mjs coverage-artifacts - if: needs.changes.outputs.code == 'true' + pattern: blob-report-* + path: apps/api/.vitest-reports + merge-multiple: true + # Native Vitest merge + coverage + gate (90% all metrics, replaces check-coverage.mjs). + - run: pnpm test:coverage:merge + if: needs.changes.outputs.code == 'true' + # Upload the human-readable merged report (html + lcov + json) this step + # already produces. `always()` so it is available precisely when the gate + # FAILS — that is when a contributor most needs to see which lines are + # missing, without having to fetch and merge the three opaque blobs by hand. + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() && needs.changes.outputs.code == 'true' + with: + name: coverage-report-merged + path: apps/api/coverage/ + retention-days: 7 build: needs: [check-draft, changes] diff --git a/.gitignore b/.gitignore index 856c0be5b..b8b547c78 100644 --- a/.gitignore +++ b/.gitignore @@ -77,7 +77,7 @@ Thumbs.db # Test coverage coverage/ -coverage-artifacts/ +.vitest-reports/ *.lcov .nyc_output/ vitest-report/ diff --git a/apps/api/package.json b/apps/api/package.json index 03eb29edd..7ef4c1fab 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -14,10 +14,9 @@ "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json", "start": "node dist/server.js", "test": "vitest run", - "test:base": "STORAGE_PROVIDER=azure_blob_storage vitest run --config vitest.base.config.ts", - "test:storage-azure": "STORAGE_PROVIDER=azure_blob_storage vitest run --config vitest.storage.config.ts", - "test:storage-minio": "STORAGE_PROVIDER=minio vitest run --config vitest.storage.config.ts", - "test:coverage": "COVERAGE_DIR=coverage-artifacts/base pnpm run test:base && COVERAGE_DIR=coverage-artifacts/storage-azure pnpm run test:storage-azure && COVERAGE_DIR=coverage-artifacts/storage-minio pnpm run test:storage-minio && node ../../scripts/check-coverage.mjs coverage-artifacts", + "test:coverage": "vitest run --coverage", + "test:ci": ": \"${LEG:?LEG must be set (base|storage-azure|storage-minio)}\" && vitest run --project=\"$LEG\" --coverage --reporter=blob --outputFile.blob=.vitest-reports/blob-$LEG.json --coverage.thresholds.lines=0 --coverage.thresholds.statements=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0", + "test:coverage:merge": "vitest run --merge-reports=.vitest-reports --coverage", "test:verify-storage-manifest": "tsx test/setup/assertStorageTestManifest.ts", "test:ui": "vitest --ui", "lint": "eslint .", diff --git a/apps/api/test/factories/appFactory.ts b/apps/api/test/factories/appFactory.ts index de0176408..7d87d76d3 100644 --- a/apps/api/test/factories/appFactory.ts +++ b/apps/api/test/factories/appFactory.ts @@ -52,10 +52,41 @@ async function buildTestAdapter( } } +/** + * Points the app's storage config at the running testcontainer BEFORE it boots. + * + * The MinIO endpoint is a dynamic testcontainer port, only known at runtime, so + * it cannot live in the static per-project `test.env` (which carries a localhost + * placeholder just to satisfy validation). We set it here, before `app.ready()`, + * so `buildStorageConfig()` — read by both the storage plugin and the + * storage-relay plugin at registration — sees the real endpoint. Azure needs + * nothing: its adapter uses the injected `connectionString`, and + * `buildStorageConfig()` only requires `AZURE_STORAGE_ACCOUNT_NAME`, which + * `test.env` already provides. + */ +function applyStorageEnvFromDescriptor( + descriptor: TestStorageDescriptor +): void { + if (descriptor.provider === StorageProvider.MINIO) { + process.env.MINIO_ENDPOINT = descriptor.endpoint; + process.env.MINIO_ACCESS_KEY = descriptor.accessKey; + process.env.MINIO_SECRET_KEY = descriptor.secretKey; + process.env.MINIO_BUCKET = descriptor.bucket; + process.env.MINIO_REGION = descriptor.region; + } +} + export async function createTestApp( databaseUrl: string, options?: CreateTestAppOptions ): Promise { + const descriptor = options?.storageDescriptor; + + // Set the real testcontainer storage endpoint before the app boots, so the + // storage + relay plugins read it at `app.ready()` (see the helper's note). + // `null`/`undefined` = storage-agnostic test; the dummy `test.env` suffices. + if (descriptor) applyStorageEnvFromDescriptor(descriptor); + const app = await createApp(false, { skipUnderPressure: true }); app.log.level = "debug"; @@ -71,18 +102,17 @@ export async function createTestApp( // storagePlugin runs during ready() and would overwrite any earlier assignment. await app.ready(); - const descriptor = options?.storageDescriptor; - if (descriptor === null) { - // The test explicitly requested storage (`storageDescriptor: - // inject("storageDescriptor")`) but the storage testcontainer failed to - // start, so globalSetup provided `null`. Fail early with a clear reason - // instead of a confusing adapter error deeper in the test. + // globalSetup provides `null` only for the container-less `base` project, so + // reaching here means a `base` test requested storage (`storageDescriptor: + // inject("storageDescriptor")`). Such a test belongs in the storage manifest + // so it runs under a storage-* project. Fail early with a clear reason + // instead of a confusing adapter error deeper in the test. (A storage + // container that fails to start now fails its project fast at globalSetup.) throw new Error( - "createTestApp received `storageDescriptor: null` — the storage " + - "testcontainer failed to start (see the globalSetup warning above). " + - "This test requires real storage. Ensure Docker is available and the " + - "storage testcontainer starts successfully." + "createTestApp received `storageDescriptor: null`. Only the container-less " + + "`base` project provides null — a test that needs real storage must be " + + "listed in the storage manifest so it runs under a storage-* project." ); } diff --git a/apps/api/test/setup/globalSetup.ts b/apps/api/test/setup/globalSetup.ts index 8be72aea0..c49d57e5b 100644 --- a/apps/api/test/setup/globalSetup.ts +++ b/apps/api/test/setup/globalSetup.ts @@ -12,66 +12,61 @@ import type { TestProject } from "vitest/node"; import { StorageProvider } from "@repo/storage"; /** - * Sets process.env for the chosen storage provider before workers are spawned, - * so the storage plugin's `buildStorageConfig()` validation passes at - * `app.ready()` in each worker. + * Maps a Vitest project name to the storage provider its tests run against. * - * Connection details are dummies — real values are injected per-app in - * `createTestApp`. The storagePlugin will still construct an adapter at boot - * (its background health check will warn), but `createTestApp` then overrides - * `app.storage` with the testcontainer-backed adapter. + * The provider selection travels via the project NAME (set in vitest.config.ts), + * not `process.env` — globalSetup runs once per project in the main process, so + * a shared `process.env.STORAGE_PROVIDER` would be last-writer-wins across the + * three projects when they run in a single command. Each project also declares + * the matching `STORAGE_PROVIDER` in its `test.env`, which Vitest applies inside + * the worker so `buildStorageConfig()` validation passes at `app.ready()`. + * + * `base` runs the storage-independent suite and boots no storage container. */ -function applyStorageEnv(descriptor: TestStorageDescriptor): void { - process.env.STORAGE_PROVIDER = descriptor.provider; - if (descriptor.provider === StorageProvider.AZURE_BLOB_STORAGE) { - process.env.AZURE_STORAGE_ACCOUNT_NAME ??= "devstoreaccount1"; - process.env.AZURE_STORAGE_CONTAINER_NAME = descriptor.containerName; - } - if (descriptor.provider === StorageProvider.MINIO) { - process.env.MINIO_ENDPOINT = descriptor.endpoint; - process.env.MINIO_ACCESS_KEY = descriptor.accessKey; - process.env.MINIO_SECRET_KEY = descriptor.secretKey; - process.env.MINIO_BUCKET = descriptor.bucket; - process.env.MINIO_REGION = descriptor.region; +function storageProviderForProject( + name: string | undefined +): StorageProvider | null { + switch (name) { + case "storage-azure": + return StorageProvider.AZURE_BLOB_STORAGE; + case "storage-minio": + return StorageProvider.MINIO; + case "base": + return null; + default: + throw new Error( + `Unknown Vitest project "${String(name)}" — expected "base", ` + + `"storage-azure", or "storage-minio". globalSetup selects the storage ` + + `provider from the project name (see vitest.config.ts).` + ); } } export default async function setup(project: TestProject) { - // The chatbot is opt-in (CHATBOT_ENABLED defaults off). Enable it for the - // suite so its routes register and the chatbot integration tests run - // (LLM_PROVIDER defaults to "mock"). Set before workers spawn so each worker - // inherits it — same mechanism as the storage env below. - process.env.CHATBOT_ENABLED = "true"; - // Database is required for all tests — let it propagate and fail fast. const { databaseUrl, container: dbContainer } = await setupTestDatabase(); - // Storage is best-effort: only the storage-manifest tests (the storage-* - // legs) need the testcontainer. If it fails to start (wrong Node.js version, - // missing Docker image, CI network issue, etc.) we still want the - // storage-independent tests to run. - // - // This invariant holds because each test script sets STORAGE_PROVIDER itself - // (base leg = azure_blob_storage), so `buildStorageConfig()` at `app.ready()` - // clears its "STORAGE_PROVIDER is required" check without the container. The - // remaining provider-required var (AZURE_STORAGE_ACCOUNT_NAME) comes from the - // CI job env, so boot succeeds in CI even when the container is down. Locally - // that var is only injected by `applyStorageEnv` on the happy path, so a - // failed container there still breaks boot unless it is exported in the shell. + // Storage is per-project: only the storage projects boot a testcontainer, each + // against its own provider (picked from the project name, so the three + // projects never contend on a shared process.env). The `base` project skips it + // entirely and provides `null`. If the container fails to start in a storage + // project we fail fast — every file there needs it, so there is nothing to fall + // back to (unlike the old shared run, base is its own container-less project). + const provider = storageProviderForProject(project.name); let storageDescriptor: TestStorageDescriptor | null = null; let storageContainer: TestStorageContainer | null = null; - try { - const storage = await setupTestStorage(); - storageDescriptor = storage.descriptor; - storageContainer = storage.container; - applyStorageEnv(storageDescriptor); - } catch (error) { - // eslint-disable-next-line no-console - console.warn( - "\n⚠️ Storage testcontainer failed to start — storage-dependent tests will fail.\n", - error - ); + if (provider) { + try { + const storage = await setupTestStorage(provider); + storageDescriptor = storage.descriptor; + storageContainer = storage.container; + } catch (error) { + // Stop the DB container already started above, but never let its shutdown + // failure mask the real storage error. + await dbContainer.stop().catch(() => {}); + throw error; + } } // Provide values BEFORE migrations/seeds so that an error in the next block @@ -97,7 +92,10 @@ export default async function setup(project: TestProject) { declare module "vitest" { export interface ProvidedContext { databaseUrl: string; - /** `null` when the storage testcontainer failed to start. */ + /** + * `null` only for the `base` project, which boots no storage container. The + * storage projects either provide a descriptor or fail fast at setup. + */ storageDescriptor: TestStorageDescriptor | null; } } diff --git a/apps/api/test/setup/storageTestManifest.ts b/apps/api/test/setup/storageTestManifest.ts index 028638af9..5e0e9651d 100644 --- a/apps/api/test/setup/storageTestManifest.ts +++ b/apps/api/test/setup/storageTestManifest.ts @@ -5,10 +5,10 @@ * * Why this list exists: * - Both storage CI legs (storage-azure and storage-minio) run ONLY these files - * against their provider (see `vitest.storage.config.ts`); the base leg - * EXCLUDES them (it runs the full suite except this manifest). Together they - * prove the storage layer works against both providers without paying to run - * every test file twice. + * against their provider (the `storage-*` projects in `vitest.config.ts`); the + * base project EXCLUDES them (it runs the full suite except this manifest). + * Together they prove the storage layer works against both providers without + * paying to run every test file twice. * - `test:verify-storage-manifest` (test/setup/assertStorageTestManifest.ts) * keeps this list honest: CI fails if a test touches storage but is missing * here, if an entry no longer exists on disk, or if an entry no longer shows diff --git a/apps/api/test/setup/testStorage.ts b/apps/api/test/setup/testStorage.ts index 20f01d886..ec97e57d8 100644 --- a/apps/api/test/setup/testStorage.ts +++ b/apps/api/test/setup/testStorage.ts @@ -109,17 +109,15 @@ async function setupMinioTestStorage(): Promise<{ } /** - * Starts the storage testcontainer matching `STORAGE_PROVIDER`. - * Defaults to Azure Blob (Azurite) when the env var is unset, to preserve the - * existing developer workflow. + * Starts the storage testcontainer for the given provider. The provider is + * chosen by the caller (globalSetup, from the Vitest project name) rather than + * read from `process.env`, so the three projects can boot different providers in + * a single run without contending on a shared env var. */ -export async function setupTestStorage(): Promise<{ +export async function setupTestStorage(provider: StorageProvider): Promise<{ descriptor: TestStorageDescriptor; container: TestStorageContainer; }> { - const provider = (process.env.STORAGE_PROVIDER ?? - StorageProvider.AZURE_BLOB_STORAGE) as StorageProvider; - if (provider === StorageProvider.MINIO) { return setupMinioTestStorage(); } @@ -127,6 +125,6 @@ export async function setupTestStorage(): Promise<{ return setupAzureTestStorage(); } throw new Error( - `Invalid STORAGE_PROVIDER for tests: "${String(provider)}". Expected ${Object.values(StorageProvider).join(" or ")}.` + `Invalid storage provider for tests: "${String(provider)}". Expected ${Object.values(StorageProvider).join(" or ")}.` ); } diff --git a/apps/api/tsconfig.eslint.json b/apps/api/tsconfig.eslint.json index dcb02252e..47044a777 100644 --- a/apps/api/tsconfig.eslint.json +++ b/apps/api/tsconfig.eslint.json @@ -1,14 +1,6 @@ { "extends": "./tsconfig.json", - "include": [ - "src", - "test", - "eslint.config.ts", - "vitest.config.ts", - "vitest.shared.ts", - "vitest.base.config.ts", - "vitest.storage.config.ts" - ], + "include": ["src", "test", "eslint.config.ts", "vitest.config.ts"], "compilerOptions": { "noEmit": true } diff --git a/apps/api/vitest.base.config.ts b/apps/api/vitest.base.config.ts deleted file mode 100644 index 5e0345ea5..000000000 --- a/apps/api/vitest.base.config.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { defineApiVitestConfig } from "./vitest.shared.js"; -import { STORAGE_TEST_MANIFEST } from "./test/setup/storageTestManifest.js"; - -// Base config: runs the full apps/api suite EXCEPT the storage-dependent files -// (test/setup/storageTestManifest.ts). Used by the `test:base` CI leg. Those -// storage files are covered separately, once per provider, by -// `test:storage-azure` / `test:storage-minio`, so excluding them here keeps the -// three legs a disjoint partition of the suite (base ∪ storage == full suite). -// -// The same manifest drives both this exclude and the storage config's include, -// so there is one source of truth; `test:verify-storage-manifest` guards it -// against drift. -export default defineApiVitestConfig({ - exclude: [...STORAGE_TEST_MANIFEST], -}); diff --git a/apps/api/vitest.config.ts b/apps/api/vitest.config.ts index 7bae38c95..e6d8e46e8 100644 --- a/apps/api/vitest.config.ts +++ b/apps/api/vitest.config.ts @@ -1,6 +1,226 @@ -import { defineApiVitestConfig } from "./vitest.shared.js"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { configDefaults, defineConfig, defineProject } from "vitest/config"; +import tsconfigPaths from "vite-tsconfig-paths"; +import { STORAGE_TEST_MANIFEST } from "./test/setup/storageTestManifest.js"; -// Default config: runs the full apps/api test suite (every file, including the -// storage-dependent ones). This is what bare `vitest` / `vitest --ui` load; the -// segmented CI legs use vitest.base.config.ts + vitest.storage.config.ts. -export default defineApiVitestConfig(); +// __dirname in ESM +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +/** Default test glob — the full apps/api suite. */ +const DEFAULT_TEST_INCLUDE = ["test/**/*.{test,spec}.{js,ts}"]; + +/** + * The apps/api coverage gate — 90% for every metric — declared here, where it is + * read, as the single source of truth. It applies to any full run (all three + * projects, or the merged blobs). The ONLY run that opts out is `test:ci`, which + * runs ONE project whose partial coverage would never clear 90; it overrides + * these to 0 on the CLI (see package.json). + */ +const COVERAGE_GATE = 90; + +/** + * Environment shared by every project. The storage-provider vars are added per + * project (see `storageEnv`), because each project targets a different provider. + * `test.env` is applied inside each worker, so it isolates the three projects + * cleanly even when they run in a single command. + */ +const SHARED_TEST_ENV = { + NODE_ENV: "test", + AUTH_PROVIDER: "forced-user", // Set AUTH_PROVIDER for all tests + FORCED_USER_IDP_ID: "test-user-idp-id", + FORCED_USER_EMAIL: "me@test.com", + LOCAL_BYPASS_REQUIRED_FIELDS: "false", + LLM_PROVIDER: "mock", + COOKIE_SECRET: "test-cookie-secret-do-not-use-in-prod", + // The chatbot is opt-in (CHATBOT_ENABLED defaults off). Enable it for the + // suite so its routes register and the chatbot integration tests run + // (LLM_PROVIDER defaults to "mock"). + CHATBOT_ENABLED: "true", +} as const; + +// Dummy storage env per project. These only satisfy `buildStorageConfig()` +// validation at `app.ready()`; the real adapter is built from the injected +// `storageDescriptor` (see test/factories/appFactory.ts). globalSetup.ts picks +// the provider — and boots the matching testcontainer — from the project NAME, +// not from these vars, so the three projects stay isolated even in a single run. +const AZURE_TEST_ENV = { + STORAGE_PROVIDER: "azure_blob_storage", + AZURE_STORAGE_ACCOUNT_NAME: "devstoreaccount1", + AZURE_STORAGE_CONTAINER_NAME: "test-files", +}; + +const MINIO_TEST_ENV = { + STORAGE_PROVIDER: "minio", + MINIO_ENDPOINT: "http://localhost:9000", + MINIO_ACCESS_KEY: "minioadmin", + MINIO_SECRET_KEY: "minioadmin", + MINIO_BUCKET: "test-files", + MINIO_REGION: "us-east-1", +}; + +interface ApiVitestProjectOverrides { + /** + * Project name. MUST be one of the names `globalSetup.ts` maps to a storage + * provider (`base`, `storage-azure`, `storage-minio`): it drives `--project` + * filtering in CI, the provider selection in globalSetup, AND the branch- + * protection check name. Do not rename without updating all three. + */ + name: string; + /** `test.include` override; defaults to the full suite. */ + include?: string[]; + /** Extra `test.exclude` globs, appended to Vitest's defaults. */ + exclude?: string[]; + /** Provider-selection dummies for `buildStorageConfig()` validation. */ + storageEnv: Record; +} + +/** + * Builds one apps/api Vitest project. Every project shares everything except its + * name, its `include`/`exclude`, and its storage-provider env. `defineProject` + * types the returned object so a stray or root-only key fails HERE at the + * definition, not silently at runtime — root-only options (coverage, reporters, + * outputFile, teardownTimeout) are set once on the root `test` below, never here. + */ +function defineApiVitestProject(overrides: ApiVitestProjectOverrides) { + return defineProject({ + plugins: [tsconfigPaths({ root: __dirname })], + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + "@test": path.resolve(__dirname, "./test"), + }, + }, + test: { + name: overrides.name, + globals: true, + environment: "node", + include: overrides.include ?? DEFAULT_TEST_INCLUDE, + // Append to (never replace) Vitest's defaults — dropping + // `configDefaults.exclude` would let node_modules/dist leak in. + exclude: [...configDefaults.exclude, ...(overrides.exclude ?? [])], + testTimeout: 30000, + hookTimeout: 30000, + pool: "threads", + // PROTOTYPE: per-file database isolation (test/setup/perFileDatabase.ts) + // gives every file its own cloned DB, so files can now run in parallel + // safely. Set to 4 to match the 4 vCPUs on GitHub's ubuntu-latest runners. + // Benchmarked against Postgres contention: 2→4 workers cut local wall-clock + // ~43% (90s→52s) with zero failures / no flakiness across all 150 files. + maxWorkers: 4, + fileParallelism: true, + globalSetup: ["./test/setup/globalSetup.ts"], + // Runs once per test file, in the worker, before the file's own hooks — + // clones a private database from the seeded template for that file. + setupFiles: ["./test/setup/perFileDatabase.ts"], + logHeapUsage: true, + server: { + deps: { + inline: [ + "@fastify/cors", + "@fastify/jwt", + "@fastify/swagger", + "@fastify/swagger-ui", + "@fastify/under-pressure", + "@fastify/rate-limit", + "@fastify/multipart", + "@fastify/autoload", + "@fastify/helmet", + ], + }, + }, + env: { ...SHARED_TEST_ENV, ...overrides.storageEnv }, + }, + }); +} + +// One config, three projects that partition the suite into disjoint sets: +// - base: the full suite EXCEPT the storage manifest — the bulk. +// - storage-azure: ONLY the storage manifest, against Azurite. +// - storage-minio: ONLY the storage manifest, against MinIO. +// base ∪ storage-* == the full suite. The storage manifest +// (test/setup/storageTestManifest.ts) is the single source of truth for both the +// base `exclude` and the storage `include`; `test:verify-storage-manifest` guards +// it against drift. +// +// Coverage, reporters, outputFile, and teardownTimeout are root-only (Vitest's +// `NonProjectOptions`): ignored inside a project, so they live here once for the +// whole run. With a single reporter run, one HTML path can't clobber across +// projects. +// +// The coverage gate (COVERAGE_GATE below) lives in `test.coverage.thresholds`, +// applied to any full run. A per-project run can't be gated on its partial view, +// so `test:ci` (one `--project`) overrides the thresholds to 0 on the CLI — the +// single opt-out, stated where it happens. Two run modes, same coverage by +// construction: +// - Local: `vitest run --coverage` runs all three projects and merges their +// coverage (v8 hit-counts) in one pass, then applies the gate — one command. +// - CI: each leg runs `--project= --coverage --reporter=blob` (gate +// off); the `coverage` job merges the blobs with `--merge-reports --coverage` +// and applies the gate. Keeps the matrix's wall-clock; still no script. +export default defineConfig({ + test: { + // Multiple reporters for better visibility. The CI legs override this with + // `--reporter=blob` on the CLI so the coverage job can merge the results. + reporters: process.env.CI ? ["default", "html"] : ["verbose", "html"], + // Kept outside ./coverage to avoid clobbering the coverage report directory + // under vitest 4.1+, which refuses to copy the report into a subdirectory of + // itself. + outputFile: { html: "./vitest-report/index.html" }, + teardownTimeout: 10000, + projects: [ + defineApiVitestProject({ + name: "base", + exclude: [...STORAGE_TEST_MANIFEST], + storageEnv: AZURE_TEST_ENV, + }), + defineApiVitestProject({ + name: "storage-azure", + include: [...STORAGE_TEST_MANIFEST], + storageEnv: AZURE_TEST_ENV, + }), + defineApiVitestProject({ + name: "storage-minio", + include: [...STORAGE_TEST_MANIFEST], + storageEnv: MINIO_TEST_ENV, + }), + ], + coverage: { + enabled: true, + provider: "v8", + // Multiple reporters for comprehensive coverage view + reporter: [ + "text", + "text-summary", + "json", + "json-summary", + "html", + "lcov", + ], + include: ["src/**/*.{js,ts}"], + exclude: [ + "node_modules/", + "test/", + "**/*.test.ts", + "**/*.spec.ts", + "**/*.example.ts", + "**/types/**", + "**/*.d.ts", + "**/dist/**", + "**/*.config.{js,ts}", + "**/server.ts", // Entry point, often hard to test + ], + // The gate. `test:ci` (single-project, partial view) overrides these to 0. + thresholds: { + lines: COVERAGE_GATE, + functions: COVERAGE_GATE, + branches: COVERAGE_GATE, + statements: COVERAGE_GATE, + }, + reportsDirectory: "./coverage", + clean: true, + cleanOnRerun: true, + }, + }, +}); diff --git a/apps/api/vitest.shared.ts b/apps/api/vitest.shared.ts deleted file mode 100644 index c2a97e3be..000000000 --- a/apps/api/vitest.shared.ts +++ /dev/null @@ -1,155 +0,0 @@ -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { configDefaults, defineConfig } from "vitest/config"; -import tsconfigPaths from "vite-tsconfig-paths"; - -// Obtener __dirname en ESM -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -// Coverage thresholds are enforced in CI, not per Vitest run. The suite is split -// into three disjoint legs (base + one per storage provider; see the `test` job -// in .github/workflows/ci.yml), so no single run sees the whole codebase and a -// per-run threshold would fail on the files that run never touches. The real -// gate is the `coverage` CI job, which merges all three legs' coverage and -// checks the union against per-metric thresholds (90% for lines, statements, -// functions, and branches; see scripts/check-coverage.mjs). These zeros keep -// each run's numbers visible in its own report without gating on a partial view. -const coverageThresholds = { - lines: 0, - functions: 0, - branches: 0, - statements: 0, -}; - -/** Default test glob — the full apps/api suite. */ -const DEFAULT_TEST_INCLUDE = ["test/**/*.{test,spec}.{js,ts}"]; - -export interface ApiVitestConfigOverrides { - /** - * Overrides `test.include`. Defaults to the full suite. The storage-only - * config (`vitest.storage.config.ts`) passes the storage manifest here so the - * `test:storage-*` legs run just the storage-dependent files. - */ - include?: string[]; - /** - * Extra `test.exclude` globs, appended to Vitest's defaults (never replacing - * them — dropping `configDefaults.exclude` would let node_modules/dist leak - * in). The base config (`vitest.base.config.ts`) passes the storage manifest - * here so `test:base` runs everything EXCEPT the storage-dependent files. - */ - exclude?: string[]; -} - -/** - * Single source of truth for the apps/api Vitest config. Every runnable config - * (`vitest.config.ts` full suite, `vitest.base.config.ts` no-storage, - * `vitest.storage.config.ts` storage manifest only) calls this, so everything - * except `test.include` / `test.exclude` stays identical across runs. - */ -export function defineApiVitestConfig( - overrides: ApiVitestConfigOverrides = {} -) { - return defineConfig({ - plugins: [ - tsconfigPaths({ - root: __dirname, - }), - ], - resolve: { - alias: { - "@": path.resolve(__dirname, "./src"), - "@test": path.resolve(__dirname, "./test"), - }, - }, - test: { - globals: true, - environment: "node", - // Multiple reporters for better visibility - reporters: process.env.CI ? ["default", "html"] : ["verbose", "html"], - include: overrides.include ?? DEFAULT_TEST_INCLUDE, - exclude: [...configDefaults.exclude, ...(overrides.exclude ?? [])], - testTimeout: 30000, - hookTimeout: 30000, - teardownTimeout: 10000, - pool: "threads", - // PROTOTYPE: per-file database isolation (test/setup/perFileDatabase.ts) - // gives every file its own cloned DB, so files can now run in parallel - // safely. Set to 4 to match the 4 vCPUs on GitHub's ubuntu-latest runners. - // Benchmarked against Postgres contention: 2→4 workers cut local wall-clock - // ~43% (90s→52s) with zero failures / no flakiness across all 150 files. - maxWorkers: 4, - fileParallelism: true, - globalSetup: ["./test/setup/globalSetup.ts"], - // Runs once per test file, in the worker, before the file's own hooks — - // clones a private database from the seeded template for that file. - setupFiles: ["./test/setup/perFileDatabase.ts"], - // Better logging for UI - logHeapUsage: true, - // Detailed output (kept outside ./coverage to avoid clobbering the - // coverage report directory under vitest 4.1+, which now refuses to - // copy the report into a subdirectory of itself). - outputFile: { - html: "./vitest-report/index.html", - }, - server: { - deps: { - inline: [ - "@fastify/cors", - "@fastify/jwt", - "@fastify/swagger", - "@fastify/swagger-ui", - "@fastify/under-pressure", - "@fastify/rate-limit", - "@fastify/multipart", - "@fastify/autoload", - "@fastify/helmet", - ], - }, - }, - coverage: { - enabled: true, - provider: "v8", - // Multiple reporters for comprehensive coverage view - reporter: [ - "text", - "text-summary", - "json", - "json-summary", - "html", - "lcov", - ], - include: ["src/**/*.{js,ts}"], - exclude: [ - "node_modules/", - "test/", - "**/*.test.ts", - "**/*.spec.ts", - "**/*.example.ts", - "**/types/**", - "**/*.d.ts", - "**/dist/**", - "**/*.config.{js,ts}", - "**/server.ts", // Entry point, often hard to test - ], - // Coverage thresholds - will show in UI - thresholds: coverageThresholds, - // More detailed reporting. Defaults to ./coverage; overridable via - // COVERAGE_DIR so the `test:coverage` script can point each leg at its - // own directory and merge them (mirrors the CI `coverage` job). - reportsDirectory: process.env.COVERAGE_DIR ?? "./coverage", - clean: true, - cleanOnRerun: true, - }, - env: { - NODE_ENV: "test", - AUTH_PROVIDER: "forced-user", // Set AUTH_PROVIDER for all tests - FORCED_USER_IDP_ID: "test-user-idp-id", - FORCED_USER_EMAIL: "me@test.com", - LOCAL_BYPASS_REQUIRED_FIELDS: "false", - LLM_PROVIDER: "mock", - COOKIE_SECRET: "test-cookie-secret-do-not-use-in-prod", - }, - }, - }); -} diff --git a/apps/api/vitest.storage.config.ts b/apps/api/vitest.storage.config.ts deleted file mode 100644 index 76dab2cd3..000000000 --- a/apps/api/vitest.storage.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { defineApiVitestConfig } from "./vitest.shared.js"; -import { STORAGE_TEST_MANIFEST } from "./test/setup/storageTestManifest.js"; - -// Storage-only config: runs ONLY the tests in the storage manifest -// (test/setup/storageTestManifest.ts). Shared by both storage CI legs — -// `test:storage-azure` and `test:storage-minio` run this same file and differ -// only by the STORAGE_PROVIDER env var (which picks the testcontainer in -// globalSetup). Everything else (globalSetup, env, pool, timeouts, coverage, -// reporters) is shared via `defineApiVitestConfig`. -export default defineApiVitestConfig({ - include: [...STORAGE_TEST_MANIFEST], -}); diff --git a/docs/development/ci-cd.md b/docs/development/ci-cd.md index a8896123c..8bcb6be7d 100644 --- a/docs/development/ci-cd.md +++ b/docs/development/ci-cd.md @@ -144,32 +144,35 @@ Verifies Prettier formatting without modifying files. If any file is not formatt Runs the Vitest + Testcontainers integration tests. Docker is available on the GitHub-hosted runner, so Testcontainers can spin up PostgreSQL, Azurite, and MinIO containers. -This is a matrix job with three legs that **partition** the suite into disjoint sets, so the storage layer is exercised against **both** storage providers without running the non-storage files more than once: +This is a matrix job with three legs that **partition** the suite into disjoint sets, so the storage layer is exercised against **both** storage providers without running the non-storage files more than once. Each leg is a **Vitest project** (`test.projects` in `apps/api/vitest.config.ts`); the leg name is both the `--project` filter and the branch-protection check name: -| Leg (check name) | `STORAGE_PROVIDER` | Command | Scope | -| ---------------------- | --------------------- | ------------------------- | --------------------------------------------------------------------------------------------- | -| `Test (base)` | `azure_blob_storage`¹ | `pnpm test:base` | The full suite **except** the storage manifest — the bulk, run once. | -| `Test (storage-azure)` | `azure_blob_storage` | `pnpm test:storage-azure` | **Only** the storage manifest (`apps/api/test/setup/storageTestManifest.ts`) against Azurite. | -| `Test (storage-minio)` | `minio` | `pnpm test:storage-minio` | **Only** the storage manifest against MinIO. | +| Leg (check name) | Provider | Scope | +| ---------------------- | -------------------- | --------------------------------------------------------------------------------------------- | +| `Test (base)` | none¹ | The full suite **except** the storage manifest — the bulk, run once. | +| `Test (storage-azure)` | `azure_blob_storage` | **Only** the storage manifest (`apps/api/test/setup/storageTestManifest.ts`) against Azurite. | +| `Test (storage-minio)` | `minio` | **Only** the storage manifest against MinIO. | -`base ∪ storage-azure ∪ storage-minio` covers the full suite, and `base` is disjoint from the storage legs. The storage manifest is the single source of truth for both the base leg's `exclude` and the storage legs' `include`; each test script sets `STORAGE_PROVIDER` itself, which selects the testcontainer in `globalSetup.ts`. +`base ∪ storage-azure ∪ storage-minio` covers the full suite, and `base` is disjoint from the storage legs. The storage manifest is the single source of truth for both the base project's `exclude` and the storage projects' `include`. Each project selects its provider — and boots the matching testcontainer — from its **name** in `globalSetup.ts` (never from a shared `process.env`, which would be last-writer-wins across projects); each project also declares the matching `STORAGE_PROVIDER` in its `test.env` so `buildStorageConfig()` validation passes at `app.ready()`. -¹ The `base` leg sets `STORAGE_PROVIDER=azure_blob_storage` only so `buildStorageConfig()` passes at `app.ready()` without depending on the storage container starting — it never touches real storage (its `app.storage` is the throwing adapter). It runs against Azurite like `storage-azure` at the container level, but exercises zero storage-manifest files. +Each leg runs `pnpm test:ci` (with `LEG` set to the matrix leg), which is `vitest run --project=$LEG --coverage --reporter=blob`. The `--reporter=blob` output carries the coverage data and is merged natively in the `coverage` job below (no external merge script). + +¹ The `base` project boots **no** storage container — it never touches real storage (its `app.storage` is the throwing adapter). Its `test.env` still sets `STORAGE_PROVIDER=azure_blob_storage` (+ a dummy account name) purely so `buildStorageConfig()` passes at `app.ready()`. Before running the tests, all three legs run `pnpm test:verify-storage-manifest` — a static gate that fails if a test touches real storage but is missing from the manifest (or vice versa). A runtime guard backs it up: a storage-agnostic test's `app.storage` is a throwing adapter, so accidental storage access fails loudly. See [Storage test manifest](#storage-test-manifest) below. -**Coverage artifact:** After each leg (even on failure), its coverage report is uploaded — `coverage-report-base`, `coverage-report-storage-azure`, and `coverage-report-storage-minio`: +**Blob report artifact:** After each leg (even on failure), its Vitest blob report is uploaded — `blob-report-base`, `blob-report-storage-azure`, and `blob-report-storage-minio`: ```yaml - uses: actions/upload-artifact@v7 if: always() with: - name: ${{ matrix.coverage_artifact }} - path: apps/api/coverage/ + name: blob-report-${{ matrix.leg }} + path: apps/api/.vitest-reports/ + include-hidden-files: true # .vitest-reports is a dotfile dir retention-days: 7 ``` -Download the artifact from the GitHub Actions run UI to inspect line-by-line coverage. +The `coverage` job downloads all three and merges them (see below). #### Storage test manifest @@ -186,12 +189,14 @@ When you add a test that uploads/reads files, add its path to `STORAGE_TEST_MANI Enforces a **per-metric coverage gate** for `apps/api`: **90%** for all four metrics — lines, statements, functions, and branches. (Branches were the last to reach the bar; #512 closed the gap with new integration suites plus targeted `v8 ignore`s of genuinely-unreachable guards, retiring the earlier intermediate 85% branch ratchet.) -Because the `test` matrix **partitions** the suite into three disjoint legs, no single leg exercises the whole codebase — so a per-run Vitest threshold cannot be used (a leg would fail on the files it never runs). Vitest's own thresholds are therefore kept at `0` (informational; the numbers still print in each leg's report — see `apps/api/vitest.shared.ts`), and the real gate lives here: +Because the `test` matrix **partitions** the suite into three disjoint legs, no single leg exercises the whole codebase — so no single leg can be gated on its own coverage (it would fail on the files it never runs). The **90%** gate is declared once in `apps/api/vitest.config.ts` (`test.coverage.thresholds`); each single-project `test` leg overrides it to `0` on the CLI (its numbers still print in its own report), and the gate is applied only after the legs are merged: + +1. Each `test` leg emits a Vitest **blob** report (`--reporter=blob`, coverage embedded) as an artifact. +2. This job (`needs: test`) downloads all three into one flat dir (`merge-multiple: true` — `vitest --merge-reports` reads the dir non-recursively and rejects subfolders; each blob is uniquely named `blob-.json`) and runs `pnpm test:coverage:merge`, which is `vitest run --merge-reports --coverage`. That command doesn't override the thresholds, so the config's 90% gate applies. Vitest merges the coverage natively — a line covered by _any_ leg counts as covered (v8 merges hit-counts) — then fails if any metric falls below its threshold (90% for all four). -1. Each `test` leg uploads its raw Istanbul coverage (`coverage-final.json`) as an artifact. -2. This job (`needs: test`) downloads all three, and `scripts/check-coverage.mjs` **merges** them — a line covered by _any_ leg counts as covered — then fails if any metric's merged percentage falls below its threshold (90% for all four metrics). +The gate lives in the config, so local (`test:coverage`) and CI (`test:coverage:merge`) use the exact same thresholds — only `test:ci` (a partial single-project run) opts out. If a `test` leg fails, this job is skipped (the PR is already blocked, and there is no complete coverage to merge). -The thresholds are per-metric constants in `scripts/check-coverage.mjs` (override locally per-metric with `COVERAGE_THRESHOLD_LINES` / `_STATEMENTS` / `_FUNCTIONS` / `_BRANCHES`, or all at once with the bare `COVERAGE_THRESHOLD` env var). If a `test` leg fails, this job is skipped (the PR is already blocked, and there is no complete coverage to merge). +The merge step also produces a human-readable report (html + lcov + json) under `apps/api/coverage/`, which this job uploads as the `coverage-report-merged` artifact (`if: always()`, so it is available **even when the gate fails** — exactly when you need to see which lines are missing). Download it straight from the run instead of merging the per-leg blobs by hand. > **Scope:** this merged gate covers `apps/api`. Frontend (`apps/web`) has its own coverage gate — a low **global floor** enforced by the `Test (web)` job (thresholds in `apps/web/vitest.config.ts`, run with `--coverage`), ratcheted up as its logic layers gain tests. See [Testing → Web unit tests](./testing.md#web-unit-tests-appsweb). @@ -212,7 +217,7 @@ Builds all apps and packages via Turborepo. The frontend build requires `VITE_AP The CI workflow references **no secrets**. All test infrastructure (PostgreSQL, Azurite, MinIO) is provided by Testcontainers on the runner, so no external Azure or MinIO credentials are required. - `build` sets `VITE_API_BASE_URL` to a placeholder. -- `test` provides dummy connection vars that satisfy the config validation in `apps/api/src/config/environment.ts`: `AZURE_STORAGE_ACCOUNT_NAME`, `AZURE_STORAGE_CONTAINER_NAME`, `MINIO_ENDPOINT`, `MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, `MINIO_BUCKET`. `STORAGE_PROVIDER` itself is set by each test script (`azure_blob_storage` for `test:base` and `test:storage-azure`, `minio` for `test:storage-minio`); the `base` leg sets it purely so boot validation passes without the storage container. None of these are real secrets — the actual connection details come from the testcontainer started by `globalSetup.ts`. +- `test` needs no storage env in the workflow: each Vitest project declares its own dummy connection vars in `test.env` (`STORAGE_PROVIDER` plus `AZURE_STORAGE_ACCOUNT_NAME` / `AZURE_STORAGE_CONTAINER_NAME` for the Azure projects, or `MINIO_ENDPOINT` / `MINIO_ACCESS_KEY` / `MINIO_SECRET_KEY` / `MINIO_BUCKET` / `MINIO_REGION` for the MinIO project). These only satisfy the config validation in `apps/api/src/config/environment.ts`; none are real secrets — the actual connection details come from the testcontainer started by `globalSetup.ts`, and the dynamic MinIO endpoint is applied from the injected descriptor in `createTestApp` before `app.ready()`. --- @@ -224,21 +229,23 @@ In the GitHub Actions run UI, expand the failing job and step to see the full ou ### Download the coverage report -The `coverage-report-base`, `coverage-report-storage-azure`, and `coverage-report-storage-minio` artifacts are uploaded after every test run. Download them from the "Artifacts" section of the run summary to see which lines are not covered. (Each leg reports coverage for only the files it ran, so no single artifact is the whole picture.) +The `coverage` job uploads the merged, human-readable report as the `coverage-report-merged` artifact (html + lcov + json, `if: always()`) — download it from the run and open `index.html` to see exactly which lines are missing. This is the quickest path when the `Coverage` gate fails. + +The per-leg `blob-report-base`, `blob-report-storage-azure`, and `blob-report-storage-minio` artifacts are also uploaded after every test run. They are Vitest blob reports (not human-readable on their own); to reproduce the merge locally, download all three into `apps/api/.vitest-reports/` and run `pnpm --filter=api exec vitest run --merge-reports --coverage`, which produces the same report under `apps/api/coverage/`. ### Reproduce locally Every CI job runs the same command you can run locally: -| CI job | Local command | -| -------------------- | -------------------------------------------------------------- | -| lint | `pnpm lint` | -| type-check | `pnpm type-check` | -| format | `pnpm format:check` (or `pnpm format` to fix) | -| test (base) | `pnpm test:verify-storage-manifest && pnpm test:base` | -| test (storage-azure) | `pnpm test:verify-storage-manifest && pnpm test:storage-azure` | -| test (storage-minio) | `pnpm test:verify-storage-manifest && pnpm test:storage-minio` | -| build | `VITE_API_BASE_URL=https://example.invalid pnpm build` | +| CI job | Local command | +| --------------- | ------------------------------------------------------ | +| lint | `pnpm lint` | +| type-check | `pnpm type-check` | +| format | `pnpm format:check` (or `pnpm format` to fix) | +| test + coverage | `pnpm test:verify-storage-manifest && pnpm test:api` | +| build | `VITE_API_BASE_URL=https://example.invalid pnpm build` | + +`pnpm test:api` runs the whole suite (all three Vitest projects) in one command, merges coverage, and applies the gate — the local equivalent of the `test` matrix + `coverage` job combined. (CI splits it across three runners for wall-clock, then merges the blobs; the config and gate are identical.) ### Common failures diff --git a/docs/development/testing.md b/docs/development/testing.md index af797fbec..a11983bdf 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -6,17 +6,17 @@ This document describes the test infrastructure, conventions, and patterns used ## Overview -The API uses **Vitest** with **Testcontainers** for integration testing. Tests run against real PostgreSQL and object-storage containers (Azurite by default, MinIO when `STORAGE_PROVIDER=minio`) — there are no mocks for the database layer. All **API** tests live under `apps/api/test/`; the web app's unit tests are **co-located** under `apps/web/src/` (see [Web unit tests](#web-unit-tests-appsweb)). - -| Aspect | Detail | -| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| Framework | Vitest 4.x | -| Test type | Integration (HTTP layer + real DB) | -| Database | Testcontainers — `postgres:18-alpine` | -| Storage | Testcontainers — Azurite (default) or MinIO, per `STORAGE_PROVIDER` | -| Authentication | `AUTH_PROVIDER=forced-user` (hardcoded for all tests) | -| Execution | Parallel — files run across workers, each file gets its own database | -| Coverage | v8 provider; Vitest per-run thresholds held at 0 (informational). The real gate is enforced in CI by merging the legs — see the coverage note below | +The API uses **Vitest** with **Testcontainers** for integration testing. Tests run against real PostgreSQL and object-storage containers (Azurite for the `storage-azure` project, MinIO for `storage-minio`) — there are no mocks for the database layer. All **API** tests live under `apps/api/test/`; the web app's unit tests are **co-located** under `apps/web/src/` (see [Web unit tests](#web-unit-tests-appsweb)). + +| Aspect | Detail | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Framework | Vitest 4.x | +| Test type | Integration (HTTP layer + real DB) | +| Database | Testcontainers — `postgres:18-alpine` | +| Storage | Testcontainers — Azurite (`storage-azure` project) or MinIO (`storage-minio` project) | +| Authentication | `AUTH_PROVIDER=forced-user` (hardcoded for all tests) | +| Execution | Parallel — files run across workers, each file gets its own database | +| Coverage | v8 provider; **90%** gate (all four metrics) declared in the config and applied to any full/merged run; the single-project `test:ci` leg overrides it to 0 — see the coverage note below | --- @@ -27,7 +27,7 @@ apps/api/test/ ├── setup/ │ ├── globalSetup.ts # Starts/stops containers; migrates + seeds the template DB │ ├── testDatabase.ts # PostgreSQL container + migration/seed helpers -│ ├── testStorage.ts # Storage container (Azurite or MinIO, per STORAGE_PROVIDER) +│ ├── testStorage.ts # Storage container (Azurite or MinIO, per project name) │ ├── perFileDatabase.ts # Clones a private DB per test file from the template │ ├── storageTestManifest.ts # Manifest of storage-dependent tests (both storage CI legs) │ └── assertStorageTestManifest.ts # Static verifier (pnpm test:verify-storage-manifest) @@ -59,15 +59,23 @@ test/features///service.test.ts # service-level unit tests ## Global Setup and Database Lifecycle -`test/setup/globalSetup.ts` runs once before all tests: +`test/setup/globalSetup.ts` runs once **per Vitest project** before that project's tests: 1. **PostgreSQL container** — `postgres:18-alpine`, credentials `testuser:testpass`, database `testdb`. Startup timeout: 180 s (accounts for first image pull in CI). -2. **Storage container** — Azurite (`mcr.microsoft.com/azure-storage/azurite`, in-memory) by default, or MinIO (`minio/minio`) when `STORAGE_PROVIDER=minio` (see `test/setup/testStorage.ts`). The `test-files` container/bucket is not pre-created — the test adapters in `@repo/storage/testing` create it lazily and idempotently. Startup timeout: 120 s. If the storage container fails, database-only tests still run. +2. **Storage container** — chosen from the **project name** (not an env var): the `storage-azure` project boots Azurite (`mcr.microsoft.com/azure-storage/azurite`, in-memory), `storage-minio` boots MinIO (`minio/minio`), and `base` boots **none** (its tests never touch storage). See `test/setup/testStorage.ts`. The `test-files` container/bucket is not pre-created — the test adapters in `@repo/storage/testing` create it lazily and idempotently. Startup timeout: 120 s. If a storage container fails to start, that project fails fast (every file in it needs the container); `base` is unaffected, as it boots none. 3. **Migrations** — `prisma migrate deploy` is executed once against the **template** database (`testdb`). 4. **Seeding** — the `@repo/seed` runner (`pnpm run seed` in `tools/seed`) with `SEEDS_DATASET=testing` populates all lookup tables (countries, job positions, methodologies, etc.) in the template. -5. **Context injection** — the template database URL (`databaseUrl`) and a storage descriptor (`storageDescriptor`, provider + connection details) are passed to workers via Vitest's `project.provide()` interface. +5. **Context injection** — the template database URL (`databaseUrl`) and a storage descriptor (`storageDescriptor`, provider + connection details, or `null` for `base`) are passed to that project's workers via Vitest's `project.provide()` interface. The dynamic MinIO endpoint is applied from that descriptor in `createTestApp` before `app.ready()`. -The teardown function stops both containers after all tests complete. +The teardown function stops the project's containers after its tests complete. + +> **Local Docker footprint.** Because `globalSetup` runs once **per project**, a +> full local `pnpm test:api` (one `vitest run --coverage` over all three +> projects) can stand up **three PostgreSQL containers plus Azurite and MinIO** +> at peak — heavier than running the legs one at a time. On a constrained machine +> or a tight Docker Desktop limit, run a single project for a lighter inner loop: +> `pnpm --filter=api exec vitest run --project=base --coverage=false` (`base` +> boots only PostgreSQL). CI is unaffected — each runner runs one `--project`. ### Per-file database isolation @@ -106,9 +114,13 @@ CREATE DATABASE "t_" TEMPLATE "testdb" # Run every test suite (API storage legs + web + seed) pnpm test -# Run all API tests (the three storage legs) +# Run all API tests (all three Vitest projects), merge coverage, apply the gate pnpm test:api +# Lighter inner loop: run a single project (base boots only PostgreSQL — no +# storage container). Skip the gate, which a partial run can't clear. +pnpm --filter=api exec vitest run --project=base --coverage=false + # Run the apps/web suite (Vitest + jsdom; builds @repo/* deps first). # Runs with coverage and enforces a global coverage floor (see the note below). pnpm test:web @@ -128,7 +140,7 @@ pnpm --filter=api test:ui > **Web coverage floor.** `pnpm test:web` runs with `--coverage` and enforces a > **low global coverage floor** (thresholds in `apps/web/vitest.config.ts`). -> Unlike the API — where the 80% gate is currently disabled — the web floor is +> Unlike the API — which enforces a merged 90% gate on all four metrics — the web floor is > deliberately low: `coverage.all` counts every file under `src/**`, and the app > is ~98% render-heavy, untested `screens/`/`components/`, so a high global gate > is impractical. The floor is a **regression guard** set just below the current @@ -344,28 +356,30 @@ Every new endpoint should have tests covering: ## Vitest Configuration Reference -Key settings in `apps/api/vitest.shared.ts`, shared by `vitest.config.ts` (full suite, the default), `vitest.base.config.ts` (`pnpm test:base` — the full suite **minus** the storage manifest, used by the `base` CI leg), and `vitest.storage.config.ts` (`pnpm test:storage-azure` / `pnpm test:storage-minio` — **only** the files in the storage manifest, used by the storage CI legs): - -| Setting | Value | Reason | -| --------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| `maxWorkers` | 4 | Run files in parallel; safe because each file has its own database | -| `fileParallelism` | true | Files run concurrently across workers | -| `testTimeout` | 30 000 ms | Allows for slower container I/O | -| `hookTimeout` | 30 000 ms | Allows beforeAll/afterAll to complete | -| `teardownTimeout` | 10 000 ms | Container shutdown grace period | -| `globalSetup` | `./test/setup/globalSetup.ts` | Container lifecycle; migrate + seed the template DB | -| `setupFiles` | `./test/setup/perFileDatabase.ts` | Clones a private database per test file | -| `coverage.thresholds` | 0% in all environments (branches, functions, lines, statements) | Forced to 0 everywhere (`process.env.CI \|\| true` in `vitest.shared.ts`); see the coverage note below | - -> **Note on coverage:** Vitest's own per-run thresholds are held at **0** -> (informational only) — `vitest.shared.ts` forces them to 0 (`process.env.CI || -true` always evaluates truthy). This is deliberate: the suite is partitioned -> into three legs (`base`, `storage-azure`, `storage-minio`) that each emit a -> **partial** coverage artifact, so no single leg exercises the whole codebase and -> a per-run Vitest threshold can't be used (a leg would fail on the files it never -> runs). The **real gate is not disabled** — it runs in CI's `coverage` job, where -> [`scripts/check-coverage.mjs`](../../scripts/check-coverage.mjs) **merges** the -> three legs' Istanbul reports (a line covered by _any_ leg counts) and fails if -> any metric falls below its threshold: **90%** for all four metrics — lines, -> statements, functions, and branches. See the [`coverage` job](./ci-cd.md#coverage) -> in the CI/CD guide. +Everything lives in `apps/api/vitest.config.ts`: a local `defineApiVitestProject` helper builds one project, and the config assembles three **projects** (`test.projects`) — `base` (the full suite **minus** the storage manifest), `storage-azure`, and `storage-minio` (**only** the storage manifest, one per provider). Coverage and the other root-only options sit once on the root `test`. One config drives both `vitest run --coverage` locally and the `--project=` legs in CI: + +| Setting | Value | Reason | +| --------------------- | --------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `maxWorkers` | 4 | Run files in parallel; safe because each file has its own database | +| `fileParallelism` | true | Files run concurrently across workers | +| `testTimeout` | 30 000 ms | Allows for slower container I/O | +| `hookTimeout` | 30 000 ms | Allows beforeAll/afterAll to complete | +| `teardownTimeout` | 10 000 ms | Container shutdown grace period | +| `globalSetup` | `./test/setup/globalSetup.ts` | Container lifecycle; migrate + seed the template DB | +| `setupFiles` | `./test/setup/perFileDatabase.ts` | Clones a private database per test file | +| `coverage.thresholds` | **90%** for all four metrics (branches, functions, lines, statements), declared in the config | The gate lives where it is read; `test:ci` (a single-project, partial run) overrides it to 0 on the CLI — see the coverage note below | + +> **Note on coverage:** the **90%** gate (all four metrics — lines, statements, +> functions, and branches) is declared once in `vitest.config.ts` +> (`test.coverage.thresholds`) and applies to any full run. The suite is +> partitioned into three projects (`base`, `storage-azure`, `storage-minio`), so a +> single project never exercises the whole codebase — a per-project threshold +> would fail on the files it never runs. The one run that opts out is `test:ci`, +> which runs a single `--project` and overrides the thresholds to **0** on the CLI; +> the gate is then applied once the projects' coverage is merged. v8 merges +> hit-counts, so a line covered by _any_ project counts as covered. +> +> - **Locally:** `pnpm test:api` runs all three projects in one `vitest run --coverage`, merges their coverage, and applies the gate. +> - **In CI:** each leg emits a blob report (`--reporter=blob`); the `coverage` job merges them (`vitest run --merge-reports --coverage`) and applies the gate. No external merge script. +> +> See the [`coverage` job](./ci-cd.md#coverage) in the CI/CD guide. diff --git a/docs/infrastructure/FileStorage.md b/docs/infrastructure/FileStorage.md index 0292a4d0f..3388450af 100644 --- a/docs/infrastructure/FileStorage.md +++ b/docs/infrastructure/FileStorage.md @@ -222,7 +222,7 @@ The setup is in `apps/api/test/setup/testStorage.ts` (storage container) and `te Neither path creates the bucket/container up front — `createAzureBlobTestAdapter`/`createMinioTestAdapter` (`@repo/storage/testing`) create it lazily and idempotently the first time a test app is built against the descriptor. -CI partitions this into three legs instead of running the full suite twice: `Test (base)` (`pnpm test:base`) runs everything **except** the storage manifest, while `Test (storage-azure)` (`pnpm test:storage-azure`) and `Test (storage-minio)` (`pnpm test:storage-minio`) run **only** the manifest against each provider. All three must pass for a PR to merge. See [Storage test manifest](../development/ci-cd.md#storage-test-manifest) for how that manifest is kept honest. +CI partitions this into three legs (Vitest projects) instead of running the full suite twice: `Test (base)` runs everything **except** the storage manifest, while `Test (storage-azure)` and `Test (storage-minio)` run **only** the manifest against each provider (each project picks its provider from its name in `globalSetup.ts`). All three must pass for a PR to merge. See [Storage test manifest](../development/ci-cd.md#storage-test-manifest) for how that manifest is kept honest. ## Operational notes diff --git a/docs/openssf/gold_badge_assesment.md b/docs/openssf/gold_badge_assesment.md index 39700f5fe..7cf88a5f5 100644 --- a/docs/openssf/gold_badge_assesment.md +++ b/docs/openssf/gold_badge_assesment.md @@ -45,13 +45,13 @@ ## Quality — Build & Tests -| Criterion | Level | Status | Evidence / Gap | -| ----------------------------- | ----- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `build_reproducible` | MUST | ⚠️ | Lockfile + pinned Docker digests help; demonstrate/verify reproducible build output (the web bundle _is_ built). | -| `test_invocation` | MUST | ✅ | `pnpm test` (standard). | -| `test_continuous_integration` | MUST | ✅ | CI runs tests on every PR (required checks). | -| `test_statement_coverage90` | MUST | ❌ | `apps/api` now enforces **90%** statement coverage in CI (the `coverage` job merges the test legs via `scripts/check-coverage.mjs`); `apps/web` is now unit-tested across its logic layers behind an enforced floor but sits at only ~8% overall statement coverage (its render-heavy `screens/`/`components/` are untested), so the project-wide 90% bar is unmet. | -| `test_branch_coverage80` | MUST | ❌ | `apps/api` now enforces **90%** branch coverage in CI (exceeds the 80% bar); `apps/web` has tests (logic layers ~100%, ~7% branches overall), so the project-wide 80% bar is unmet until its UI is covered. | +| Criterion | Level | Status | Evidence / Gap | +| ----------------------------- | ----- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `build_reproducible` | MUST | ⚠️ | Lockfile + pinned Docker digests help; demonstrate/verify reproducible build output (the web bundle _is_ built). | +| `test_invocation` | MUST | ✅ | `pnpm test` (standard). | +| `test_continuous_integration` | MUST | ✅ | CI runs tests on every PR (required checks). | +| `test_statement_coverage90` | MUST | ❌ | `apps/api` now enforces **90%** statement coverage in CI (the `coverage` job merges the test legs natively via `vitest run --merge-reports --coverage`); `apps/web` is now unit-tested across its logic layers behind an enforced floor but sits at only ~8% overall statement coverage (its render-heavy `screens/`/`components/` are untested), so the project-wide 90% bar is unmet. | +| `test_branch_coverage80` | MUST | ❌ | `apps/api` now enforces **90%** branch coverage in CI (exceeds the 80% bar); `apps/web` has tests (logic layers ~100%, ~7% branches overall), so the project-wide 80% bar is unmet until its UI is covered. | ## Security diff --git a/docs/openssf/passing_badge_assesment.md b/docs/openssf/passing_badge_assesment.md index c3002c329..6266f6523 100644 --- a/docs/openssf/passing_badge_assesment.md +++ b/docs/openssf/passing_badge_assesment.md @@ -54,21 +54,21 @@ ## Quality -| Criterion | Level | Status | Evidence / Gap | -| ----------------------------- | --------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `build` | MUST | ✅ | `pnpm build` via Turborepo rebuilds from source. | -| `build_common_tools` | SUGGESTED | ✅ | pnpm, Turbo, tsc, Vite. | -| `build_floss_tools` | SHOULD | ✅ | Buildable with FLOSS tools (Node, pnpm, …). | -| `test` | MUST | ✅ | `apps/api` Vitest + Testcontainers suite (**151** API test files, 145 integration) plus an `apps/web` Vitest + jsdom + React Testing Library unit suite, documented in `docs/development/testing.md`. | -| `test_invocation` | SHOULD | ✅ | `pnpm test` (standard). | -| `test_most` | SUGGESTED | ⚠️ | `apps/api` coverage is now enforced in CI (90% lines/statements/functions/branches, merged across legs by `scripts/check-coverage.mjs`; Vitest's per-run thresholds stay 0 by design), and `apps/web` now has a Vitest + jsdom + RTL suite over its logic layers (behind an enforced floor), though most of its render-heavy UI is still untested (~5% function coverage overall). | -| `test_continuous_integration` | SUGGESTED | ✅ | CI runs the suite on every PR (`.github/workflows/ci.yml`); the `Test` jobs are **required** status checks. | -| `test_policy` | MUST | ✅ | "Definition of done" in `CONTRIBUTING.md` requires tests for new functionality. | -| `tests_are_added` | MUST | ✅ | Test files accompany feature work throughout history. | -| `tests_documented_added` | SUGGESTED | ✅ | Policy documented in the contributing guide. | -| `warnings` | MUST | ✅ | ESLint + TypeScript `strict` enabled. | -| `warnings_fixed` | MUST | ✅ | CI enforces `--max-warnings=0` (required "Lint" check). | -| `warnings_strict` | SUGGESTED | ✅ | TS `strict: true`, typed-lint (`recommendedTypeChecked`). | +| Criterion | Level | Status | Evidence / Gap | +| ----------------------------- | --------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `build` | MUST | ✅ | `pnpm build` via Turborepo rebuilds from source. | +| `build_common_tools` | SUGGESTED | ✅ | pnpm, Turbo, tsc, Vite. | +| `build_floss_tools` | SHOULD | ✅ | Buildable with FLOSS tools (Node, pnpm, …). | +| `test` | MUST | ✅ | `apps/api` Vitest + Testcontainers suite (**151** API test files, 145 integration) plus an `apps/web` Vitest + jsdom + React Testing Library unit suite, documented in `docs/development/testing.md`. | +| `test_invocation` | SHOULD | ✅ | `pnpm test` (standard). | +| `test_most` | SUGGESTED | ⚠️ | `apps/api` coverage is now enforced in CI (90% lines/statements/functions/branches, merged across legs natively by `vitest run --merge-reports --coverage`; Vitest's per-run thresholds stay 0 by design), and `apps/web` now has a Vitest + jsdom + RTL suite over its logic layers (behind an enforced floor), though most of its render-heavy UI is still untested (~5% function coverage overall). | +| `test_continuous_integration` | SUGGESTED | ✅ | CI runs the suite on every PR (`.github/workflows/ci.yml`); the `Test` jobs are **required** status checks. | +| `test_policy` | MUST | ✅ | "Definition of done" in `CONTRIBUTING.md` requires tests for new functionality. | +| `tests_are_added` | MUST | ✅ | Test files accompany feature work throughout history. | +| `tests_documented_added` | SUGGESTED | ✅ | Policy documented in the contributing guide. | +| `warnings` | MUST | ✅ | ESLint + TypeScript `strict` enabled. | +| `warnings_fixed` | MUST | ✅ | CI enforces `--max-warnings=0` (required "Lint" check). | +| `warnings_strict` | SUGGESTED | ✅ | TS `strict: true`, typed-lint (`recommendedTypeChecked`). | ## Security diff --git a/docs/openssf/silver_badge_assesment.md b/docs/openssf/silver_badge_assesment.md index 193ebe2a5..3736be3ca 100644 --- a/docs/openssf/silver_badge_assesment.md +++ b/docs/openssf/silver_badge_assesment.md @@ -44,27 +44,27 @@ ## Quality -| Criterion | Level | Status | Evidence / Gap | -| --------------------------------- | ------ | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `coding_standards` | MUST | ✅ | ESLint config + Prettier + `docs/development/` conventions. | -| `coding_standards_enforced` | MUST | ✅ | ESLint (`--max-warnings=0`) + `format:check` as **required** CI checks. | -| `build_standard_variables` | MUST | ➖ | No native compiler/linker (JS/TS). | -| `build_preserve_debug` | SHOULD | ⚠️ | Source maps available; confirm they're produced/retained where useful. | -| `build_non_recursive` | MUST | ✅ | Turborepo orchestrates by dependency graph (not recursive make). | -| `build_repeatable` | MUST | ⚠️ | Lockfile + pinned Docker digests aid repeatability; bit-for-bit not verified. | -| `installation_common` | MUST | ⚠️ | App (not a package): `pnpm install` + Docker/compose — document uninstall/teardown. | -| `installation_standard_variables` | MUST | ➖ | Not an OS-installed artifact (no `DESTDIR`). | -| `installation_development_quick` | MUST | ✅ | README + docker-compose spin up a dev env + tests quickly. | -| `external_dependencies` | MUST | ✅ | `package.json` + `pnpm-lock.yaml` (machine-processable). | -| `dependency_monitoring` | MUST | ✅ | Dependabot (npm/docker/actions) + `pnpm audit` CI gate + `minimumReleaseAge` + Dependabot security updates. | -| `updateable_reused_components` | MUST | ✅ | pnpm workspace + Dependabot make updates straightforward. | -| `interfaces_current` | SHOULD | ⚠️ | Deps kept current; no systematic deprecation scan. | -| `automated_integration_testing` | MUST | ✅ | CI runs the suite on each PR (required checks) and uploads coverage artifacts. | -| `regression_tests_added50` | MUST | ⚠️ | Likely in practice but not measured; track "test-per-bug" going forward. | -| `test_statement_coverage80` | MUST | ❌ | `apps/api` now enforces **≥90%** statement coverage in CI — the `coverage` job merges the three test legs and gates via `scripts/check-coverage.mjs` (Vitest's own per-run thresholds stay at 0 by design). `apps/web` now has a Vitest + jsdom + React Testing Library suite over its logic layers (behind an enforced coverage floor) but only ~8% overall statement coverage, so the project-wide ≥80% bar is unmet. | -| `test_policy_mandated` | MUST | ⚠️ | `CONTRIBUTING.md` requires tests; formalize it as a mandated policy. | -| `tests_documented_added` | MUST | ✅ | Documented in the contributing guide. | -| `warnings_strict` | MUST | ✅ | TS `strict` + typed ESLint, zero-warnings CI. | +| Criterion | Level | Status | Evidence / Gap | +| --------------------------------- | ------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `coding_standards` | MUST | ✅ | ESLint config + Prettier + `docs/development/` conventions. | +| `coding_standards_enforced` | MUST | ✅ | ESLint (`--max-warnings=0`) + `format:check` as **required** CI checks. | +| `build_standard_variables` | MUST | ➖ | No native compiler/linker (JS/TS). | +| `build_preserve_debug` | SHOULD | ⚠️ | Source maps available; confirm they're produced/retained where useful. | +| `build_non_recursive` | MUST | ✅ | Turborepo orchestrates by dependency graph (not recursive make). | +| `build_repeatable` | MUST | ⚠️ | Lockfile + pinned Docker digests aid repeatability; bit-for-bit not verified. | +| `installation_common` | MUST | ⚠️ | App (not a package): `pnpm install` + Docker/compose — document uninstall/teardown. | +| `installation_standard_variables` | MUST | ➖ | Not an OS-installed artifact (no `DESTDIR`). | +| `installation_development_quick` | MUST | ✅ | README + docker-compose spin up a dev env + tests quickly. | +| `external_dependencies` | MUST | ✅ | `package.json` + `pnpm-lock.yaml` (machine-processable). | +| `dependency_monitoring` | MUST | ✅ | Dependabot (npm/docker/actions) + `pnpm audit` CI gate + `minimumReleaseAge` + Dependabot security updates. | +| `updateable_reused_components` | MUST | ✅ | pnpm workspace + Dependabot make updates straightforward. | +| `interfaces_current` | SHOULD | ⚠️ | Deps kept current; no systematic deprecation scan. | +| `automated_integration_testing` | MUST | ✅ | CI runs the suite on each PR (required checks) and uploads a merged, human-readable coverage report artifact. | +| `regression_tests_added50` | MUST | ⚠️ | Likely in practice but not measured; track "test-per-bug" going forward. | +| `test_statement_coverage80` | MUST | ❌ | `apps/api` now enforces **≥90%** statement coverage in CI — the `coverage` job merges the three test legs natively (`vitest run --merge-reports --coverage`) and gates by flag (Vitest's own per-run thresholds stay at 0 by design). `apps/web` now has a Vitest + jsdom + React Testing Library suite over its logic layers (behind an enforced coverage floor) but only ~8% overall statement coverage, so the project-wide ≥80% bar is unmet. | +| `test_policy_mandated` | MUST | ⚠️ | `CONTRIBUTING.md` requires tests; formalize it as a mandated policy. | +| `tests_documented_added` | MUST | ✅ | Documented in the contributing guide. | +| `warnings_strict` | MUST | ✅ | TS `strict` + typed ESLint, zero-warnings CI. | ## Security diff --git a/package.json b/package.json index 2c8a56f9e..04fe4e60e 100644 --- a/package.json +++ b/package.json @@ -17,11 +17,10 @@ "start:api": "turbo run start --filter=api", "start:web": "turbo run start --filter=web", "test": "pnpm test:api && pnpm test:web && pnpm test:seed", - "test:api": "pnpm test:base && pnpm test:storage-azure && pnpm test:storage-minio", + "test:api": "turbo run test:coverage --filter=api", "test:seed": "turbo run test --filter=@repo/seed", - "test:base": "turbo run test:base --filter=api", - "test:storage-azure": "turbo run test:storage-azure --filter=api", - "test:storage-minio": "turbo run test:storage-minio --filter=api", + "test:ci": "turbo run test:ci --filter=api", + "test:coverage:merge": "turbo run test:coverage:merge --filter=api", "test:verify-storage-manifest": "turbo run test:verify-storage-manifest --filter=api", "test:web": "turbo run test --filter=web", "lint": "turbo run lint -- --max-warnings=0", @@ -37,7 +36,6 @@ }, "devDependencies": { "cpy-cli": "^7.0.0", - "istanbul-lib-coverage": "^3.2.2", "prettier": "^3.9.4", "prettier-plugin-tailwindcss": "^0.8.0", "rimraf": "^6.1.3", diff --git a/packages/eslint-config/base.ts b/packages/eslint-config/base.ts index c4a716cec..1f25cbd89 100644 --- a/packages/eslint-config/base.ts +++ b/packages/eslint-config/base.ts @@ -13,7 +13,7 @@ export const config: Linter.Config[] = [ "dist/**", "build/**", "coverage/**", - "coverage-artifacts/**", + ".vitest-reports/**", "vitest-report/**", "node_modules/**", ], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 02defb2f1..9ff5c999f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,9 +49,6 @@ importers: cpy-cli: specifier: ^7.0.0 version: 7.0.0 - istanbul-lib-coverage: - specifier: ^3.2.2 - version: 3.2.2 prettier: specifier: ^3.9.4 version: 3.9.4 diff --git a/scripts/check-coverage.mjs b/scripts/check-coverage.mjs deleted file mode 100644 index d611721c1..000000000 --- a/scripts/check-coverage.mjs +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env node -// Enforces the apps/api coverage gate in CI. -// -// The apps/api test suite is split across three CI legs that form a disjoint -// partition of the suite (base + one leg per storage provider; see the `test` -// job in .github/workflows/ci.yml). No single leg exercises the whole codebase, -// so a per-run Vitest threshold cannot reflect true coverage — a leg would fail -// on the files it never runs. Instead each leg uploads its raw Istanbul coverage -// (coverage-final.json) as an artifact, and the `coverage` job downloads all -// three legs and runs this script. -// -// It merges the legs' coverage into one map (a line covered by ANY leg counts as -// covered) and fails if any merged metric is below its threshold. This is the -// real gate; Vitest's own thresholds stay at 0 (informational only — see -// apps/api/vitest.shared.ts for why). -// -// Thresholds are per-metric and all four are now held to 90%. Branch coverage -// was the last to reach the bar (most remaining uncovered branches were -// error/edge paths behind integration-test setup, plus defensive guards); #512 -// closed that gap with new integration suites plus targeted `v8 ignore`s of -// genuinely-unreachable guards, retiring the earlier intermediate 85% branch -// ratchet. Each is overridable via env: -// COVERAGE_THRESHOLD_LINES / _STATEMENTS / _FUNCTIONS / _BRANCHES. A bare -// COVERAGE_THRESHOLD (no suffix) overrides the default for every metric that -// lacks its own per-metric override. -// -// Usage: node scripts/check-coverage.mjs [dir] -// [dir] defaults to "coverage-artifacts" and is searched recursively for -// every coverage-final.json (one per downloaded leg artifact). - -import { globSync, readFileSync } from "node:fs"; -import libCoverage from "istanbul-lib-coverage"; - -const { createCoverageMap, createCoverageSummary } = libCoverage; - -// Per-metric defaults. All four metrics sit at 90%. -const DEFAULT_THRESHOLDS = { - lines: 90, - statements: 90, - functions: 90, - branches: 90, -}; - -// Resolve a metric's threshold: its own env override wins, then a bare -// COVERAGE_THRESHOLD, then the built-in default. -const globalOverride = process.env.COVERAGE_THRESHOLD; -const thresholdFor = (metric) => { - const perMetric = process.env[`COVERAGE_THRESHOLD_${metric.toUpperCase()}`]; - if (perMetric !== undefined) return Number(perMetric); - if (globalOverride !== undefined) return Number(globalOverride); - return DEFAULT_THRESHOLDS[metric]; -}; - -const searchDir = process.argv[2] ?? "coverage-artifacts"; - -const files = globSync(`${searchDir}/**/coverage-final.json`); - -if (files.length === 0) { - console.error( - `✖ No coverage-final.json found under "${searchDir}/". ` + - `Did the test legs upload their coverage artifacts?` - ); - process.exit(1); -} - -const map = createCoverageMap({}); -for (const file of files) { - map.merge(JSON.parse(readFileSync(file, "utf8"))); -} - -const summary = createCoverageSummary(); -for (const f of map.files()) { - summary.merge(map.fileCoverageFor(f).toSummary()); -} - -console.log( - `Merged coverage from ${files.length} leg(s) across ${map.files().length} file(s):` -); - -const metrics = ["lines", "statements", "functions", "branches"]; -let failed = false; -for (const metric of metrics) { - const { covered, total, pct } = summary.data[metric]; - const threshold = thresholdFor(metric); - // A metric with no measurable entries (total === 0) cannot be below target. - const ok = total === 0 || pct >= threshold; - if (!ok) failed = true; - console.log( - ` ${ok ? "✓" : "✗"} ${metric.padEnd(11)} ${pct.toFixed(2).padStart(6)}% (${covered}/${total}) [min ${threshold}%]` - ); -} - -if (failed) { - console.error( - `\n✖ Coverage is below threshold for one or more metrics (see above). ` + - `Download the leg coverage artifacts for the per-file breakdown.` - ); - process.exit(1); -} - -console.log(`\n✔ Coverage meets all thresholds.`); diff --git a/turbo.json b/turbo.json index e82056d48..1eb44f77b 100644 --- a/turbo.json +++ b/turbo.json @@ -20,7 +20,6 @@ "DEV", "APP_VERSION", "CI", - "COVERAGE_DIR", "AZURE_STORAGE_ACCOUNT_NAME", "AZURE_STORAGE_CONTAINER_NAME", "AZURE_STORAGE_TENANT_ID", @@ -78,21 +77,26 @@ "inputs": ["$TURBO_DEFAULT$", "*tests/**"], "outputs": [] }, - "test:base": { + "test:coverage": { "cache": false, "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", "*tests/**"], "outputs": [] }, - "test:storage-azure": { + "test:ci": { "cache": false, "dependsOn": ["^build"], + "passThroughEnv": ["LEG"], "inputs": ["$TURBO_DEFAULT$", "*tests/**"], "outputs": [] }, - "test:storage-minio": { + "test:coverage:merge": { "cache": false, - "dependsOn": ["^build"], + // No `^build`: `vitest run --merge-reports --coverage` runs no tests — it + // reads the blob JSON and maps coverage onto apps/api/src/** from the + // checkout. It never loads globalSetup/setupFiles and imports no @repo/* + // module, so building the workspace deps here is wasted CI time. + "dependsOn": [], "inputs": ["$TURBO_DEFAULT$", "*tests/**"], "outputs": [] },