diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 10c10133c8..e15d03a5d8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -23,9 +23,7 @@ e2e_config: &e2e_config setup: { platform: [], arch: [] } adjustments: - with: { platform: mac, arch: arm64 } - # Windows E2E temporarily disabled while AINFRA-2588 investigates Windows E2E hangs in Buildkite. - # See https://linear.app/a8c/issue/AINFRA-2588/investigate-studio-windows-e2e-hangs-in-buildkite - # - with: { platform: windows, arch: x64 } + - with: { platform: windows, arch: x64 } notify: - github_commit_status: context: E2E Tests diff --git a/apps/cli/commands/site/tests/create.e2e.test.ts b/apps/cli/commands/site/tests/create.e2e.test.ts deleted file mode 100644 index d7a0025616..0000000000 --- a/apps/cli/commands/site/tests/create.e2e.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * @vitest-environment node - * - * Real end-to-end test for `studio site create`. Unlike create.test.ts (which - * mocks the command's dependencies), this spawns the built CLI binary and - * creates an actual site, verifying the real persisted state on disk. - * - * Requires the CLI to be built first (`npm run cli:build`); the suite skips - * itself otherwise. Tagged `e2e` so it runs in the slower (release/manual) - * suite rather than on every PR — run with `npm test -- --tagsFilter='e2e'`. - */ -import fs from 'fs'; -import path from 'path'; -import { afterEach, describe, expect, it } from 'vitest'; -import { - cleanupCliEnv, - cliE2ePrerequisitesMet, - readCliConfig, - runCli, - setupCliEnv, - type CliEnv, -} from './helpers/cli-e2e'; - -describe.skipIf( ! cliE2ePrerequisitesMet() )( 'CLI e2e: studio site create', () => { - let env: CliEnv | undefined; - - afterEach( () => { - if ( env ) { - cleanupCliEnv( env ); - env = undefined; - } - } ); - - it( 'creates a site with a custom name', { tags: [ 'e2e' ], timeout: 120_000 }, async () => { - env = setupCliEnv(); - const siteName = 'Custom E2E Site'; - const sitePath = path.join( env.sitesDir, 'custom-e2e-site' ); - - const result = await runCli( - [ - 'site', - 'create', - '--name', - siteName, - '--path', - sitePath, - '--wp', - 'latest', - '--no-start', - '--skip-browser', - '--skip-log-details', - ], - env - ); - - expect( result.code, result.stderr ).toBe( 0 ); - - // The site is persisted to the real cli.json with the custom name. - const config = readCliConfig( env ); - expect( config.sites ).toHaveLength( 1 ); - const [ site ] = config.sites; - expect( site.name ).toBe( siteName ); - expect( site.path ).toBe( sitePath ); - expect( site.phpVersion ).toBeTruthy(); - expect( site.running ).toBe( false ); - - // Real WordPress core files were copied into the site directory. - // (wp-config.php is generated at server start, which --no-start skips.) - expect( fs.existsSync( path.join( sitePath, 'wp-load.php' ) ) ).toBe( true ); - expect( fs.existsSync( path.join( sitePath, 'wp-includes', 'version.php' ) ) ).toBe( true ); - } ); - - it( - 'creates a site with a custom domain and HTTPS', - { tags: [ 'e2e' ], timeout: 120_000 }, - async () => { - env = setupCliEnv(); - const siteName = 'Domain E2E Site'; - const sitePath = path.join( env.sitesDir, 'domain-e2e-site' ); - const customDomain = 'custom-e2e.local'; - - const result = await runCli( - [ - 'site', - 'create', - '--name', - siteName, - '--path', - sitePath, - '--wp', - 'latest', - '--domain', - customDomain, - '--https', - '--no-start', - '--skip-browser', - '--skip-log-details', - ], - env - ); - - expect( result.code, result.stderr ).toBe( 0 ); - - // The custom domain and HTTPS preference are persisted to cli.json. - // (--no-start skips the hosts-file / certificate setup that running would do.) - const config = readCliConfig( env ); - expect( config.sites ).toHaveLength( 1 ); - const [ site ] = config.sites; - expect( site.name ).toBe( siteName ); - expect( site.customDomain ).toBe( customDomain ); - expect( site.enableHttps ).toBe( true ); - expect( site.running ).toBe( false ); - - expect( fs.existsSync( path.join( sitePath, 'wp-load.php' ) ) ).toBe( true ); - } - ); -} ); diff --git a/apps/cli/commands/site/tests/helpers/cli-e2e.ts b/apps/cli/commands/site/tests/helpers/cli-e2e.ts deleted file mode 100644 index aff90d1faa..0000000000 --- a/apps/cli/commands/site/tests/helpers/cli-e2e.ts +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Harness for real end-to-end CLI integration tests. - * - * Spawns the built CLI binary (`dist/cli/main.mjs`) against an isolated config - * directory so tests exercise the real `studio site create` flow — real file - * copying, real `cli.json` persistence — without mocking, touching the - * developer's `~/.studio`, or needing the desktop app. - */ -import { spawn } from 'child_process'; -import { randomUUID } from 'crypto'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; - -// `dist/cli/main.mjs` relative to this file (apps/cli/commands/site/tests/helpers). -const CLI_MAIN = path.resolve( import.meta.dirname, '../../../../dist/cli/main.mjs' ); - -// The real bundled WordPress that ships with Studio. `getServerFilesPath()` -// derives from the config directory, so the harness symlinks this into the -// isolated config dir to let `--wp latest` copy it offline and deterministically. -const REAL_SERVER_FILES = path.join( os.homedir(), '.studio', 'server-files' ); -const BUNDLED_LATEST_WP = path.join( REAL_SERVER_FILES, 'wordpress-versions', 'latest' ); - -export interface CliEnv { - root: string; - configDir: string; - sitesDir: string; - cliConfigPath: string; - daemonHome: string; -} - -export interface CliResult { - code: number | null; - stdout: string; - stderr: string; -} - -/** - * Whether the prerequisites for spawning the CLI are present: the built binary - * and the bundled WordPress files. Used to skip the suite with a clear signal - * when the CLI hasn't been built (run `npm run cli:build` first). - */ -export function cliE2ePrerequisitesMet(): boolean { - return fs.existsSync( CLI_MAIN ) && fs.existsSync( BUNDLED_LATEST_WP ); -} - -/** - * Creates an isolated config + sites directory for a single CLI run. - */ -export function setupCliEnv(): CliEnv { - const root = path.join( os.tmpdir(), `studio-cli-e2e-${ randomUUID() }` ); - const configDir = path.join( root, 'config' ); - const sitesDir = path.join( root, 'sites' ); - // Each run gets its own process-manager daemon (via STUDIO_PROCESS_MANAGER_HOME - // in runCli) so `site start`/`stop` never touch the developer's real daemon or - // sites. Keep it SHORT and directly under tmpdir: the daemon's control socket is - // a Unix domain socket (~104-char limit on macOS), so nesting under the long - // `root` overflows it and the connection fails with EINVAL. - const daemonHome = path.join( os.tmpdir(), `scd-${ randomUUID().slice( 0, 8 ) }` ); - fs.mkdirSync( configDir, { recursive: true } ); - fs.mkdirSync( sitesDir, { recursive: true } ); - fs.mkdirSync( daemonHome, { recursive: true } ); - - // Reuse the real bundled WordPress without copying hundreds of MB. The copy - // the CLI performs only reads from here, so the symlink is never written to. - fs.symlinkSync( REAL_SERVER_FILES, path.join( configDir, 'server-files' ), 'junction' ); - - // Pre-seed cli.json with a recent dependency-check timestamp so the spawned - // CLI skips its 24h WordPress-version update: keeps the run offline and - // deterministic, and avoids writing through the server-files symlink. - const cliConfigPath = path.join( configDir, 'cli.json' ); - fs.writeFileSync( - cliConfigPath, - JSON.stringify( { - version: 1, - sites: [], - snapshots: [], - lastDependencyCheckTime: Date.now(), - } ) - ); - - return { root, configDir, sitesDir, cliConfigPath, daemonHome }; -} - -export function cleanupCliEnv( env: CliEnv ): void { - fs.rmSync( env.root, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 } ); - fs.rmSync( env.daemonHome, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 } ); -} - -/** - * Runs the built CLI with the given arguments against the isolated environment. - * Resolves with the exit code and captured output once the process exits. - */ -export function runCli( args: string[], env: CliEnv ): Promise< CliResult > { - return new Promise( ( resolve, reject ) => { - const child = spawn( process.execPath, [ CLI_MAIN, ...args ], { - // Non-TTY stdio so the CLI runs fully non-interactively. - stdio: [ 'ignore', 'pipe', 'pipe' ], - env: { - ...process.env, - DEV_CONFIG_DIR: env.configDir, - STUDIO_PROCESS_MANAGER_HOME: env.daemonHome, - }, - } ); - - let stdout = ''; - let stderr = ''; - child.stdout.on( 'data', ( chunk ) => ( stdout += chunk.toString() ) ); - child.stderr.on( 'data', ( chunk ) => ( stderr += chunk.toString() ) ); - child.on( 'error', reject ); - child.on( 'close', ( code ) => resolve( { code, stdout, stderr } ) ); - } ); -} - -/** - * Reads the persisted cli.json from the isolated environment. - */ -export function readCliConfig( env: CliEnv ): { - sites: Array< Record< string, unknown > >; - [ key: string ]: unknown; -} { - return JSON.parse( fs.readFileSync( env.cliConfigPath, 'utf-8' ) ); -} diff --git a/apps/cli/commands/site/tests/site-management.e2e.test.ts b/apps/cli/commands/site/tests/site-management.e2e.test.ts deleted file mode 100644 index b436587ea9..0000000000 --- a/apps/cli/commands/site/tests/site-management.e2e.test.ts +++ /dev/null @@ -1,211 +0,0 @@ -/** - * @vitest-environment node - * - * Real end-to-end tests for Studio's site-management operations: renaming a - * site, changing its PHP version, updating the WordPress site title, and - * deleting a site (with and without removing its files). Unlike the unit - * suites that mock the daemon and config layer, this spawns the built CLI and - * asserts the real persisted state in cli.json and on disk. - * - * Requires the CLI to be built first (`npm run cli:build`); the suite skips - * itself otherwise. Tagged `e2e` so it runs in the slower (release/manual) - * suite rather than on every PR — run with `npm test -- --tagsFilter='e2e'`. - */ -import fs from 'fs'; -import path from 'path'; -import { SupportedPHPVersions } from '@studio/common/types/php-versions'; -import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; -import { - cleanupCliEnv, - cliE2ePrerequisitesMet, - readCliConfig, - runCli, - setupCliEnv, - type CliEnv, -} from './helpers/cli-e2e'; - -function findSite( env: CliEnv, sitePath: string ): Record< string, unknown > | undefined { - return readCliConfig( env ).sites.find( ( site ) => site.path === sitePath ); -} - -/** - * `--runtime sandbox` keeps the run hermetic; `--no-start` defers the slow - * WordPress install, so callers needing a live WordPress start the site. - */ -async function createStoppedSite( env: CliEnv, name: string, dirName: string ): Promise< string > { - const sitePath = path.join( env.sitesDir, dirName ); - const result = await runCli( - [ - 'site', - 'create', - '--name', - name, - '--path', - sitePath, - '--wp', - 'latest', - '--runtime', - 'sandbox', - '--no-start', - '--skip-browser', - '--skip-log-details', - ], - env - ); - expect( result.code, result.stderr ).toBe( 0 ); - return sitePath; -} - -describe.skipIf( ! cliE2ePrerequisitesMet() )( 'CLI e2e: studio site management', () => { - // The edit cases are independent, so create the site once (a slow WordPress - // copy) and run them in order against it. - describe( 'editing a site', () => { - let env: CliEnv | undefined; - let sitePath = ''; - - beforeAll( async () => { - env = setupCliEnv(); - sitePath = await createStoppedSite( env, 'Editable E2E Site', 'editable-e2e-site' ); - }, 120_000 ); - - afterAll( async () => { - if ( ! env ) { - return; - } - await runCli( [ 'site', 'stop', '--all' ], env ); - cleanupCliEnv( env ); - env = undefined; - }, 60_000 ); - - it( 'renames a site via site set --name', { tags: [ 'e2e' ], timeout: 60_000 }, async () => { - if ( ! env ) { - throw new Error( 'CLI e2e env was not initialised' ); - } - - const newName = 'Renamed E2E Site'; - const result = await runCli( [ 'site', 'set', '--path', sitePath, '--name', newName ], env ); - expect( result.code, result.stderr ).toBe( 0 ); - - expect( findSite( env, sitePath )?.name ).toBe( newName ); - } ); - - it( - 'changes the PHP version via site set --php', - { tags: [ 'e2e' ], timeout: 60_000 }, - async () => { - if ( ! env ) { - throw new Error( 'CLI e2e env was not initialised' ); - } - - const currentPhp = findSite( env, sitePath )?.phpVersion; - const targetPhp = SupportedPHPVersions.find( ( version ) => version !== currentPhp ); - if ( ! targetPhp ) { - throw new Error( 'No alternative supported PHP version available to test against' ); - } - - const result = await runCli( - [ 'site', 'set', '--path', sitePath, '--php', targetPhp ], - env - ); - expect( result.code, result.stderr ).toBe( 0 ); - - expect( findSite( env, sitePath )?.phpVersion ).toBe( targetPhp ); - } - ); - - it( - 'updates the WordPress site title via wp option update blogname', - { tags: [ 'e2e' ], timeout: 180_000 }, - async () => { - if ( ! env ) { - throw new Error( 'CLI e2e env was not initialised' ); - } - - // blogname lives in the WordPress database, which only exists once the site - // has been started — `create --no-start` copies core files but defers the - // WordPress install to the first server start. - const startResult = await runCli( - [ 'site', 'start', '--path', sitePath, '--skip-browser', '--skip-log-details' ], - env - ); - expect( startResult.code, startResult.stderr ).toBe( 0 ); - - const newTitle = 'Renamed via WP-CLI'; - const updateResult = await runCli( - [ 'wp', 'option', 'update', 'blogname', newTitle, '--path', sitePath ], - env - ); - expect( updateResult.code, updateResult.stderr ).toBe( 0 ); - - const getResult = await runCli( - [ 'wp', 'option', 'get', 'blogname', '--path', sitePath ], - env - ); - expect( getResult.code, getResult.stderr ).toBe( 0 ); - // PHP deprecation notices can precede the value on stdout, so assert - // against the last non-empty line rather than the whole buffer. - const lines = getResult.stdout - .split( '\n' ) - .map( ( line ) => line.trim() ) - .filter( Boolean ); - expect( lines.at( -1 ) ).toBe( newTitle ); - } - ); - } ); - - // Deleting is destructive, so each case gets its own freshly created site. - describe( 'deleting a site', () => { - let env: CliEnv | undefined; - - afterEach( async () => { - if ( ! env ) { - return; - } - await runCli( [ 'site', 'stop', '--all' ], env ); - cleanupCliEnv( env ); - env = undefined; - }, 60_000 ); - - it( - 'deletes a site but keeps its files with --no-files', - { tags: [ 'e2e' ], timeout: 120_000 }, - async () => { - env = setupCliEnv(); - const sitePath = await createStoppedSite( - env, - 'Keep Files E2E Site', - 'keep-files-e2e-site' - ); - expect( findSite( env, sitePath ) ).toBeTruthy(); - - const result = await runCli( [ 'site', 'delete', '--path', sitePath, '--no-files' ], env ); - expect( result.code, result.stderr ).toBe( 0 ); - - expect( findSite( env, sitePath ) ).toBeUndefined(); - expect( fs.existsSync( path.join( sitePath, 'wp-load.php' ) ) ).toBe( true ); - } - ); - - it( - 'deletes a site and removes its directory', - { tags: [ 'e2e' ], timeout: 120_000 }, - async () => { - env = setupCliEnv(); - const sitePath = await createStoppedSite( - env, - 'Remove Files E2E Site', - 'remove-files-e2e-site' - ); - expect( findSite( env, sitePath ) ).toBeTruthy(); - expect( fs.existsSync( sitePath ) ).toBe( true ); - - const result = await runCli( [ 'site', 'delete', '--path', sitePath ], env ); - expect( result.code, result.stderr ).toBe( 0 ); - - // Default delete moves the directory to trash, so it's gone from disk. - expect( findSite( env, sitePath ) ).toBeUndefined(); - expect( fs.existsSync( sitePath ) ).toBe( false ); - } - ); - } ); -} ); diff --git a/apps/cli/commands/site/tests/start-stop.e2e.test.ts b/apps/cli/commands/site/tests/start-stop.e2e.test.ts deleted file mode 100644 index 1df0e4d774..0000000000 --- a/apps/cli/commands/site/tests/start-stop.e2e.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -/** - * @vitest-environment node - * - * Real end-to-end test for the `studio site start` / `stop` lifecycle: unlike - * start.test.ts / stop.test.ts (which mock the daemon), this spawns the built CLI, - * boots a real WordPress server, and checks the live state via `studio site list`. - * - * Needs the CLI built first (skips otherwise). Tagged `e2e` (slower manual suite): - * `npm test -- --tagsFilter='e2e'`. - */ -import path from 'path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { - cleanupCliEnv, - cliE2ePrerequisitesMet, - runCli, - setupCliEnv, - type CliEnv, -} from './helpers/cli-e2e'; - -/** - * Whether the CLI reports the site at `sitePath` as running, via - * `studio site list --format json` (stdout is clean JSON; progress goes to stderr). - */ -async function isSiteRunning( env: CliEnv, sitePath: string ): Promise< boolean > { - const result = await runCli( [ 'site', 'list', '--format', 'json' ], env ); - expect( result.code, result.stderr ).toBe( 0 ); - const sites = JSON.parse( result.stdout.trim() ) as Array< { - path?: string; - running?: boolean; - } >; - return sites.find( ( site ) => site.path === sitePath )?.running === true; -} - -describe.skipIf( ! cliE2ePrerequisitesMet() )( 'CLI e2e: studio site start/stop', () => { - let env: CliEnv | undefined; - let sitePath = ''; - - // Create the site once (no --start); the ordered cases below share it — start - // must precede stop, and one create avoids a second slow WordPress copy. - // - // `--runtime sandbox` (bundled Playground/WASM) keeps this hermetic. Native PHP - // would download its ~25 MB binary into the config dir on first run, so covering - // it hermetically needs CI to provision that binary — a follow-up. - beforeAll( async () => { - env = setupCliEnv(); - sitePath = path.join( env.sitesDir, 'lifecycle-e2e-site' ); - - const result = await runCli( - [ - 'site', - 'create', - '--name', - 'Lifecycle E2E Site', - '--path', - sitePath, - '--wp', - 'latest', - '--runtime', - 'sandbox', - '--no-start', - '--skip-browser', - '--skip-log-details', - ], - env - ); - expect( result.code, result.stderr ).toBe( 0 ); - }, 120_000 ); - - // Stop everything and remove the isolated env even if a case failed, so no - // daemon/server/port leaks. `stop --all` also kills the isolated daemon. - afterAll( async () => { - if ( ! env ) { - return; - } - await runCli( [ 'site', 'stop', '--all' ], env ); - cleanupCliEnv( env ); - env = undefined; - }, 60_000 ); - - it( 'starts a site', { tags: [ 'e2e' ], timeout: 180_000 }, async () => { - if ( ! env ) { - throw new Error( 'CLI e2e env was not initialised' ); - } - - const result = await runCli( - [ 'site', 'start', '--path', sitePath, '--skip-browser', '--skip-log-details' ], - env - ); - expect( result.code, result.stderr ).toBe( 0 ); - - expect( await isSiteRunning( env, sitePath ) ).toBe( true ); - } ); - - it( 'stops a site', { tags: [ 'e2e' ], timeout: 120_000 }, async () => { - if ( ! env ) { - throw new Error( 'CLI e2e env was not initialised' ); - } - - const result = await runCli( [ 'site', 'stop', '--path', sitePath ], env ); - expect( result.code, result.stderr ).toBe( 0 ); - - expect( await isSiteRunning( env, sitePath ) ).toBe( false ); - } ); -} ); diff --git a/vitest.shared.ts b/vitest.shared.ts index 94daf20cb0..ce71a12dcc 100644 --- a/vitest.shared.ts +++ b/vitest.shared.ts @@ -5,15 +5,6 @@ export default defineConfig( { test: { pool: 'threads', globals: true, - // Registered test tags for selective runs, e.g. `npm test -- --tagsFilter='e2e'` - // or excluding the slow real-CLI tests with `--tagsFilter='!e2e'`. - tags: [ - { - name: 'e2e', - description: - 'Real end-to-end tests that spawn the built CLI and create real sites. Require `npm run cli:build` first; run in the slower (release/manual) suite, not per-PR.', - }, - ], environment: 'jsdom', environmentOptions: { customExportConditions: [ 'node', 'node-addons' ],