diff --git a/justfile b/justfile index 634343fd747..93b18c423aa 100644 --- a/justfile +++ b/justfile @@ -142,6 +142,7 @@ eslint: //src/cloudflare:cloudflare@eslint \ //src/node:node@eslint \ //src/pyodide:pyodide_static@eslint \ + //src/pyodide/tools:patch_pyodide_asm_lib@eslint \ //src/wpt:wpt-all@tsproject@eslint \ //types:types_lib@eslint diff --git a/package.json b/package.json index c427612ca98..18091a13eae 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,10 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", + "@types/estree": "^1.0.9", "@types/node": "^25.9.5", + "acorn": "^8.18.0", + "astring": "^1.9.0", "capnp-es": "0.0.16", "chrome-remote-interface": "^0.34.0", "esbuild": "^0.27.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec30c6c00be..bfe029847ac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,18 @@ importers: '@eslint/js': specifier: ^10.0.1 version: 10.0.1(eslint@10.8.1) + '@types/estree': + specifier: ^1.0.9 + version: 1.0.9 '@types/node': specifier: ^25.9.5 version: 25.9.5 + acorn: + specifier: ^8.18.0 + version: 8.18.0 + astring: + specifier: ^1.9.0 + version: 1.9.0 capnp-es: specifier: 0.0.16 version: 0.0.16(typescript@6.0.3) @@ -1615,6 +1624,10 @@ packages: array-timsort@1.0.3: resolution: {integrity: sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ==} + astring@1.9.0: + resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} + hasBin: true + asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} @@ -4234,6 +4247,8 @@ snapshots: array-timsort@1.0.3: {} + astring@1.9.0: {} + asynckit@0.4.0: {} aws4fetch@1.0.20: {} diff --git a/src/pyodide/helpers.bzl b/src/pyodide/helpers.bzl index d9088b61abf..95af7cb1bf0 100644 --- a/src/pyodide/helpers.bzl +++ b/src/pyodide/helpers.bzl @@ -1,4 +1,5 @@ load("@aspect_rules_esbuild//esbuild:defs.bzl", "esbuild") +load("@aspect_rules_js//js:defs.bzl", "js_run_binary") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_skylib//rules:expand_template.bzl", "expand_template") load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library") @@ -30,6 +31,18 @@ def _copy_to_generated(src, version = None, out_name = None, name = None): name += "@" + version copy_file(name = name, src = src, out = _out_path(out_name, version)) +def _bin_relative_path(path): + # js_run_binary runs its tool with the bin directory as the working directory. Returns the + # path, relative to that directory, of a file in the current package. Works whether this + # package is in the main repository or (as when workerd is a dependency) an external one. + package = native.package_name() + if package: + path = package + "/" + path + repo = native.repo_name() + if repo: + path = "external/" + repo + "/" + path + return path + def _copy_and_capnp_embed(src): out_name = _out_name(src) _copy_to_generated(src) @@ -159,132 +172,6 @@ def pyodide_static(): tsconfig_json = "tsconfig.json", ) -_PRELUDE = """ -import { - addEventListener, - getRandomValues, - location, - monotonicDateNow, - newWasmModule, - patchedApplyFunc, - patchedLoadLibData, - reportUndefinedSymbolsPatched, - wasmInstantiate, - patched_PyEM_CountFuncParams, -} from "pyodide-internal:pool/builtin_wrappers"; -""" - -# pyodide.asm.mjs patches -# TODO: all of these should be fixed by linking our own Pyodide or by upstreaming. -_REPLACEMENTS_COMMON = [ - [ - "new WebAssembly.Module", - "newWasmModule", - ], - [ - "WebAssembly.instantiate", - "wasmInstantiate", - ], - [ - "Date.now", - "monotonicDateNow", - ], - [ - "reportUndefinedSymbols()", - "reportUndefinedSymbolsPatched(Module)", - ], - [ - "crypto.getRandomValues(", - "getRandomValues(Module, ", - ], - [ - # Direct eval disallowed in esbuild, see: - # https://esbuild.github.io/content-types/#direct-eval - "eval(func)", - "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()", - ], - [ - "eval(data)", - "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()", - ], - [ - "eval(UTF8ToString(ptr))", - "(() => {throw new Error('Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file\\'s contents')})()", - ], - # Dynamic linking patches: - # library lookup - [ - "function loadLibData(){", - """ - function loadLibData(){ - var libData = patchedLoadLibData(Module, libName, flags.rpath); - return flags.loadAsync ? Promise.resolve(libData) : libData; - } - function dummiedOutOrigLoadLibData(){ - """, - ], - # for ensuring memory base of dynlib is stable when restoring snapshots - [ - "getMemory(", - "Module.getMemoryPatched(Module, libName, ", - ], - [ - "function _PyEM_CountFuncParams(func){", - "function _PyEM_CountFuncParams(func){ return patched_PyEM_CountFuncParams(Module, func);", - ], - [ - "var tableBase=metadata.tableSize?wasmTable.length:0;", - "var tableBase=metadata.tableSize?wasmTable.length:0;" + - "Module.snapshotDebug && console.log('loadWebAssemblyModule', libName, memoryBase, tableBase);", - ], - # to ensure we report every fatal error, not just the first one - [ - 'console.error("Recursive call to fatal_error. Inner error was:");', - 'console.error("Recursive call to fatal_error. Inner error was:");\n' + - "try { API.on_fatal?.(e); } catch(e2) { console.error(e2); }\n", - ], -] - -_REPLACEMENTS_COMMON_0_26_0_28 = [ - # for 0.28.2 or earlier, pyodide.asm.js was a commonjs module - [ - # Convert pyodide.asm.js into an es6 module. - # When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us - # automatically. - "var _createPyodideModule", - _PRELUDE + "export const _createPyodideModule", - ], - [ - "globalThis._createPyodideModule = _createPyodideModule;", - "", - ], - # to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7 - [ - "nullToUndefined(func.apply(", - "nullToUndefined(patchedApplyFunc(API, func, ", - ], - [ - "nullToUndefined(Function.prototype.apply.apply", - "nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply", - ], -] - -_REPLACEMENTS_COMMON_314 = [ - # for 314 or later, pyodide.asm.mjs is es6 module - [ - "export default _createPyodideModule;", - # still expose _createPyodideModule for compatibility (import { _createPyodideModule }) - _PRELUDE + "export default _createPyodideModule; export { _createPyodideModule };", - ], -] - -_REPLACEMENTS = { - "0.26.0a2": _REPLACEMENTS_COMMON + _REPLACEMENTS_COMMON_0_26_0_28, - "0.28.2": _REPLACEMENTS_COMMON + _REPLACEMENTS_COMMON_0_26_0_28, - "314.0.4": _REPLACEMENTS_COMMON + _REPLACEMENTS_COMMON_314, - "314.0.6": _REPLACEMENTS_COMMON + _REPLACEMENTS_COMMON_314, -} - def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_mjs = None, python_stdlib_zip = None, emscripten_setup_override = None): pyodide_package = "@pyodide-%s//" % version if not pyodide_asm_wasm: @@ -304,11 +191,26 @@ def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_mjs = None, _copy_to_generated(python_stdlib_zip, version, out_name = "python_stdlib.zip") - expand_template( + # Apply workerd's patches to the Emscripten-generated module (see tools/patch_pyodide_asm.ts). + # The upstream file is first copied into this package so that both the tool's input and + # output can be addressed relative to the bin directory it runs in. + upstream_asm_mjs = _out_path("pyodide.asm.upstream.mjs", version) + patched_asm_mjs = _out_path("pyodide.asm.mjs", version) + _copy_to_generated(pyodide_asm_mjs, version, out_name = "pyodide.asm.upstream.mjs") + js_run_binary( name = "pyodide.asm.mjs@rule@" + version, - out = _out_path("pyodide.asm.mjs", version), - substitutions = dict(_REPLACEMENTS[version]), - template = pyodide_asm_mjs, + srcs = [upstream_asm_mjs], + outs = [patched_asm_mjs], + args = [ + "--version", + version, + "--input", + _bin_relative_path(upstream_asm_mjs), + "--output", + _bin_relative_path(patched_asm_mjs), + ], + mnemonic = "PatchPyodideAsm", + tool = Label("//src/pyodide/tools:patch_pyodide_asm"), ) js_file( diff --git a/src/pyodide/tools/BUILD.bazel b/src/pyodide/tools/BUILD.bazel new file mode 100644 index 00000000000..222c2e1932f --- /dev/null +++ b/src/pyodide/tools/BUILD.bazel @@ -0,0 +1,24 @@ +load("@aspect_rules_js//js:defs.bzl", "js_binary") +load("//:build/wd_ts_project.bzl", "wd_ts_project") + +# Build-time tool that applies workerd's patches to Pyodide's pyodide.asm.js / pyodide.asm.mjs as +# AST rewrites. Used by python_bundles() in //src/pyodide:helpers.bzl. +wd_ts_project( + name = "patch_pyodide_asm_lib", + srcs = ["patch_pyodide_asm.ts"], + eslintrc_json = "eslint.config.mjs", + tsconfig_json = "tsconfig.json", + deps = [ + "//:node_modules/@types/estree", + "//:node_modules/@types/node", + "//:node_modules/acorn", + "//:node_modules/astring", + ], +) + +js_binary( + name = "patch_pyodide_asm", + data = [":patch_pyodide_asm_lib"], + entry_point = "patch_pyodide_asm.js", + visibility = ["//src/pyodide:__pkg__"], +) diff --git a/src/pyodide/tools/eslint.config.mjs b/src/pyodide/tools/eslint.config.mjs new file mode 100644 index 00000000000..a6874e4c1e0 --- /dev/null +++ b/src/pyodide/tools/eslint.config.mjs @@ -0,0 +1,3 @@ +import { baseConfig } from '../../../tools/base.eslint.config.mjs'; + +export default baseConfig(); diff --git a/src/pyodide/tools/patch_pyodide_asm.ts b/src/pyodide/tools/patch_pyodide_asm.ts new file mode 100644 index 00000000000..5f1ef51b410 --- /dev/null +++ b/src/pyodide/tools/patch_pyodide_asm.ts @@ -0,0 +1,556 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Build-time tool that patches Pyodide's Emscripten-generated `pyodide.asm.js` / `pyodide.asm.mjs` +// so that it can run inside workerd. The file is parsed into an ESTree AST with acorn, the patches +// below are applied as AST rewrites, and the result is printed back to JavaScript with astring. +// +// Each patch declares how many times it must match in each Pyodide version. A mismatch fails the +// build, so a Pyodide upgrade that moves or removes a patch site is caught immediately instead of +// silently shipping an unpatched runtime. +// +// TODO: all of these should be fixed by linking our own Pyodide or by upstreaming. +// +// Usage: patch_pyodide_asm --version --input --output + +import { readFileSync, writeFileSync } from 'node:fs'; +import { parseArgs } from 'node:util'; +import { + parse, + parseExpressionAt, + type AnyNode, + type CallExpression, + type Declaration, + type ExportNamedDeclaration, + type Expression, + type FunctionDeclaration, + type Identifier, + type ModuleDeclaration, + type NewExpression, + type Node, + type Options, + type Statement, + type VariableDeclaration, +} from 'acorn'; +import { generate } from 'astring'; +import type { Node as EstreeNode } from 'estree'; + +const ACORN_OPTIONS: Options = { ecmaVersion: 'latest', sourceType: 'module' }; +// Replacement snippets are parsed out of context, so relax the checks that depend on it. +const SNIPPET_OPTIONS: Options = { + ...ACORN_OPTIONS, + allowReturnOutsideFunction: true, +}; + +// Versions whose pyodide.asm.js is a CommonJS/UMD-style script that we convert into an ES module. +// Later versions ship pyodide.asm.mjs which is already an ES module. +const COMMONJS_VERSIONS = ['0.26.0a2', '0.28.2']; + +const PRELUDE = ` +import { + addEventListener, + getRandomValues, + location, + monotonicDateNow, + newWasmModule, + patchedApplyFunc, + patchedLoadLibData, + reportUndefinedSymbolsPatched, + wasmInstantiate, + patched_PyEM_CountFuncParams, +} from "pyodide-internal:pool/builtin_wrappers"; +`; + +// Direct eval is disallowed in esbuild, see: https://esbuild.github.io/content-types/#direct-eval +const EVAL_REPLACEMENT = `(() => { + throw new Error( + "Internal Emscripten code tried to eval, this should not happen, please file a bug report with your requirements.txt file's contents" + ); +})()`; + +// --------------------------------------------------------------------------------------------- +// AST helpers + +type TopLevelStatement = Statement | ModuleDeclaration; + +/** + * What holds the node being visited: the parent node when the node sits in a single-node field, + * the statement/element list when it sits in an array, or null for the root. + */ +type Container = AnyNode | AnyNode[] | null; + +/** Parse a snippet into a single expression node. */ +function expr(code: string): Expression { + return parseExpressionAt(code, 0, SNIPPET_OPTIONS); +} + +/** Parse a snippet into a list of statement nodes, which may include import/export declarations. */ +function stmts(code: string): TopLevelStatement[] { + return parse(code, SNIPPET_OPTIONS).body; +} + +/** Parse a snippet into exactly one plain (non-module-declaration) statement node. */ +function stmt(code: string): Statement { + const body = stmts(code); + const [only] = body; + if (body.length !== 1 || only === undefined) { + throw new Error(`expected exactly one statement in snippet: ${code}`); + } + if ( + only.type === 'ImportDeclaration' || + only.type === 'ExportNamedDeclaration' || + only.type === 'ExportDefaultDeclaration' || + only.type === 'ExportAllDeclaration' + ) { + throw new Error(`expected a plain statement, got a ${only.type}: ${code}`); + } + return only; +} + +/** + * Build a node that has no position in the source. astring only reads positions when emitting a + * source map, which we don't. + */ +function synthetic(fields: Omit): T { + return { start: 0, end: 0, ...fields } as T; +} + +function identifier(name: string): Identifier { + return synthetic({ type: 'Identifier', name }); +} + +function isNode(value: unknown): value is AnyNode { + return ( + typeof value === 'object' && + value !== null && + typeof (value as { type?: unknown }).type === 'string' + ); +} + +function isIdentifier( + node: AnyNode | null | undefined, + name: string +): node is Identifier { + return node?.type === 'Identifier' && node.name === name; +} + +/** + * Returns the dotted path of a non-computed member chain rooted at an identifier, e.g. + * "Function.prototype.apply.apply", or undefined if the node is not such a chain. + */ +function memberPath(node: AnyNode): string | undefined { + if (node.type === 'Identifier') { + return node.name; + } + if ( + node.type === 'MemberExpression' && + !node.computed && + node.property.type === 'Identifier' + ) { + const objectPath = memberPath(node.object); + if (objectPath !== undefined) { + return `${objectPath}.${node.property.name}`; + } + } + return undefined; +} + +function isCallOf(node: AnyNode, calleePath: string): node is CallExpression { + return ( + node.type === 'CallExpression' && memberPath(node.callee) === calleePath + ); +} + +function isNewOf(node: AnyNode, calleePath: string): node is NewExpression { + return ( + node.type === 'NewExpression' && memberPath(node.callee) === calleePath + ); +} + +function isFunctionDeclarationNamed( + node: AnyNode, + name: string +): node is FunctionDeclaration { + return node.type === 'FunctionDeclaration' && isIdentifier(node.id, name); +} + +/** True for `var/let/const ...` declaring exactly one identifier. */ +function isVariableDeclarationOf( + node: AnyNode, + name: string +): node is VariableDeclaration { + return ( + node.type === 'VariableDeclaration' && + node.declarations.length === 1 && + isIdentifier(node.declarations[0]?.id, name) + ); +} + +/** Build `callee(...args)` from a callee snippet and argument nodes. */ +function call( + calleeCode: string, + args: CallExpression['arguments'] +): CallExpression { + return synthetic({ + type: 'CallExpression', + callee: expr(calleeCode), + arguments: args, + optional: false, + }); +} + +/** Build `export ` or, without a declaration, `export { ...names };`. */ +function exportNamed({ + declaration = null, + names = [], +}: { + declaration?: Declaration | null; + names?: string[]; +}): ExportNamedDeclaration { + return synthetic({ + type: 'ExportNamedDeclaration', + declaration, + specifiers: names.map((name) => + synthetic({ + type: 'ExportSpecifier', + local: identifier(name), + exported: identifier(name), + }) + ), + source: null, + attributes: [], + }); +} + +// --------------------------------------------------------------------------------------------- +// Patches +// +// The tree is visited post-order, so replacements are never re-visited and patches never see each +// other's output. + +/** Number of required matches, either for every version or per version with a `default`. */ +type ExpectedCount = number | Record; + +/** A replacement node, or, when the matched node lives in a statement list, an array of statements that replaces it (empty deletes it). */ +type Replacement = AnyNode | AnyNode[]; + +interface Patch { + name: string; + expected: ExpectedCount; + /** Whether this node is a patch site. `parent` is what contains the node (see Container). */ + match: (node: AnyNode, parent: Container) => boolean; + replace: (node: AnyNode) => Replacement; +} + +/** + * Define a patch whose `match` is a type predicate, so that `replace` receives the narrowed node + * type. `replace` is only ever called with nodes that `match` accepted. + */ +function patch(spec: { + name: string; + expected: ExpectedCount; + match: (node: AnyNode, parent: Container) => node is T; + replace: (node: T) => Replacement; +}): Patch { + return { ...spec, replace: (node) => spec.replace(node as T) }; +} + +const COMMON_PATCHES: Patch[] = [ + patch({ + name: 'new WebAssembly.Module(...) -> newWasmModule(...)', + expected: { '0.26.0a2': 6, default: 4 }, + match: (node): node is NewExpression => isNewOf(node, 'WebAssembly.Module'), + replace: (node) => call('newWasmModule', node.arguments), + }), + { + name: 'WebAssembly.instantiate -> wasmInstantiate', + expected: 2, + match: (node) => memberPath(node) === 'WebAssembly.instantiate', + replace: () => expr('wasmInstantiate'), + }, + { + name: 'Date.now -> monotonicDateNow', + expected: { '0.26.0a2': 18, default: 22 }, + match: (node) => memberPath(node) === 'Date.now', + replace: () => expr('monotonicDateNow'), + }, + { + name: 'reportUndefinedSymbols() -> reportUndefinedSymbolsPatched(Module)', + expected: 3, + match: (node) => + isCallOf(node, 'reportUndefinedSymbols') && node.arguments.length === 0, + replace: () => expr('reportUndefinedSymbolsPatched(Module)'), + }, + patch({ + name: 'crypto.getRandomValues(...) -> getRandomValues(Module, ...)', + expected: 1, + match: (node): node is CallExpression => + isCallOf(node, 'crypto.getRandomValues'), + replace: (node) => + call('getRandomValues', [expr('Module'), ...node.arguments]), + }), + { + name: 'direct eval(...) -> throw', + expected: 6, + match: (node) => isCallOf(node, 'eval'), + replace: () => expr(EVAL_REPLACEMENT), + }, + // Dynamic linking patches. + patch({ + // Library lookup: route dynamic library loading through our own loader, keeping the original + // function around (renamed) so that references to any of its locals stay valid. + name: 'function loadLibData() -> patchedLoadLibData', + expected: 1, + match: (node): node is FunctionDeclaration => + isFunctionDeclarationNamed(node, 'loadLibData'), + replace: (node) => [ + stmt(` + function loadLibData() { + var libData = patchedLoadLibData(Module, libName, flags.rpath); + return flags.loadAsync ? Promise.resolve(libData) : libData; + } + `), + { ...node, id: identifier('dummiedOutOrigLoadLibData') }, + ], + }), + patch({ + // Ensure the memory base of a dynlib is stable when restoring snapshots. + name: 'getMemory(...) -> Module.getMemoryPatched(Module, libName, ...)', + expected: 1, + match: (node): node is CallExpression => isCallOf(node, 'getMemory'), + replace: (node) => + call('Module.getMemoryPatched', [ + expr('Module'), + expr('libName'), + ...node.arguments, + ]), + }), + patch({ + // Only 0.26.0a2 still has this function; later versions restructured it upstream. + name: 'function _PyEM_CountFuncParams(func) -> patched_PyEM_CountFuncParams', + expected: { '0.26.0a2': 1, default: 0 }, + match: (node): node is FunctionDeclaration => + isFunctionDeclarationNamed(node, '_PyEM_CountFuncParams'), + replace: (node) => ({ + ...node, + body: { + ...node.body, + body: [ + stmt('return patched_PyEM_CountFuncParams(Module, func);'), + ...node.body.body, + ], + }, + }), + }), + { + name: 'log loadWebAssemblyModule after `var tableBase = ...`', + expected: 1, + match: (node, parent) => + isVariableDeclarationOf(node, 'tableBase') && Array.isArray(parent), + replace: (node) => [ + node, + stmt( + "Module.snapshotDebug && console.log('loadWebAssemblyModule', libName, memoryBase, tableBase);" + ), + ], + }, +]; + +// pyodide.asm.js in these versions is a CommonJS/UMD-style script; convert it to an ES module. +// When we link our own Pyodide we can pass `-sES6_MODULE` to the linker and it will do this for us +// automatically. +const COMMONJS_PATCHES: Patch[] = [ + patch({ + name: 'var _createPyodideModule -> prelude + export const _createPyodideModule', + expected: 1, + match: (node, parent): node is VariableDeclaration => + Array.isArray(parent) && + isVariableDeclarationOf(node, '_createPyodideModule'), + replace: (node) => [ + ...stmts(PRELUDE), + exportNamed({ declaration: { ...node, kind: 'const' } }), + ], + }), + { + name: 'remove `globalThis._createPyodideModule = _createPyodideModule;`', + expected: 1, + match: (node, parent) => + Array.isArray(parent) && + node.type === 'ExpressionStatement' && + node.expression.type === 'AssignmentExpression' && + node.expression.operator === '=' && + memberPath(node.expression.left) === 'globalThis._createPyodideModule' && + isIdentifier(node.expression.right, '_createPyodideModule'), + replace: () => [], + }, + patch({ + // To fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7, which is included + // upstream from 0.28 onwards. + name: 'nullToUndefined(func.apply(...)) -> nullToUndefined(patchedApplyFunc(API, func, ...))', + expected: { '0.26.0a2': 2, default: 0 }, + match: (node): node is CallExpression & { arguments: [CallExpression] } => + isCallOf(node, 'nullToUndefined') && + node.arguments.length === 1 && + node.arguments[0] !== undefined && + isCallOf(node.arguments[0], 'func.apply'), + replace: (node) => + call('nullToUndefined', [ + call('patchedApplyFunc', [ + expr('API'), + expr('func'), + ...node.arguments[0].arguments, + ]), + ]), + }), +]; + +// pyodide.asm.mjs in later versions is already an ES module. +const ES_MODULE_PATCHES: Patch[] = [ + { + name: 'export default _createPyodideModule -> prelude + default and named export', + expected: 1, + match: (node, parent) => + Array.isArray(parent) && + node.type === 'ExportDefaultDeclaration' && + isIdentifier(node.declaration, '_createPyodideModule'), + replace: (node) => [ + ...stmts(PRELUDE), + node, + // Still expose _createPyodideModule for compatibility (import { _createPyodideModule }). + exportNamed({ names: ['_createPyodideModule'] }), + ], + }, +]; + +function patchesForVersion(version: string): Patch[] { + const extra = COMMONJS_VERSIONS.includes(version) + ? COMMONJS_PATCHES + : ES_MODULE_PATCHES; + return [...COMMON_PATCHES, ...extra]; +} + +function expectedCount(spec: Patch, version: string): number { + if (typeof spec.expected === 'number') { + return spec.expected; + } + const count = spec.expected[version] ?? spec.expected['default']; + if (count === undefined) { + throw new Error( + `patch "${spec.name}" has no expected count for version ${version}` + ); + } + return count; +} + +// --------------------------------------------------------------------------------------------- +// Tree walking + +/** + * Post-order traversal applying the patches. `container[key]` is the child being visited: either a + * node in an object field, or a node at an index in a statement/element array. Returns how much + * the containing array grew (or shrank) if the node was replaced by a statement list, so that the + * caller can resume iteration after the replacement. + */ +function visit( + node: AnyNode, + container: Container, + key: string | number, + patches: Patch[], + counts: Map +): number { + // Visit children first so that replacements are never re-visited. + const fields = node as unknown as Record; + for (const childKey of Object.keys(fields)) { + const child = fields[childKey]; + if (Array.isArray(child)) { + // Iterate by index so that splices performed by replacements are accounted for. + const list = child as unknown[]; + for (let i = 0; i < list.length; i++) { + const element = list[i]; + if (isNode(element)) { + i += visit(element, list as AnyNode[], i, patches, counts); + } + } + } else if (isNode(child)) { + visit(child, node, childKey, patches, counts); + } + } + + for (const spec of patches) { + if (!spec.match(node, container)) { + continue; + } + counts.set(spec.name, (counts.get(spec.name) ?? 0) + 1); + const replacement = spec.replace(node); + if (Array.isArray(replacement)) { + if (!Array.isArray(container) || typeof key !== 'number') { + throw new Error( + `patch "${spec.name}" produced a statement list but matched a node that is not in a list` + ); + } + container.splice(key, 1, ...replacement); + // A subsequent patch cannot match this node anymore, so return immediately. + return replacement.length - 1; + } + if (container === null) { + throw new Error(`patch "${spec.name}" tried to replace the root node`); + } + if (Array.isArray(container)) { + container[key as number] = replacement; + } else { + (container as unknown as Record)[key] = replacement; + } + return 0; + } + return 0; +} + +function patchSource(source: string, version: string): string { + const patches = patchesForVersion(version); + const ast = parse(source, ACORN_OPTIONS); + const counts = new Map(); + visit(ast, null, 0, patches, counts); + + const mismatches: string[] = []; + for (const spec of patches) { + const actual = counts.get(spec.name) ?? 0; + const expected = expectedCount(spec, version); + if (actual !== expected) { + mismatches.push( + ` ${spec.name}: expected ${expected} match(es), got ${actual}` + ); + } + } + if (mismatches.length > 0) { + throw new Error( + `Pyodide ${version}: patch site count mismatch. Either Pyodide changed, or the expected ` + + `counts in patch_pyodide_asm.ts need to be updated:\n${mismatches.join('\n')}` + ); + } + + // acorn's node types are a superset of ESTree's (they carry `start`/`end`), but the two type + // libraries model literals differently, so the compiler does not consider them assignable. + return generate(ast as unknown as EstreeNode); +} + +function main(): void { + const { values } = parseArgs({ + options: { + version: { type: 'string' }, + input: { type: 'string' }, + output: { type: 'string' }, + }, + strict: true, + }); + const { version, input, output } = values; + if (version === undefined || input === undefined || output === undefined) { + throw new Error( + 'usage: --version --input --output ' + ); + } + const source = readFileSync(input, 'utf8'); + writeFileSync(output, patchSource(source, version)); +} + +main(); diff --git a/src/pyodide/tools/tsconfig.json b/src/pyodide/tools/tsconfig.json new file mode 100644 index 00000000000..da159952d63 --- /dev/null +++ b/src/pyodide/tools/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tools/base.tsconfig.json", + "compilerOptions": { + "module": "commonjs", + "lib": ["ESNext"], + "types": ["@types/node"], + "sourceMap": true, + "rootDir": ".", + "verbatimModuleSyntax": false, + "noEmit": false + }, + "include": ["*.ts"] +} diff --git a/src/pyodide/tsconfig.json b/src/pyodide/tsconfig.json index fe0dde7149c..bd8d7d8d7c8 100644 --- a/src/pyodide/tsconfig.json +++ b/src/pyodide/tsconfig.json @@ -14,5 +14,5 @@ "allowArbitraryExtensions": true }, "include": ["./**/*"], - "exclude": ["./**/generated/**/*", "node_modules", "**/*.mjs"] + "exclude": ["./**/generated/**/*", "./tools/**/*", "node_modules", "**/*.mjs"] }