diff --git a/.buildconfig b/.buildconfig index a518dff8eedcc..ac9d2913c9535 100644 --- a/.buildconfig +++ b/.buildconfig @@ -1,6 +1,6 @@ PG_VERSION=17.5 PG_BRANCH=REL_17_5_WASM -SDK_VERSION=3.1.74.12.0 +SDK_VERSION=3.1.74.11.11 SDKROOT=/tmp/sdk GETZIC=false ZIC=/usr/sbin/zic diff --git a/.gitignore b/.gitignore index e76ef7cc785ea..2e67d7c8e6671 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,12 @@ lib*.pc /tmp_install/ /portlock/ /dist + +# emscripten build excludes +*.cjs +*.wasm +pglite.data +pglite.js +pglite.html +*.map +pglite-wasm/excluded.imports diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000..9317a32956edc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "pglite/pg_ivm"] + path = pglite/pg_ivm + url = https://github.com/sraoss/pg_ivm.git +[submodule "pglite/pgvector"] + path = pglite/vector + url = https://github.com/pgvector/pgvector.git diff --git a/build-pglite.sh b/build-pglite.sh new file mode 100755 index 0000000000000..8fd4b25572087 --- /dev/null +++ b/build-pglite.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +### NOTES ### +# $INSTALL_PREFIX is expected to point to the installation folder of various libraries built to wasm (see pglite-builder) +############# + +# final output folder +INSTALL_FOLDER=${INSTALL_FOLDER:-"/install/pglite"} + +# build with optimizations by default aka release +PGLITE_CFLAGS="-O2" +if [ "$DEBUG" = true ] +then + echo "pglite: building debug version." + PGLITE_CFLAGS="-g -gsource-map --no-wasm-opt" +else + echo "pglite: building release version." + # we shouldn't need to do this, but there's a bug somewhere that prevents a successful build if this is set + unset DEBUG +fi + +echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS" + +# run ./configure only if config.status is older than this file +# TODO: we should ALSO check if any of the PGLITE_CFLAGS have changed and trigger a ./configure if they did!!! +REF_FILE="build-pglite.sh" +CONFIG_STATUS="config.status" +RUN_CONFIGURE=false + +if [ ! -f "$CONFIG_STATUS" ]; then + echo "$CONFIG_STATUS does not exist, need to run ./configure" + RUN_CONFIGURE=true +elif [ "$REF_FILE" -nt "$CONFIG_STATUS" ]; then + echo "$CONFIG_STATUS is older than $REF_FILE. Need to run ./configure." + RUN_CONFIGURE=true +else + echo "$CONFIG_STATUS exists and is newer than $REF_FILE. ./configure will NOT be run." +fi + +# Step 1: configure the project +if [ "$RUN_CONFIGURE" = true ]; then + LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } +else + echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}" +fi + +# Step 2: make and install all except pglite +emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscripten -j' ; exit 21; } +emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; } + +# Step 3.1: make all contrib extensions - do not install +emmake make PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; } +# Step 3.2: make dist contrib extensions - this will create an archive for each extension +emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; } +# the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive + +# Step 4: make and dist other extensions +SAVE_PATH=$PATH +PATH=$PATH:$INSTALL_FOLDER/bin +emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite || { echo 'error: emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 41; } +emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist ' ; exit 42; } +PATH=$SAVE_PATH + +# Step 5: make and install pglite +# we define here "all" emscripten flags in order to allow native builds (like libpglite) +EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction" +PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ +-sSUPPORT_LONGJMP=emscripten \ +-sFORCE_FILESYSTEM=1 \ +-sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ +-sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \ +-sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \ +-sERROR_ON_UNDEFINED_SYMBOLS=1 \ +-sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS" + +# Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available. +PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } diff --git a/build-with-docker.sh b/build-with-docker.sh new file mode 100755 index 0000000000000..6a849b52ab863 --- /dev/null +++ b/build-with-docker.sh @@ -0,0 +1,13 @@ +# although we could use any path inside docker, using the same path as on the host +# allows the DWARF info (when building in DEBUG) to contain the correct file paths +DOCKER_WORKSPACE=$(pwd) + +docker run $@ \ + --rm \ + -e DEBUG=${DEBUG:-false} \ + --workdir=${DOCKER_WORKSPACE} \ + -v .:${DOCKER_WORKSPACE}:rw \ + -v ./dist:/install/pglite:rw \ + electricsql/pglite-builder:3.1.74_2 \ + ./build-pglite.sh + diff --git a/clean-pglite.sh b/clean-pglite.sh new file mode 100755 index 0000000000000..4f72e3fa60a8b --- /dev/null +++ b/clean-pglite.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +emmake make PORTNAME=emscripten -C src/backend uninstall; emmake make PORTNAME=emscripten -C src/backend clean; +emmake make PORTNAME=emscripten -C pglite/ clean; emmake make PORTNAME=emscripten -C pglite/ uninstall; +emmake make PORTNAME=emscripten -C contrib/ clean; emmake make PORTNAME=emscripten -C contrib/ uninstall; +emmake make PORTNAME=emscripten -C pglite clean; emmake make PORTNAME=emscripten -C pglite uninstall; +emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall; + +echo "removing config.status" +rm config.status \ No newline at end of file diff --git a/contrib/Makefile b/contrib/Makefile index abd780f277405..603636cd4d248 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -90,6 +90,7 @@ endif # Missing: # start-scripts \ (does not have a makefile) +include ./dist.mk $(recurse) $(recurse_always) diff --git a/contrib/dist.mk b/contrib/dist.mk new file mode 100644 index 0000000000000..df3933f01fb82 --- /dev/null +++ b/contrib/dist.mk @@ -0,0 +1,27 @@ +# contrib/dist.mk +# +# Package each contrib extension into its own .tar.gz archive + +prefix ?= /install/pglite +CONTRIB_BUILD_ROOT := /tmp/extensions/build +ARCHIVE_DIR := /install/pglite/extensions + +CONTRIBS := $(SUBDIRS) + +# Default target: build tarballs for all contribs +dist: $(addsuffix .tar.gz,$(CONTRIBS)) + +# Pattern rule: build $(EXT).tar.gz for each contrib +%.tar.gz: + @echo "=== Staging $* ===" + rm -rf $(CONTRIB_BUILD_ROOT)/$* + bash -c 'mkdir -p $(CONTRIB_BUILD_ROOT)/$*/$(prefix)/{bin,lib,share/extension,share/doc,share/postgresql/extension,share/postgresql/tsearch_data,include}' + $(MAKE) -C $* install DESTDIR=$(CONTRIB_BUILD_ROOT)/$* + @echo "=== Packaging $* ===" + mkdir -p $(ARCHIVE_DIR) + cd $(CONTRIB_BUILD_ROOT)/$*/$(prefix) && \ + files=$$(find . -type f -o -type l | sed 's|^\./||') && \ + tar -czf $(ARCHIVE_DIR)/$*.tar.gz $$files +# tar -C $(CONTRIB_BUILD_ROOT)/$*/$(prefix) -czf $(ARCHIVE_DIR)/$*.tar.gz . + +.PHONY: dist diff --git a/docker_rc.sh b/docker_rc.sh new file mode 100644 index 0000000000000..bd9c42f40d1d5 --- /dev/null +++ b/docker_rc.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +WORKSPACE=$(pwd) + +echo " + +SDK prepare + +" + +pushd / + # this is minimum required to **use** sdk on docker/debian 12, not build it + apt-get update && apt-get --yes install git wget curl lz4 xz-utils bison flex pkg-config autoconf make + + if [ -f $WORKSPACE/sdk.tar.lz4 ] + then + tar xf $WORKSPACE/sdk.tar.lz4 --use-compress-program=lz4 + fi + + if [ -d $SDKROOT/wasisdk/upstream ] + then + echo "wasi sdk common support is already installed" + else + tar xf ${WORKSPACE}/prebuilt/wasi-sdk-25.tar.xz + fi + + if [ -d $SDKROOT/wasisdk/upstream/lib ] + then + echo "wasi sdk $(arch) support is already installed" + else + pushd $SDKROOT/wasisdk + if arch|grep -q aarch64 + then + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-arm64-linux.tar.gz -O/tmp/sdk.tar.gz + else + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-linux.tar.gz -O/tmp/sdk.tar.gz + fi + tar xfz /tmp/sdk.tar.gz && rm /tmp/sdk.tar.gz + mv wasi-sdk-25.0-*/{bin,lib} upstream/ + popd + fi + +popd + +echo " + +Setting up SDK shell + + +Applying Hotfixes +------------------------------------------------------------------------------------------ +" + +echo "linker fix" +cp -vf hotfix/library_dylink.js ${SDKROOT}/emsdk/upstream/emscripten/src/ + + + +echo "------------------------------------------------------------------------------------------" + + +${SDKROOT}/wasm32-bi-emscripten-shell.sh diff --git a/hotfix/library_dylink.js b/hotfix/library_dylink.js new file mode 100644 index 0000000000000..8a44339b30670 --- /dev/null +++ b/hotfix/library_dylink.js @@ -0,0 +1,1288 @@ +/** + * @license + * Copyright 2020 The Emscripten Authors + * SPDX-License-Identifier: MIT + * + * Dynamic library loading + */ + +#if !RELOCATABLE +#error "library_dylink.js requires RELOCATABLE" +#endif + +var LibraryDylink = { +#if FILESYSTEM + $registerWasmPlugin__deps: ['$preloadPlugins'], + $registerWasmPlugin: () => { + // Use string keys here to avoid minification since the plugin consumer + // also uses string keys. + var wasmPlugin = { + 'promiseChainEnd': Promise.resolve(), + 'canHandle': (name) => { + return !Module['noWasmDecoding'] && name.endsWith('.so') + }, + 'handle': (byteArray, name, onload, onerror) => { + // loadWebAssemblyModule can not load modules out-of-order, so rather + // than just running the promises in parallel, this makes a chain of + // promises to run in series. + wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name, {})).then( + (exports) => { +#if DYLINK_DEBUG + dbg(`registering preloadedWasm: ${name}`); +#endif + preloadedWasm[name] = exports; + onload(byteArray); + }, + (error) => { + err(`failed to instantiate wasm: ${name}: ${error}`); + onerror(); + }); + } + }; + preloadPlugins.push(wasmPlugin); + }, + + $preloadedWasm__deps: ['$registerWasmPlugin'], + $preloadedWasm__postset: ` + registerWasmPlugin(); + `, + $preloadedWasm: {}, +#endif // FILESYSTEM + + $isSymbolDefined: (symName) => { + // Ignore 'stub' symbols that are auto-generated as part of the original + // `wasmImports` used to instantiate the main module. + var existing = wasmImports[symName]; + if (!existing || existing.stub) { + return false; + } +#if ASYNCIFY + // Even if a symbol exists in wasmImports, and is not itself a stub, it + // could be an ASYNCIFY wrapper function that wraps a stub function. + if (symName in asyncifyStubs && !asyncifyStubs[symName]) { + return false; + } +#endif + return true; + }, + + // Dynamic version of shared.py:make_invoke. This is needed for invokes + // that originate from side modules since these are not known at JS + // generation time. +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + $createInvokeFunction__internal: true, + $createInvokeFunction__deps: ['$dynCall', 'setThrew', '$stackSave', '$stackRestore'], + $createInvokeFunction: (sig) => (ptr, ...args) => { + var sp = stackSave(); + try { + return dynCall(sig, ptr, args); + } catch(e) { + stackRestore(sp); + // Create a try-catch guard that rethrows the Emscripten EH exception. +#if EXCEPTION_STACK_TRACES + // Exceptions thrown from C++ and longjmps will be an instance of + // EmscriptenEH. + if (!(e instanceof EmscriptenEH)) throw e; +#else + // Exceptions thrown from C++ will be a pointer (number) and longjmp + // will throw the number Infinity. Use the compact and fast "e !== e+0" + // test to check if e was not a Number. + if (e !== e+0) throw e; +#endif + _setThrew(1, 0); +#if WASM_BIGINT + // In theory this if statement could be done on + // creating the function, but I just added this to + // save wasting code space as it only happens on exception. + if (sig[0] == "j") return 0n; +#endif + } + }, +#endif + + // Resolve a global symbol by name. This is used during module loading to + // resolve imports, and by `dlsym` when used with `RTLD_DEFAULT`. + // Returns both the resolved symbol (i.e. a function or a global) along with + // the canonical name of the symbol (in some cases is modify the symbol as + // part of the loop process, so that actual symbol looked up has a different + // name). + $resolveGlobalSymbol__deps: ['$isSymbolDefined', +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + '$createInvokeFunction', +#endif + ], + $resolveGlobalSymbol__internal: true, + $resolveGlobalSymbol: (symName, direct = false) => { + var sym; +#if !WASM_BIGINT + // First look for the orig$ symbol which is the symbol without i64 + // legalization performed. + if (direct && ('orig$' + symName in wasmImports)) { + symName = 'orig$' + symName; + } +#endif + if (isSymbolDefined(symName)) { + sym = wasmImports[symName]; + } +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + // Asm.js-style exception handling: invoke wrapper generation + else if (symName.startsWith('invoke_')) { + // Create (and cache) new invoke_ functions on demand. + sym = wasmImports[symName] = createInvokeFunction(symName.split('_')[1]); + } +#endif +#if !DISABLE_EXCEPTION_CATCHING + else if (symName.startsWith('__cxa_find_matching_catch_')) { + // When the main module is linked we create whichever variants of + // `__cxa_find_matching_catch_` (see jsifier.js) that we know are needed, + // but a side module loaded at runtime might need different/additional + // variants so we create those dynamically. + sym = wasmImports[symName] = (...args) => { +#if MEMORY64 + args = args.map(Number); +#endif + var rtn = findMatchingCatch(args); + return {{{ to64('rtn') }}}; + } + } +#endif + return {sym, name: symName}; + }, + + $GOT: {}, + $currentModuleWeakSymbols: '=new Set({{{ JSON.stringify(Array.from(WEAK_IMPORTS)) }}})', + + // Create globals to each imported symbol. These are all initialized to zero + // and get assigned later in `updateGOT` + $GOTHandler__internal: true, + $GOTHandler__deps: ['$GOT', '$currentModuleWeakSymbols'], + $GOTHandler: { + get(obj, symName) { + var rtn = GOT[symName]; + if (!rtn) { + rtn = GOT[symName] = new WebAssembly.Global({'value': '{{{ POINTER_WASM_TYPE }}}', 'mutable': true}); +#if DYLINK_DEBUG == 2 + dbg("new GOT entry: " + symName); +#endif + } + if (!currentModuleWeakSymbols.has(symName)) { + // Any non-weak reference to a symbol marks it as `required`, which + // enabled `reportUndefinedSymbols` to report undefeind symbol errors + // correctly. + rtn.required = true; + } + return rtn; + } + }, + + $isInternalSym__internal: true, + $isInternalSym: (symName) => { + // TODO: find a way to mark these in the binary or avoid exporting them. + return [ + '__cpp_exception', + '__c_longjmp', + '__wasm_apply_data_relocs', + '__dso_handle', + '__tls_size', + '__tls_align', + '__set_stack_limits', + '_emscripten_tls_init', + '__wasm_init_tls', + '__wasm_call_ctors', + '__start_em_asm', + '__stop_em_asm', + '__start_em_js', + '__stop_em_js', + ].includes(symName) || symName.startsWith('__em_js__') +#if SPLIT_MODULE + // Exports synthesized by wasm-split should be prefixed with '%' + || symName[0] == '%' +#endif + ; + }, + + $updateGOT__internal: true, + $updateGOT__deps: ['$GOT', '$isInternalSym', '$addFunction', '$getFunctionAddress'], + $updateGOT: (exports, replace) => { +#if DYLINK_DEBUG + dbg("updateGOT: adding " + Object.keys(exports).length + " symbols"); +#endif + for (var symName in exports) { + if (isInternalSym(symName)) { + continue; + } + + var value = exports[symName]; +#if !WASM_BIGINT + if (symName.startsWith('orig$')) { + symName = symName.split('$')[1]; + replace = true; + } +#endif + + GOT[symName] ||= new WebAssembly.Global({'value': '{{{ POINTER_WASM_TYPE }}}', 'mutable': true}); + if (replace || GOT[symName].value == 0) { +#if DYLINK_DEBUG == 2 + dbg(`updateGOT: before: ${symName} : ${GOT[symName].value}`); +#endif + if (typeof value == 'function') { + GOT[symName].value = {{{ to64('addFunction(value)') }}}; +#if DYLINK_DEBUG == 2 + dbg(`updateGOT: FUNC: ${symName} : ${GOT[symName].value}`); +#endif + } else if (typeof value == {{{ POINTER_JS_TYPE }}}) { + GOT[symName].value = value; + } else { + err(`unhandled export type for '${symName}': ${typeof value}`); + } +#if DYLINK_DEBUG == 2 + dbg(`updateGOT: after: ${symName} : ${GOT[symName].value} (${value})`); +#endif + } +#if DYLINK_DEBUG + else if (GOT[symName].value != value) { + dbg(`updateGOT: EXISTING SYMBOL: ${symName} : ${GOT[symName].value} (${value})`); + } +#endif + } +#if DYLINK_DEBUG + dbg("done updateGOT"); +#endif + }, + + // Applies relocations to exported things. + $relocateExports__internal: true, + $relocateExports__deps: ['$updateGOT'], + $relocateExports__docs: '/** @param {boolean=} replace */', + $relocateExports: (exports, memoryBase, replace) => { + var relocated = {}; + + for (var e in exports) { + var value = exports[e]; +#if SPLIT_MODULE + // Do not modify exports synthesized by wasm-split + if (e.startsWith('%')) { + relocated[e] = value + continue; + } +#endif + if (typeof value == 'object') { + // a breaking change in the wasm spec, globals are now objects + // https://github.com/WebAssembly/mutable-global/issues/1 + value = value.value; + } + if (typeof value == {{{ POINTER_JS_TYPE }}}) { + value += {{{ to64('memoryBase') }}}; + } + relocated[e] = value; + } + updateGOT(relocated, replace); + return relocated; + }, + + $reportUndefinedSymbols__internal: true, + $reportUndefinedSymbols__deps: ['$GOT', '$resolveGlobalSymbol'], + $reportUndefinedSymbols: () => { +#if DYLINK_DEBUG + dbg('reportUndefinedSymbols'); +#endif + for (var [symName, entry] of Object.entries(GOT)) { + if (entry.value == 0) { + var value = resolveGlobalSymbol(symName, true).sym; + if (!value && !entry.required) { + // Ignore undefined symbols that are imported as weak. +#if DYLINK_DEBUG + dbg(`ignoring undefined weak symbol: ${symName}`); +#endif + continue; + } +#if ASSERTIONS + assert(value, `undefined symbol '${symName}'. perhaps a side module was not linked in? if this global was expected to arrive from a system library, try to build the MAIN_MODULE with EMCC_FORCE_STDLIBS=1 in the environment`); +#endif +#if DYLINK_DEBUG == 2 + dbg(`assigning dynamic symbol from main module: ${symName} -> ${prettyPrint(value)}`); +#endif + if (typeof value == 'function') { + /** @suppress {checkTypes} */ + entry.value = {{{ to64('addFunction(value, value.sig)') }}}; +#if DYLINK_DEBUG == 2 + dbg(`assigning table entry for : ${symName} -> ${entry.value}`); +#endif + } else if (typeof value == 'number') { + entry.value = {{{ to64('value') }}}; +#if MEMORY64 + } else if (typeof value == 'bigint') { + entry.value = value; +#endif + } else { + throw new Error(`bad export type for '${symName}': ${typeof value}`); + } + } + } +#if DYLINK_DEBUG + dbg('done reportUndefinedSymbols'); +#endif + }, + + // dynamic linker/loader (a-la ld.so on ELF systems) + $LDSO__deps: ['$newDSO'], + $LDSO: { + // name -> dso [refcount, name, module, global]; Used by dlopen + loadedLibsByName: {}, + // handle -> dso; Used by dlsym + loadedLibsByHandle: {}, + init() { +#if ASSERTIONS + // This function needs to run after the initial wasmImports object + // as been created. + assert(wasmImports); +#endif + newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports); + }, + }, + + $dlSetError__internal: true, + $dlSetError__deps: ['__dl_seterr', '$stringToUTF8OnStack', '$stackSave', '$stackRestore'], + $dlSetError: (msg) => { +#if DYLINK_DEBUG + dbg(`dlSetError: ${msg}`); +#endif + var sp = stackSave(); + var cmsg = stringToUTF8OnStack(msg); + ___dl_seterr(cmsg, 0); + stackRestore(sp); + }, + + // We support some amount of allocation during startup in the case of + // dynamic linking, which needs to allocate memory for dynamic libraries that + // are loaded. That has to happen before the main program can start to run, + // because the main program needs those linked in before it runs (so we can't + // use normally malloc from the main program to do these allocations). + // + // Allocate memory even if malloc isn't ready yet. The allocated memory here + // must be zero initialized since its used for all static data, including bss. + $getMemory__noleakcheck: true, + $getMemory__deps: ['$GOT', '__heap_base', '$alignMemory', 'calloc'], + $getMemory: (size) => { + // After the runtime is initialized, we must only use sbrk() normally. +#if DYLINK_DEBUG + dbg("getMemory: " + size + " runtimeInitialized=" + runtimeInitialized); +#endif + if (runtimeInitialized) { + // Currently we don't support freeing of static data when modules are + // unloaded via dlclose. This function is tagged as `noleakcheck` to + // avoid having this reported as leak. + return _calloc(size, 1); + } + var ret = ___heap_base; + // Keep __heap_base stack aligned. + var end = ret + alignMemory(size, {{{ STACK_ALIGN }}}); +#if ASSERTIONS + assert(end <= HEAP8.length, 'failure to getMemory - memory growth etc. is not supported there, call malloc/sbrk directly or increase INITIAL_MEMORY'); +#endif + ___heap_base = end; + GOT['__heap_base'].value = {{{ to64('end') }}}; + return ret; + }, + + // returns the side module metadata as an object + // { memorySize, memoryAlign, tableSize, tableAlign, neededDynlibs} + $getDylinkMetadata__deps: ['$UTF8ArrayToString'], + $getDylinkMetadata__internal: true, + $getDylinkMetadata: (binary) => { + var offset = 0; + var end = 0; + + function getU8() { + return binary[offset++]; + } + + function getLEB() { + var ret = 0; + var mul = 1; + while (1) { + var byte = binary[offset++]; + ret += ((byte & 0x7f) * mul); + mul *= 0x80; + if (!(byte & 0x80)) break; + } + return ret; + } + + function getString() { + var len = getLEB(); + offset += len; + return UTF8ArrayToString(binary, offset - len, len); + } + + /** @param {string=} message */ + function failIf(condition, message) { + if (condition) throw new Error(message); + } + + var name = 'dylink.0'; + if (binary instanceof WebAssembly.Module) { + var dylinkSection = WebAssembly.Module.customSections(binary, name); + if (dylinkSection.length === 0) { + name = 'dylink' + dylinkSection = WebAssembly.Module.customSections(binary, name); + } + failIf(dylinkSection.length === 0, 'need dylink section'); + binary = new Uint8Array(dylinkSection[0]); + end = binary.length + } else { + var int32View = new Uint32Array(new Uint8Array(binary.subarray(0, 24)).buffer); +#if SUPPORT_BIG_ENDIAN + var magicNumberFound = int32View[0] == 0x6d736100 || int32View[0] == 0x0061736d; +#else + var magicNumberFound = int32View[0] == 0x6d736100; +#endif + failIf(!magicNumberFound, 'need to see wasm magic number'); // \0asm + // we should see the dylink custom section right after the magic number and wasm version + failIf(binary[8] !== 0, 'need the dylink section to be first') + offset = 9; + var section_size = getLEB(); //section size + end = offset + section_size; + name = getString(); + } + + var customSection = { neededDynlibs: [], tlsExports: new Set(), weakImports: new Set() }; + if (name == 'dylink') { + customSection.memorySize = getLEB(); + customSection.memoryAlign = getLEB(); + customSection.tableSize = getLEB(); + customSection.tableAlign = getLEB(); + // shared libraries this module needs. We need to load them first, so that + // current module could resolve its imports. (see tools/shared.py + // WebAssembly.make_shared_library() for "dylink" section extension format) + var neededDynlibsCount = getLEB(); + for (var i = 0; i < neededDynlibsCount; ++i) { + var libname = getString(); + customSection.neededDynlibs.push(libname); + } + } else { + failIf(name !== 'dylink.0'); + var WASM_DYLINK_MEM_INFO = 0x1; + var WASM_DYLINK_NEEDED = 0x2; + var WASM_DYLINK_EXPORT_INFO = 0x3; + var WASM_DYLINK_IMPORT_INFO = 0x4; + var WASM_SYMBOL_TLS = 0x100; + var WASM_SYMBOL_BINDING_MASK = 0x3; + var WASM_SYMBOL_BINDING_WEAK = 0x1; + while (offset < end) { + var subsectionType = getU8(); + var subsectionSize = getLEB(); + if (subsectionType === WASM_DYLINK_MEM_INFO) { + customSection.memorySize = getLEB(); + customSection.memoryAlign = getLEB(); + customSection.tableSize = getLEB(); + customSection.tableAlign = getLEB(); + } else if (subsectionType === WASM_DYLINK_NEEDED) { + var neededDynlibsCount = getLEB(); + for (var i = 0; i < neededDynlibsCount; ++i) { + libname = getString(); + customSection.neededDynlibs.push(libname); + } + } else if (subsectionType === WASM_DYLINK_EXPORT_INFO) { + var count = getLEB(); + while (count--) { + var symname = getString(); + var flags = getLEB(); + if (flags & WASM_SYMBOL_TLS) { + customSection.tlsExports.add(symname); + } + } + } else if (subsectionType === WASM_DYLINK_IMPORT_INFO) { + var count = getLEB(); + while (count--) { + var modname = getString(); + var symname = getString(); + var flags = getLEB(); + if ((flags & WASM_SYMBOL_BINDING_MASK) == WASM_SYMBOL_BINDING_WEAK) { + customSection.weakImports.add(symname); + } + } + } else { +#if ASSERTIONS + err(`unknown dylink.0 subsection: ${subsectionType}`) +#endif + // unknown subsection + offset += subsectionSize; + } + } + } + +#if ASSERTIONS + var tableAlign = Math.pow(2, customSection.tableAlign); + assert(tableAlign === 1, `invalid tableAlign ${tableAlign}`); + assert(offset == end); +#endif + +#if DYLINK_DEBUG + dbg(`dylink needed:${customSection.neededDynlibs}`); +#endif + + return customSection; + }, + +#if DYNCALLS || !WASM_BIGINT + $registerDynCallSymbols: (exports) => { + for (var [sym, exp] of Object.entries(exports)) { + if (sym.startsWith('dynCall_') && !Module.hasOwnProperty(sym)) { + Module[sym] = exp; + } + } + }, +#endif + + // Module.symbols <- libModule.symbols (flags.global handler) + $mergeLibSymbols__deps: ['$isSymbolDefined'], + $mergeLibSymbols: (exports, libName) => { +#if DYNCALLS || !WASM_BIGINT + registerDynCallSymbols(exports); +#endif + // add symbols into global namespace TODO: weak linking etc. + for (var [sym, exp] of Object.entries(exports)) { +#if ASSERTIONS == 2 + if (isSymbolDefined(sym)) { + var curr = wasmImports[sym], next = exp; + // don't warn on functions - might be odr, linkonce_odr, etc. + if (!(typeof curr == 'function' && typeof next == 'function')) { + err(`warning: symbol '${sym}' from '${libName}' already exists (duplicate symbol? or weak linking, which isn't supported yet?)`); // + [curr, ' vs ', next]); + } + } +#endif + + // When RTLD_GLOBAL is enabled, the symbols defined by this shared object + // will be made available for symbol resolution of subsequently loaded + // shared objects. + // + // We should copy the symbols (which include methods and variables) from + // SIDE_MODULE to MAIN_MODULE. + const setImport = (target) => { +#if ASYNCIFY + if (target in asyncifyStubs) { + asyncifyStubs[target] = exp; + } +#endif + if (!isSymbolDefined(target)) { + wasmImports[target] = exp; + } + } + setImport(sym); + +#if !hasExportedSymbol('main') + // Special case for handling of main symbol: If a side module exports + // `main` that also acts a definition for `__main_argc_argv` and vice + // versa. + const main_alias = '__main_argc_argv'; + if (sym == 'main') { + setImport(main_alias) + } + if (sym == main_alias) { + setImport('main') + } +#endif + } + }, + +#if DYLINK_DEBUG + $dumpTable__deps: ['$wasmTable'], + $dumpTable: () => { + var len = wasmTable.length; + for (var i = {{{ toIndexType(0) }}} ; i < len; i++) { + dbg(`table: ${i} : ${wasmTable.get(i)}`); + } + }, +#endif + + // Loads a side module from binary data or compiled Module. Returns the module's exports or a + // promise that resolves to its exports if the loadAsync flag is set. + $loadWebAssemblyModule__docs: ` + /** + * @param {string=} libName + * @param {Object=} localScope + * @param {number=} handle + */`, + $loadWebAssemblyModule__deps: [ + '$loadDynamicLibrary', '$getMemory', + '$relocateExports', '$resolveGlobalSymbol', '$GOTHandler', + '$getDylinkMetadata', '$alignMemory', + '$currentModuleWeakSymbols', + '$updateTableMap', + '$wasmTable', + ], + $loadWebAssemblyModule: (binary, flags, libName, localScope, handle) => { +#if DYLINK_DEBUG + dbg(`loadWebAssemblyModule: ${libName}`); +#endif + var metadata = getDylinkMetadata(binary); + currentModuleWeakSymbols = metadata.weakImports; +#if ASSERTIONS + var originalTable = wasmTable; +#endif + + // loadModule loads the wasm module after all its dependencies have been loaded. + // can be called both sync/async. + function loadModule() { + // The first thread to load a given module needs to allocate the static + // table and memory regions. Later threads re-use the same table region + // and can ignore the memory region (since memory is shared between + // threads already). + // If `handle` is specified than it is assumed that the calling thread has + // exclusive access to it for the duration of this function. See the + // locking in `dynlink.c`. + var firstLoad = !handle || !{{{ makeGetValue('handle', C_STRUCTS.dso.mem_allocated, 'i8') }}}; + if (firstLoad) { + // alignments are powers of 2 + var memAlign = Math.pow(2, metadata.memoryAlign); + // prepare memory + var memoryBase = metadata.memorySize ? alignMemory(getMemory(metadata.memorySize + memAlign), memAlign) : 0; // TODO: add to cleanups + var tableBase = metadata.tableSize ? {{{ from64Expr('wasmTable.length') }}} : 0; + if (handle) { + {{{ makeSetValue('handle', C_STRUCTS.dso.mem_allocated, '1', 'i8') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.mem_addr, 'memoryBase', '*') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.mem_size, 'metadata.memorySize', 'i32') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.table_addr, 'tableBase', '*') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.table_size, 'metadata.tableSize', 'i32') }}}; + } + } else { + memoryBase = {{{ makeGetValue('handle', C_STRUCTS.dso.mem_addr, '*') }}}; + tableBase = {{{ makeGetValue('handle', C_STRUCTS.dso.table_addr, '*') }}}; + } + + var tableGrowthNeeded = tableBase + metadata.tableSize - {{{ from64Expr('wasmTable.length') }}}; + if (tableGrowthNeeded > 0) { +#if DYLINK_DEBUG + dbg("loadModule: growing table: " + tableGrowthNeeded); +#endif + wasmTable.grow({{{ toIndexType('tableGrowthNeeded') }}}); + } +#if DYLINK_DEBUG + dbg("loadModule: memory[" + memoryBase + ":" + (memoryBase + metadata.memorySize) + "]" + + " table[" + tableBase + ":" + (tableBase + metadata.tableSize) + "]"); +#endif + + // This is the export map that we ultimately return. We declare it here + // so it can be used within resolveSymbol. We resolve symbols against + // this local symbol map in the case there they are not present on the + // global Module object. We need this fallback because Modules sometime + // need to import their own symbols + var moduleExports; + + function resolveSymbol(sym) { + var resolved = resolveGlobalSymbol(sym).sym; + if (!resolved && localScope) { + resolved = localScope[sym]; + } + if (!resolved) { + resolved = moduleExports[sym]; + } +#if ASSERTIONS + assert(resolved, `undefined symbol '${sym}'. perhaps a side module was not linked in? if this global was expected to arrive from a system library, try to build the MAIN_MODULE with EMCC_FORCE_STDLIBS=1 in the environment`); +#endif + return resolved; + } + + // TODO kill ↓↓↓ (except "symbols local to this module", it will likely be + // not needed if we require that if A wants symbols from B it has to link + // to B explicitly: similarly to -Wl,--no-undefined) + // + // wasm dynamic libraries are pure wasm, so they cannot assist in + // their own loading. When side module A wants to import something + // provided by a side module B that is loaded later, we need to + // add a layer of indirection, but worse, we can't even tell what + // to add the indirection for, without inspecting what A's imports + // are. To do that here, we use a JS proxy (another option would + // be to inspect the binary directly). + var proxyHandler = { + get(stubs, prop) { + // symbols that should be local to this module + switch (prop) { + case '__memory_base': + return {{{ to64('memoryBase') }}}; + case '__table_base': + return {{{ to64('tableBase') }}}; +#if MEMORY64 +#if MEMORY64 == 2 + case '__memory_base32': + return memoryBase; +#endif + case '__table_base32': + return tableBase; +#endif + } + if (prop in wasmImports && !wasmImports[prop].stub) { + // No stub needed, symbol already exists in symbol table + return wasmImports[prop]; + } + // Return a stub function that will resolve the symbol + // when first called. + if (!(prop in stubs)) { + var resolved; + stubs[prop] = (...args) => { + resolved ||= resolveSymbol(prop); + if (!resolved) { +/* + if (prop==='getTempRet0') + return __emscripten_tempret_get(...args); + if (prop==='setTempRet0') + return __emscripten_tempret_set(...args); +*/ + throw new Error(); + } + return resolved(...args); + }; + } + return stubs[prop]; + } + }; + var proxy = new Proxy({}, proxyHandler); + var info = { + 'GOT.mem': new Proxy({}, GOTHandler), + 'GOT.func': new Proxy({}, GOTHandler), + 'env': proxy, + '{{{ WASI_MODULE_NAME }}}': proxy, + }; + + function postInstantiation(module, instance) { +#if ASSERTIONS + // the table should be unchanged + assert(wasmTable === originalTable); +#endif +#if PTHREADS + if (!ENVIRONMENT_IS_PTHREAD && libName) { +#if DYLINK_DEBUG + dbg(`registering sharedModules: ${libName}`) +#endif + // cache all loaded modules in `sharedModules`, which gets passed + // to new workers when they are created. + sharedModules[libName] = module; + } +#endif + // add new entries to functionsInTableMap + updateTableMap(tableBase, metadata.tableSize); + moduleExports = relocateExports(instance.exports, memoryBase); +#if ASYNCIFY + moduleExports = Asyncify.instrumentWasmExports(moduleExports); +#endif + if (!flags.allowUndefined) { + reportUndefinedSymbols(); + } +#if STACK_OVERFLOW_CHECK >= 2 + // If the runtime has already been initialized we set the stack limits + // now. Otherwise this is delayed until `setDylinkStackLimits` is + // called after initialization. + if (moduleExports['__set_stack_limits'] && runtimeInitialized) { + moduleExports['__set_stack_limits']({{{ to64('_emscripten_stack_get_base()') }}}, {{{ to64('_emscripten_stack_get_end()') }}}); + } +#endif + +#if MAIN_MODULE + function addEmAsm(addr, body) { + var args = []; + var arity = 0; + for (; arity < 16; arity++) { + if (body.indexOf('$' + arity) != -1) { + args.push('$' + arity); + } else { + break; + } + } + args = args.join(','); + var func = `(${args}) => { ${body} };`; +#if DYLINK_DEBUG + dbg(`adding new EM_ASM constant at: ${ptrToString(start)}`); +#endif + {{{ makeEval('ASM_CONSTS[start] = eval(func)') }}}; + } + + // Add any EM_ASM function that exist in the side module + if ('__start_em_asm' in moduleExports) { + var start = moduleExports['__start_em_asm']; + var stop = moduleExports['__stop_em_asm']; + {{{ from64('start') }}} + {{{ from64('stop') }}} + while (start < stop) { + var jsString = UTF8ToString(start); + addEmAsm(start, jsString); + start = HEAPU8.indexOf(0, start) + 1; + } + } + + function addEmJs(name, cSig, body) { + // The signature here is a C signature (e.g. "(int foo, char* bar)"). + // See `create_em_js` in emcc.py` for the build-time version of this + // code. + var jsArgs = []; + cSig = cSig.slice(1, -1) + if (cSig != 'void') { + cSig = cSig.split(','); + for (var i in cSig) { + var jsArg = cSig[i].split(' ').pop(); + jsArgs.push(jsArg.replaceAll('*', '')); + } + } + var func = `(${jsArgs}) => ${body};`; +#if DYLINK_DEBUG + dbg(`adding new EM_JS function: ${jsArgs} = ${func}`); +#endif + {{{ makeEval('moduleExports[name] = eval(func)') }}}; + } + + for (var name in moduleExports) { + if (name.startsWith('__em_js__')) { + var start = moduleExports[name] + var jsString = UTF8ToString({{{ from64Expr('start') }}}); + // EM_JS strings are stored in the data section in the form + // SIG<::>BODY. + var parts = jsString.split('<::>'); + addEmJs(name.replace('__em_js__', ''), parts[0], parts[1]); + delete moduleExports[name]; + } + } +#endif + + // initialize the module +#if PTHREADS + // Only one thread should call __wasm_call_ctors, but all threads need + // to call _emscripten_tls_init + registerTLSInit(moduleExports['_emscripten_tls_init'], instance.exports, metadata) + if (firstLoad) { +#endif + var applyRelocs = moduleExports['__wasm_apply_data_relocs']; + if (applyRelocs) { + if (runtimeInitialized) { +#if DYLINK_DEBUG + dbg('applyRelocs'); +#endif + applyRelocs(); + } else { + __RELOC_FUNCS__.push(applyRelocs); + } + } + var init = moduleExports['__wasm_call_ctors']; + if (init) { + if (runtimeInitialized) { + init(); + } else { + // we aren't ready to run compiled code yet + __ATINIT__.push(init); + } + } +#if PTHREADS + } +#endif + return moduleExports; + } + + if (flags.loadAsync) { + if (binary instanceof WebAssembly.Module) { + var instance = new WebAssembly.Instance(binary, info); + return Promise.resolve(postInstantiation(binary, instance)); + } + return WebAssembly.instantiate(binary, info).then( + (result) => postInstantiation(result.module, result.instance) + ); + } + + var module = binary instanceof WebAssembly.Module ? binary : new WebAssembly.Module(binary); + var instance = new WebAssembly.Instance(module, info); + return postInstantiation(module, instance); + } + + // now load needed libraries and the module itself. + if (flags.loadAsync) { + return metadata.neededDynlibs + .reduce((chain, dynNeeded) => chain.then(() => + loadDynamicLibrary(dynNeeded, flags, localScope) + ), Promise.resolve()) + .then(loadModule); + } + + metadata.neededDynlibs.forEach((needed) => loadDynamicLibrary(needed, flags, localScope)); + return loadModule(); + }, + +#if STACK_OVERFLOW_CHECK >= 2 + // Sometimes we load libraries before runtime initialization. In this case + // we delay calling __set_stack_limits (which must be called for each + // module). + $setDylinkStackLimits: (stackTop, stackMax) => { + for (var name in LDSO.loadedLibsByName) { +#if DYLINK_DEBUG + dbg(`setDylinkStackLimits for '${name}'`); +#endif + var lib = LDSO.loadedLibsByName[name]; + lib.exports['__set_stack_limits']?.({{{ to64("stackTop") }}}, {{{ to64("stackMax") }}}); + } + }, +#endif + + $newDSO: (name, handle, syms) => { + var dso = { + refcount: Infinity, + name, + exports: syms, + global: true, + }; + LDSO.loadedLibsByName[name] = dso; + if (handle != undefined) { + LDSO.loadedLibsByHandle[handle] = dso; + } + return dso; + }, + + // loadDynamicLibrary loads dynamic library @ lib URL / path and returns + // handle for loaded DSO. + // + // Several flags affect the loading: + // + // - if flags.global=true, symbols from the loaded library are merged into global + // process namespace. Flags.global is thus similar to RTLD_GLOBAL in ELF. + // + // - if flags.nodelete=true, the library will be never unloaded. Flags.nodelete + // is thus similar to RTLD_NODELETE in ELF. + // + // - if flags.loadAsync=true, the loading is performed asynchronously and + // loadDynamicLibrary returns corresponding promise. + // + // If a library was already loaded, it is not loaded a second time. However + // flags.global and flags.nodelete are handled every time a load request is made. + // Once a library becomes "global" or "nodelete", it cannot be removed or unloaded. + $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', + '$isInternalSym', '$mergeLibSymbols', '$newDSO', + '$asyncLoad', +#if FILESYSTEM + '$preloadedWasm', +#endif +#if DYNCALLS || !WASM_BIGINT + '$registerDynCallSymbols', +#endif + ], + $loadDynamicLibrary__docs: ` + /** + * @param {number=} handle + * @param {Object=} localScope + */`, + $loadDynamicLibrary: function(libName, flags = {global: true, nodelete: true}, localScope, handle) { +#if DYLINK_DEBUG + dbg(`loadDynamicLibrary: ${libName} handle: ${handle}`); + dbg(`existing: ${Object.keys(LDSO.loadedLibsByName)}`); +#endif + // when loadDynamicLibrary did not have flags, libraries were loaded + // globally & permanently + + var dso = LDSO.loadedLibsByName[libName]; + if (dso) { + // the library is being loaded or has been loaded already. +#if ASSERTIONS + assert(dso.exports !== 'loading', `Attempt to load '${libName}' twice before the first load completed`); +#endif + if (!flags.global) { + if (localScope) { + Object.assign(localScope, dso.exports); + } +#if DYNCALLS || !WASM_BIGINT + registerDynCallSymbols(dso.exports); +#endif + } else if (!dso.global) { + // The library was previously loaded only locally but not + // we have a request with global=true. + dso.global = true; + mergeLibSymbols(dso.exports, libName) + } + // same for "nodelete" + if (flags.nodelete && dso.refcount !== Infinity) { + dso.refcount = Infinity; + } + dso.refcount++ + if (handle) { + LDSO.loadedLibsByHandle[handle] = dso; + } + return flags.loadAsync ? Promise.resolve(true) : true; + } + + // allocate new DSO + dso = newDSO(libName, handle, 'loading'); + dso.refcount = flags.nodelete ? Infinity : 1; + dso.global = flags.global; + + // libName -> libData + function loadLibData() { +#if PTHREADS + var sharedMod = sharedModules[libName]; +#if DYLINK_DEBUG + dbg(`checking sharedModules: ${libName}: ${sharedMod ? 'found' : 'not found'}`); +#endif + if (sharedMod) { + return flags.loadAsync ? Promise.resolve(sharedMod) : sharedMod; + } +#endif + + // for wasm, we can use fetch for async, but for fs mode we can only imitate it + if (handle) { + var data = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data, '*') }}}; + var dataSize = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data_size, '*') }}}; + if (data && dataSize) { + var libData = HEAP8.slice(data, data + dataSize); + return flags.loadAsync ? Promise.resolve(libData) : libData; + } + } + + var libFile = locateFile(libName); + if (flags.loadAsync) { + return asyncLoad(libFile); + } + + // load the binary synchronously + if (!readBinary) { + throw new Error(`${libFile}: file not found, and synchronous loading of external files is not available`); + } + return readBinary(libFile); + } + + // libName -> exports + function getExports() { +#if FILESYSTEM + // lookup preloaded cache first + var preloaded = preloadedWasm[libName]; +#if DYLINK_DEBUG + dbg(`checking preloadedWasm: ${libName}: ${preloaded ? 'found' : 'not found'}`); +#endif + if (preloaded) { + return flags.loadAsync ? Promise.resolve(preloaded) : preloaded; + } +#endif + + // module not preloaded - load lib data and create new module from it + if (flags.loadAsync) { + return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, libName, localScope, handle)); + } + + return loadWebAssemblyModule(loadLibData(), flags, libName, localScope, handle); + } + + // module for lib is loaded - update the dso & global namespace + function moduleLoaded(exports) { + if (dso.global) { + mergeLibSymbols(exports, libName); + } else if (localScope) { + Object.assign(localScope, exports); +#if DYNCALLS || !WASM_BIGINT + registerDynCallSymbols(exports); +#endif + } + dso.exports = exports; + } + + if (flags.loadAsync) { +#if DYLINK_DEBUG + dbg("loadDynamicLibrary: done (async)"); +#endif + return getExports().then((exports) => { + moduleLoaded(exports); + return true; + }); + } + + moduleLoaded(getExports()); +#if DYLINK_DEBUG + dbg("loadDynamicLibrary: done"); +#endif + return true; + }, + + $loadDylibs__internal: true, + $loadDylibs__deps: ['$loadDynamicLibrary', '$reportUndefinedSymbols'], + $loadDylibs: () => { + if (!dynamicLibraries.length) { +#if DYLINK_DEBUG + dbg('loadDylibs: no libraries to preload'); +#endif + reportUndefinedSymbols(); + return; + } + +#if DYLINK_DEBUG + dbg(`loadDylibs: ${dynamicLibraries}`); +#endif + + // Load binaries asynchronously + addRunDependency('loadDylibs'); + dynamicLibraries + .reduce((chain, lib) => chain.then(() => + loadDynamicLibrary(lib, {loadAsync: true, global: true, nodelete: true, allowUndefined: true}) + ), Promise.resolve()) + .then(() => { + // we got them all, wonderful + reportUndefinedSymbols(); + removeRunDependency('loadDylibs'); + #if DYLINK_DEBUG + dbg('loadDylibs done!'); + #endif + }); + }, + + // void* dlopen(const char* filename, int flags); + $dlopenInternal__deps: ['$ENV', '$dlSetError', '$PATH'], + $dlopenInternal: (handle, jsflags) => { + // void *dlopen(const char *file, int mode); + // http://pubs.opengroup.org/onlinepubs/009695399/functions/dlopen.html + var filename = UTF8ToString(handle + {{{ C_STRUCTS.dso.name }}}); + var flags = {{{ makeGetValue('handle', C_STRUCTS.dso.flags, 'i32') }}}; +#if DYLINK_DEBUG + dbg(`dlopenInternal: ${filename}`); +#endif + filename = PATH.normalize(filename); + var searchpaths = []; + + var global = Boolean(flags & {{{ cDefs.RTLD_GLOBAL }}}); + var localScope = global ? null : {}; + + // We don't care about RTLD_NOW and RTLD_LAZY. + var combinedFlags = { + global, + nodelete: Boolean(flags & {{{ cDefs.RTLD_NODELETE }}}), + loadAsync: jsflags.loadAsync, + } + + if (jsflags.loadAsync) { + return loadDynamicLibrary(filename, combinedFlags, localScope, handle); + } + + try { + return loadDynamicLibrary(filename, combinedFlags, localScope, handle) + } catch (e) { +#if ASSERTIONS + err(`Error in loading dynamic library ${filename}: ${e}`); +#endif + dlSetError(`Could not load dynamic lib: ${filename}\n${e}`); + return 0; + } + }, + + _dlopen_js__deps: ['$dlopenInternal'], +#if ASYNCIFY + _dlopen_js__async: true, +#endif + _dlopen_js: (handle) => { +#if ASYNCIFY + return Asyncify.handleSleep((wakeUp) => { + dlopenInternal(handle, { loadAsync: true }) + .then(wakeUp) + .catch(() => wakeUp(0)); + }); +#else + return dlopenInternal(handle, { loadAsync: false }); +#endif + }, + + // Async version of dlopen. + _emscripten_dlopen_js__deps: ['$dlopenInternal', '$callUserCallback', '$dlSetError'], + _emscripten_dlopen_js: (handle, onsuccess, onerror, user_data) => { + /** @param {Object=} e */ + function errorCallback(e) { + var filename = UTF8ToString(handle + {{{ C_STRUCTS.dso.name }}}); + dlSetError(`'Could not load dynamic lib: ${filename}\n${e}`); + {{{ runtimeKeepalivePop() }}} + callUserCallback(() => {{{ makeDynCall('vpp', 'onerror') }}}(handle, user_data)); + } + function successCallback() { + {{{ runtimeKeepalivePop() }}} + callUserCallback(() => {{{ makeDynCall('vpp', 'onsuccess') }}}(handle, user_data)); + } + + {{{ runtimeKeepalivePush() }}} + var promise = dlopenInternal(handle, { loadAsync: true }); + if (promise) { + promise.then(successCallback, errorCallback); + } else { + errorCallback(); + } + }, + + _dlsym_catchup_js: (handle, symbolIndex) => { +#if DYLINK_DEBUG + dbg("_dlsym_catchup: handle=" + ptrToString(handle) + " symbolIndex=" + symbolIndex); +#endif + var lib = LDSO.loadedLibsByHandle[handle]; + var symDict = lib.exports; + var symName = Object.keys(symDict)[symbolIndex]; + var sym = symDict[symName]; + var result = addFunction(sym, sym.sig); +#if DYLINK_DEBUG + dbg(`_dlsym_catchup: result=${result}`); +#endif + return result; + }, + + // void* dlsym(void* handle, const char* symbol); + _dlsym_js__deps: ['$dlSetError', '$getFunctionAddress', '$addFunction'], + _dlsym_js: (handle, symbol, symbolIndex) => { + // void *dlsym(void *restrict handle, const char *restrict name); + // http://pubs.opengroup.org/onlinepubs/009695399/functions/dlsym.html + symbol = UTF8ToString(symbol); +#if DYLINK_DEBUG + dbg(`dlsym_js: ${symbol}`); +#endif + var result; + var newSymIndex; + + var lib = LDSO.loadedLibsByHandle[handle]; +#if ASSERTIONS + assert(lib, `Tried to dlsym() from an unopened handle: ${handle}`); +#endif + if (!lib.exports.hasOwnProperty(symbol) || lib.exports[symbol].stub) { + dlSetError(`Tried to lookup unknown symbol "${symbol}" in dynamic lib: ${lib.name}`) + return 0; + } + newSymIndex = Object.keys(lib.exports).indexOf(symbol); +#if !WASM_BIGINT + var origSym = 'orig$' + symbol; + result = lib.exports[origSym]; + if (result) { + newSymIndex = Object.keys(lib.exports).indexOf(origSym); + } + else +#endif + result = lib.exports[symbol]; + + if (typeof result == 'function') { +#if DYLINK_DEBUG + dbg(`dlsym_js: ${symbol} getting table slot for: ${result}`); +#endif + +#if ASYNCIFY + // Asyncify wraps exports, and we need to look through those wrappers. + if ('orig' in result) { + result = result.orig; + } +#endif + var addr = getFunctionAddress(result); + if (addr) { +#if DYLINK_DEBUG + dbg(`symbol already exists in table: ${symbol}`); +#endif + result = addr; + } else { + // Insert the function into the wasm table. If its a direct wasm + // function the second argument will not be needed. If its a JS + // function we rely on the `sig` attribute being set based on the + // `__sig` specified in library JS file. + result = addFunction(result, result.sig); +#if DYLINK_DEBUG + dbg(`adding symbol to table: ${symbol}`); +#endif + {{{ makeSetValue('symbolIndex', 0, 'newSymIndex', '*') }}}; + } + } +#if DYLINK_DEBUG + dbg(`dlsym_js: ${symbol} -> ${result}`); +#endif + return result; + }, +}; + +addToLibrary(LibraryDylink); diff --git a/other/PGPASSFILE b/other/PGPASSFILE new file mode 100644 index 0000000000000..b8a0b5b6a58ac --- /dev/null +++ b/other/PGPASSFILE @@ -0,0 +1,3 @@ +localhost:5432:postgres:password:md532e12f215ba27cb750c9e093ce4b5127 +localhost:5432:postgres:postgres:md53175bce1d3201d16594cebf9d7eb3f9d +localhost:5432:postgres:login:md5d5745f9425eceb269f9fe01d0bef06ff diff --git a/other/empty b/other/empty new file mode 100644 index 0000000000000..646d10863717a --- /dev/null +++ b/other/empty @@ -0,0 +1 @@ +PGlite is the best! \ No newline at end of file diff --git a/other/password b/other/password new file mode 100644 index 0000000000000..f3097ab13082b --- /dev/null +++ b/other/password @@ -0,0 +1 @@ +password diff --git a/pglite-wasm/builder/Dockerfile b/pglite-wasm/builder/Dockerfile new file mode 100644 index 0000000000000..cf613dd2ddc85 --- /dev/null +++ b/pglite-wasm/builder/Dockerfile @@ -0,0 +1,94 @@ +ARG EMSDK_VER=3.1.74 +ARG BUILDPLATFORM +# we only need to build on one platform, since we're interested in the WASM output +# building on the native (BUILDPLATFORM) is much faster, so use that +# remove "-arm64" suffix if building on x86_64 +FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER} AS builder + +ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm + +RUN apt update && apt upgrade -y && apt install -y \ + xz-utils autoconf libtool automake pkgconf bison flex + +SHELL ["/bin/bash", "-c"] + +RUN mkdir -p /install/libs /install/exports + +WORKDIR /install/libs + +# zlib CAN be installed by using the emsdk like this +# RUN embuilder --pic --verbose build zlib + +WORKDIR /src + +# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn" +ENV EMCC_COMMON_FLAGS="-O2 -fPIC" + +WORKDIR /src +RUN curl -L https://www.zlib.net/zlib-1.3.1.tar.gz | tar -xz +WORKDIR /src/zlib-1.3.1 +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --static --prefix=/install/libs +RUN emmake make -j && emmake make install +RUN ${LLVM_NM} /install/libs/lib/libz.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libz.exports + +WORKDIR /src +RUN curl -L https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.14.5/libxml2-v2.14.5.tar.gz | tar -xz +WORKDIR /src/libxml2-v2.14.5 +RUN ./autogen.sh --with-python=no +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs +RUN emmake make -j && emmake make install +# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS +RUN ${LLVM_NM} /install/libs/lib/libxml2.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libxml2.exports + +WORKDIR /src +RUN curl -L https://gitlab.gnome.org/GNOME/libxslt/-/archive/v1.1.43/libxslt-v1.1.43.tar.gz | tar -xz +WORKDIR /src/libxslt-v1.1.43 +RUN ./autogen.sh --with-python=no +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs --with-libxml-src=/src/libxml2-v2.14.5/ --with-pic=yes +RUN emmake make -j && emmake make install +# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS +RUN ${LLVM_NM} /install/libs/lib/libxslt.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libxslt.exports + +WORKDIR /src +RUN curl -L https://github.com/openssl/openssl/releases/download/openssl-3.0.17/openssl-3.0.17.tar.gz | tar xz +WORKDIR /src/openssl-3.0.17 +RUN emconfigure ./Configure no-tests linux-generic64 --prefix=/install/libs +RUN sed -i 's|^CROSS_COMPILE.*$|CROSS_COMPILE=|g' Makefile # see https://github.com/emscripten-core/emscripten/issues/19597#issue-1754476454 +RUN emmake make -j && emmake make install +# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS +RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libssl.exports + +WORKDIR /src +RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz +# COPY . . +# RUN tar xvf uuid-1.6.2.tar.gz +WORKDIR /src/uuid-1.6.2 +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --build=aarch64-unknown-linux-gnu --enable-shared=no --enable-static=yes --with-perl=no --with-perl-compat=no --prefix=/install/libs --with-php=no --with-pic=yes +RUN emmake make -j && emmake make install || true # install tries to strip the wasm, but it doesnt recognize the format, so ignore atm +WORKDIR /install/libs/lib +RUN ln -s libuuid.a libossp-uuid.a # contrib extensions use -lossp-uuid +RUN ${LLVM_NM} /install/libs/lib/libossp-uuid.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libossp-uuid.exports + +ARG TARGETARCH +FROM emscripten/emsdk:${EMSDK_VER} AS base-amd64 +FROM emscripten/emsdk:${EMSDK_VER}-arm64 AS base-arm64 +FROM base-${TARGETARCH} AS runner + +RUN apt update && apt upgrade -y && apt install -y \ + xz-utils autoconf libtool automake pkgconf bison flex + +# this is where the libraries will be installed and subsequently where the LIBS and INCLUDES can be found +ARG INSTALL_PREFIX=/install/libs +ENV INSTALL_PREFIX=${INSTALL_PREFIX} + +ARG LIB_EXPORTS_DIR=/install/exports +ENV LIB_EXPORTS_DIR=${LIB_EXPORTS_DIR} + +COPY --from=builder /install/libs ${INSTALL_PREFIX} +COPY --from=builder /install/exports ${LIB_EXPORTS_DIR} + +# allow access to anyone +RUN chmod -R 777 /install + +# needed in building pglite.wasm +ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm \ No newline at end of file diff --git a/pglite-wasm/excluded.pglite.imports b/pglite-wasm/excluded.pglite.imports new file mode 100644 index 0000000000000..e0d801f2348b5 --- /dev/null +++ b/pglite-wasm/excluded.pglite.imports @@ -0,0 +1,32 @@ +__indirect_function_table +invoke_di +invoke_didi +invoke_dii +invoke_i +invoke_ii +invoke_iii +invoke_iiii +invoke_iiiii +invoke_iiiiii +invoke_iiiiiii +invoke_iiiiiiii +invoke_iij +invoke_ij +invoke_ijj +invoke_j +invoke_ji +invoke_jiiiii +invoke_v +invoke_vi +invoke_vii +invoke_viii +invoke_viiidi +invoke_viiii +invoke_viiiii +invoke_viiiiii +invoke_viiiiii +invoke_viiiiiiii +invoke_viiiiiiiii +invoke_viiji +invoke_vij +invoke_vijiiidjiiii diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports new file mode 100644 index 0000000000000..7a5d30d895111 --- /dev/null +++ b/pglite-wasm/included.pglite.imports @@ -0,0 +1,33 @@ +close +fcntl +free +getpid +gettimeofday +gmtime +interactive_one +ioctl +isalnum +isxdigit +lowerstr +main +malloc +memcmp +memcpy +memset +nanosleep +open +pgl_backend +pgl_initdb +pgl_shutdown +rand +read +readstoplist +realloc +searchstoplist +set_read_write_cbs +socket +srand +strcmp +strftime +strlen +strtoul \ No newline at end of file diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index fc3590961d5d6..2c2fa478c43ad 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -9,94 +9,32 @@ volatile sigjmp_buf local_sigjmp_buf; // track back how many ex raised in steps of the loop until sucessfull clear_error volatile int canary_ex = 0; -// track back mode used for last reply <0 socketfiles , 0== repl , > 0 cma addr -volatile int channel = 0; - -/* TODO : prevent multiple write and write while reading ? */ -volatile int cma_wsize = 0; -volatile int cma_rsize = 0; // also defined in postgres.c for pqcomm -volatile bool sockfiles = false; // also defined in postgres.c for pqcomm - -__attribute__((export_name("get_buffer_size"))) -int -get_buffer_size(int fd) { - return (CMA_MB * 1024 * 1024) / CMA_FD; -} - -// TODO add query size -__attribute__((export_name("get_buffer_addr"))) -int -get_buffer_addr(int fd) { - return 1 + ( get_buffer_size(fd) *fd); -} - -__attribute__((export_name("get_channel"))) -int -get_channel() { - return channel; -} - - -__attribute__((export_name("interactive_read"))) -int -interactive_read() { - /* should cma_rsize should be reset here ? */ - return cma_wsize; -} - - -static void pg_prompt() { - fprintf(stdout,"pg> %c\n", 4); +// read FROM JS +// (i guess return number of bytes written) +// ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len); +//typedef ssize_t (*pglite_read_t)(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ unsigned int *address_len); +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +extern pglite_read_t pglite_read; + +// write TO JS +// (i guess return number of bytes read) +// ssize_t pglite_write(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +// typedef ssize_t (*pglite_write_t)(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +typedef ssize_t (*pglite_write_t)(void *buffer, size_t length); +extern pglite_write_t pglite_write; + +__attribute__((export_name("set_read_write_cbs"))) +void +set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) { + pglite_read = read_cb; + pglite_write = write_cb; } extern void AbortTransaction(void); extern void CleanupTransaction(void); extern void ClientAuthentication(Port *port); -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; - - - -/* -init sequence -___________________________________ -SubPostmasterMain / (forkexec) - InitPostmasterChild - shm attach - preload - - BackendInitialize(Port *port) -> collect initial packet - - pq_init(); - whereToSendOutput = DestRemote; - status = ProcessStartupPacket(port, false, false); - pq_startmsgread - pq_getbytes from pq_recvbuf - TODO: place PqRecvBuffer (8K) in lower mem for zero copy - - PerformAuthentication - ClientAuthentication(port) - CheckPasswordAuth SYNC!!!! ( sendAuthRequest flush -> recv_password_packet ) - InitShmemAccess/InitProcess/CreateSharedMemoryAndSemaphores - - BackendRun(port) - PostgresMain - - --> pq_flush() is synchronous - - -buffer sizes: - - https://github.com/postgres/postgres/blob/master/src/backend/libpq/pqcomm.c#L118 - - https://github.com/postgres/postgres/blob/master/src/common/stringinfo.c#L28 - - -*/ extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); -extern void pq_recvbuf_fill(FILE* fp, int packetlen); #define PG_MAX_AUTH_TOKEN_LENGTH 65535 static char * @@ -187,44 +125,13 @@ static void io_init(bool in_auth, bool out_auth) { MyProcPort->canAcceptConnections = CAC_OK; #endif ClientAuthInProgress = out_auth; - SOCKET_FILE = NULL; - SOCKET_DATA = 0; PDEBUG("\n\n\n# 165: io_init --------- Ready for CLIENT ---------"); } volatile bool is_wire = true; -extern char * cma_port; extern void pq_startmsgread(void); -__attribute__((export_name("interactive_write"))) -void -interactive_write(int size) { - cma_rsize = size; - cma_wsize = 0; -} - -__attribute__((export_name("use_wire"))) -void -use_wire(int state) { -#if PGDEBUG - force_echo=true; -#endif - if (state>0) { -#if PGDEBUG - printf("\n\n# 194: PACKET START: wire mode, repl off, echo %d\n", force_echo); -#endif - is_wire = true; - is_repl = false; - } else { -#if PGDEBUG - printf("\n\n# 200: PACKET START: repl mode, no wire, echo %d\n", force_echo); -#endif - is_wire = false; - is_repl = true; - } -} - __attribute__((export_name("clear_error"))) void clear_error() { @@ -236,11 +143,11 @@ clear_error() { idle_in_transaction_timeout_enabled = false; idle_session_timeout_enabled = false; DoingCommandRead = false; -puts("# 239:" __FILE__ ); + pq_comm_reset(); EmitErrorReport(); debug_query_string = NULL; -puts("# 243:" __FILE__ ); + AbortCurrentTransaction(); if (am_walsender) @@ -279,16 +186,6 @@ puts("# 243:" __FILE__ ); send_ready_for_query = true; } -void discard_input(){ - if (!cma_rsize) - return; - pq_startmsgread(); - for (int i = 0; i < cma_rsize; i++) { - pq_getbyte(); - } - pq_endmsgread(); -} - void startup_auth() { /* code is in handshake/auth domain so read whole msg now */ @@ -301,7 +198,6 @@ startup_auth() { sf_connected++; PDEBUG("# 273: sending auth request"); //ClientAuthentication(MyProcPort); - discard_input(); ClientAuthInProgress = true; md5Salt[0]=0x01; @@ -339,7 +235,6 @@ startup_pass(bool check) { pfree(passwd); } else { PDEBUG("# 310: auth skip"); - discard_input(); } ClientAuthInProgress = false; @@ -365,24 +260,20 @@ PDEBUG("# 330: TODO: set a pgl started flag"); volatile int sf_connected = 0; } -extern void pg_startcma(); - __attribute__((export_name("interactive_one"))) void -interactive_one() { - int peek = -1; /* preview of firstchar with no pos change */ +interactive_one(int packetlen, int peek) { + // int peek = -1; /* preview of firstchar with no pos change */ int firstchar = 0; /* character read from getc() */ bool pipelining = true; StringInfoData input_message; StringInfoData *inBuf; FILE *stream ; FILE *fp = NULL; - int packetlen; + // int packetlen; bool had_notification = notifyInterruptPending; bool notified = false; // send_ready_for_query = false; -if (cma_rsize<0) - goto resume_on_error; if (!MyProcPort) { PDEBUG("# 353: client created"); @@ -390,33 +281,13 @@ if (cma_rsize<0) } #if PGDEBUG + puts("\n\n# 369: interactive_one"); if (notifyInterruptPending) PDEBUG("# 371: has notification !"); #endif - // this could be pg_flush in sync mode. - // but in fact we are writing socket data that was piled up previous frame async. - if (SOCKET_DATA>0) { - puts("# 361: ERROR flush after frame"); - goto wire_flush; - } - - if (!cma_rsize) { - // no cma : reading from file. writing to file. - if (!SOCKET_FILE) { - SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - MyProcPort->sock = fileno(SOCKET_FILE); - } - } else { - // prepare file reply queue, just in case of cma overflow - // if unused the file will be kept open till next query. - if (!SOCKET_FILE) { - SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - } - } - MemoryContextSwitchTo(MessageContext); - MemoryContextResetAndDeleteChildren(MessageContext); + MemoryContextReset(MessageContext); initStringInfo(&input_message); @@ -439,134 +310,21 @@ if (cma_rsize<0) } send_ready_for_query = false; } -// postgres.c 4627 - DoingCommandRead = true; - -#if defined(EMUL_CMA) - // temp fix for -O0 but less efficient than literal - #define IO ((char *)(1+(int)cma_port)) -#else - #define IO ((char *)(1)) -#endif - -/* - * in cma mode (cma_rsize>0), client call the wire loop itself waiting synchronously for the results - * in socketfiles mode, the wire loop polls a pseudo socket made from incoming and outgoing files. - * in repl mode (cma_rsize==0) output is on stdout not cma/socketfiles wire. - * repl mode is the simpleset mode where stdin is just copied into input buffer (limited by CMA size). - * TODO: allow to redirect stdout for fully external repl. - */ - - peek = IO[0]; - packetlen = cma_rsize; - - if (packetlen) { - sockfiles = false; - if (!is_repl) { - whereToSendOutput = DestRemote; - if (!is_wire) - PDEBUG("# 439: repl message in cma buffer !"); - } else { - if (is_wire) - PDEBUG("# 442: wire message in cma buffer for REPL !"); - whereToSendOutput = DestDebug; - } - } else { - fp = fopen(PGS_IN, "r"); -puts("# 475:" PGS_IN "\r\n"); - // read file in socket buffer for SocketBackend to consumme. - if (fp) { - fseek(fp, 0L, SEEK_END); - packetlen = ftell(fp); - if (packetlen) { - // set to always true if no REPL. -// is_wire = true; - resetStringInfo(inBuf); - rewind(fp); - /* peek on first char */ - peek = getc(fp); - rewind(fp); - if (is_repl && !is_wire) { - // sql in buffer - for (int i=0; i socketfiles -> repl + whereToSendOutput = DestRemote; #if PGDEBUG if (packetlen) IO[packetlen]=0; // wire blocks are not zero terminated - printf("\n# 524: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d cma=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen,cma_rsize, peek, IO); + printf("\n# 524: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen, peek, IO); #endif resetStringInfo(inBuf); - // when cma buffer is used to fake stdin, data is not read by socket/wire backend. - if (is_repl) { - for (int i=0; i0; +#if PGDEBUG + if (!pipelining) { + printf("# 556: end of wire, rfq=%d\n", send_ready_for_query); + } else { + printf("# 558: no end of wire -> pipelining, rfq=%d\n", send_ready_for_query); + } +#endif } else { /* nowire */ // pipelining = false; @@ -641,111 +395,34 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); if (pipelining) { pipelining = pq_buffer_remaining_data()>0; if (pipelining && send_ready_for_query) { +puts("# 631: PIPELINING + rfq"); ReadyForQuery(whereToSendOutput); send_ready_for_query = false; } } } -resume_on_error: - if (!is_repl) { -wire_flush: - if (!ClientAuthInProgress) { - /* process notifications (SYNC) */ - if (notifyInterruptPending) - ProcessNotifyInterrupt(false); - - if (send_ready_for_query) { -// PDEBUG("# 602: end packet - sending rfq\n"); - ReadyForQuery(DestRemote); - //done at postgres.c 4623 - send_ready_for_query = false; - } else { - PDEBUG("# 606: end packet - with no rfq\n"); - } - } else { - PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); - } - - if (SOCKET_DATA>0) { - if (sockfiles) { - channel = -1; - if (cma_wsize) { - puts("ERROR: cma was not flushed before socketfile interface"); - } - } else { - /* wsize may have increased with previous rfq so assign here */ - cma_wsize = SOCKET_DATA; - channel = cma_rsize + 2; - } - if (SOCKET_FILE) { - int outb = SOCKET_DATA; - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - SOCKET_DATA = 0; - - if (cma_wsize) { - PDEBUG("# 672: cma and sockfile ???\n"); - } - - if (sockfiles) { -#if PGDEBUG - printf("# 675: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); -#endif - rename(PGS_OLOCK, PGS_OUT); - } - } else { -#if PGDEBUG - printf("\n# 681: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); -#endif - SOCKET_DATA = 0; - } +wire_flush: + if (!ClientAuthInProgress) { + /* process notifications (SYNC) */ + if (notifyInterruptPending) + ProcessNotifyInterrupt(false); + if (send_ready_for_query) { + PDEBUG("# 602: end packet - sending rfq\n"); + ReadyForQuery(DestRemote); + //done at postgres.c 4623 + send_ready_for_query = false; } else { - cma_wsize = 0; - PDEBUG("# 698: no data, send empty ?"); -// TODO: dedup 739 - if (sockfiles) { - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - rename(PGS_OLOCK, PGS_OUT); - } + PDEBUG("# 606: end packet - with no rfq\n"); } } else { - pg_prompt(); -#if PGDEBUG - puts("# 683: repl output"); - if (SOCKET_DATA>0) { - puts("# 686: socket has data"); - if (sockfiles) - printf("# 688: socket file not flushed -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", SOCKET_DATA); - } else { -// TODO: dedup 723 - if (sockfiles) { - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - rename(PGS_OLOCK, PGS_OUT); - } - } - if (cma_wsize) - puts("ERROR: cma was not flushed before socketfile interface"); -#endif - } -return_early:; - /* always FD CLEANUP */ - if (fp) { - fclose(fp); - unlink(PGS_IN); + PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); } - - // always free kernel buffer !!! - cma_rsize = 0; - IO[0] = 0; - - #undef IO - +return_early:; // reset EX counter canary_ex = 0; + pq_flush(); } -#undef PGL_LOOP +#undef PGL_LOOP \ No newline at end of file diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index 65e5d7ac01d2c..bbfa698606dcb 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -26,10 +26,12 @@ #include /* chdir */ #include /* mkdir */ +#if defined(__EMSCRIPTEN__) +#include +#endif + // globals -#define MemoryContextResetAndDeleteChildren(...) -// #define SpinLockInit(...) @@ -43,7 +45,6 @@ volatile char *PGUSER; const char *progname; -volatile bool is_repl = true; volatile bool is_node = true; volatile bool is_embed = false; volatile int pgl_idb_status; @@ -60,7 +61,6 @@ volatile int async_restart = 1; #define WASM_PGDATA WASM_PREFIX "/base" -#define CMA_FD 1 extern bool IsPostmasterEnvironment; @@ -92,21 +92,8 @@ volatile bool send_ready_for_query = true; volatile bool idle_in_transaction_timeout_enabled = false; volatile bool idle_session_timeout_enabled = false; - -void -pg_free(void *ptr) { - free(ptr); -} - #include "../backend/tcop/postgres.c" - -// initdb + start on fd (pipe emulation) - - -static bool force_echo = false; - - #include "pgl_mains.c" #include "pgl_stubs.h" @@ -115,16 +102,11 @@ static bool force_echo = false; #include "pgl_initdb.c" - // interactive_one, heart of the async loop. - #include "./interactive_one.c" - static void main_pre(int argc, char *argv[]) { - - char key[256]; int i = 0; // extra env is always after normal args @@ -172,8 +154,8 @@ main_pre(int argc, char *argv[]) { #if defined(__EMSCRIPTEN__) EM_ASM( { Module.is_worker = (typeof WorkerGlobalScope !== 'undefined') && self instanceof WorkerGlobalScope; - Module.FD_BUFFER_MAX = $0; Module.emscripten_copy_to = console.warn;} - , (CMA_MB * 1024 * 1024) / CMA_FD); /* ( global mem start / num fd max ) */ + Module.FD_BUFFER_MAX = $0; Module.emscripten_copy_to = console.warn;} // tdrz: what is FD_BUFFER_MAX? + , (12 * 1024 * 1024)); /* ( global mem start / num fd max ) */ if (is_node) { setenv("ENVIRONMENT", "node", 1); @@ -192,7 +174,6 @@ main_pre(int argc, char *argv[]) { console.warn("prerun(C-web) worker=", Module.is_worker);} ); # endif - is_repl = true; } // *INDENT-OFF* EM_ASM({ @@ -484,7 +465,8 @@ __attribute__ ((export_name("pgl_backend"))) #else remove(IDB_PIPE_BOOT); #endif - stdin = fdopen(saved_stdin, "r"); + // tdrz: I've commented this out!!! + // stdin = fdopen(saved_stdin, "r"); PDEBUG("# 479: initdb faking shutdown to complete WAL/OID states"); pg_proc_exit(66); @@ -560,35 +542,11 @@ PDEBUG("# 498: initdb faking shutdown to complete WAL/OID states in single mode" printf("# 550: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL")); #endif - progname = get_progname(argv[0]); - startup_hacks(progname); - g_argv = argv; - g_argc = argc; - - is_repl = strlen(getenv("REPL")) && getenv("REPL")[0] != 'N'; - is_embed = true; - - if (!is_repl) { - PDEBUG("# 562: exit with live runtime (nodb)"); - return 0; - } -#if defined(__wasi__) - - -#else - /* - main_post(); - - PDEBUG("# 565: repl"); - // so it is repl - main_repl(); + progname = get_progname(argv[0]); + g_argv = argv; + g_argc = argc; + is_embed = true; - if (is_node) { - PDEBUG("# 570: node repl"); - pg_repl_raf(); - } - */ - emscripten_force_exit(exit_code); -#endif + emscripten_force_exit(exit_code); return exit_code; } diff --git a/pglite-wasm/pg_proto.c b/pglite-wasm/pg_proto.c index fd21210f420d3..2ba14a846e5db 100644 --- a/pglite-wasm/pg_proto.c +++ b/pglite-wasm/pg_proto.c @@ -313,7 +313,6 @@ if (sf_connected) { PDEBUG("ERROR: more exits than connections"); } PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); - is_repl = false; ignore_till_sync = false; send_ready_for_query = false; diff --git a/pglite-wasm/pgl_initdb.c b/pglite-wasm/pgl_initdb.c index 19bf7fd405457..9536b257e7cea 100644 --- a/pglite-wasm/pgl_initdb.c +++ b/pglite-wasm/pgl_initdb.c @@ -49,10 +49,5 @@ pg_chmod(const char * path, int mode_t) { #include "bin/initdb/initdb.c" -void use_socketfile(void) { - is_repl = true; - is_embed = false; -} - diff --git a/pglite-wasm/pgl_mains.c b/pglite-wasm/pgl_mains.c index 98f425a6e4116..7dc200745a4d7 100644 --- a/pglite-wasm/pgl_mains.c +++ b/pglite-wasm/pgl_mains.c @@ -45,7 +45,7 @@ interactive_file() { * query input buffer in the cleared MessageContext. */ MemoryContextSwitchTo(MessageContext); - MemoryContextResetAndDeleteChildren(MessageContext); + MemoryContextReset(MessageContext); initStringInfo(&input_message); inBuf = &input_message; diff --git a/pglite-wasm/pgl_sjlj.c b/pglite-wasm/pgl_sjlj.c index 6c05dafa9253f..469053a2705b2 100644 --- a/pglite-wasm/pgl_sjlj.c +++ b/pglite-wasm/pgl_sjlj.c @@ -57,11 +57,6 @@ #endif #if !defined(INITDB_SINGLE) -#if PGDEBUG - if (is_repl) - pg_prompt(); -#endif - if (pq_buffer_remaining_data()>0) { if (canary_ex++ > 8) abort(); diff --git a/pglite-wasm/pgl_stubs.h b/pglite-wasm/pgl_stubs.h index 2ad262c7f0096..5367add820685 100644 --- a/pglite-wasm/pgl_stubs.h +++ b/pglite-wasm/pgl_stubs.h @@ -14,8 +14,6 @@ // option_parse_int parse_sync_method #include "../src/fe_utils/option_utils.c" - - static void init_locale(const char *categoryname, int category, const char *locale) { if (pg_perm_setlocale(category, locale) == NULL && @@ -30,16 +28,7 @@ PostgresMain(const char *dbname, const char *username) { } -void -startup_hacks(const char *progname) { -#ifdef PG16 - SpinLockInit(&dummy_spinlock); -#endif -} - - // embedded initdb requirements - void get_restricted_token(void) { // stub @@ -76,7 +65,6 @@ pg_strdup(const char *in) { return tmp; } - char * simple_prompt(const char *prompt, bool echo) { return pg_strdup(""); diff --git a/pglite-wasm/pglite-modpython.c b/pglite-wasm/pglite-modpython.c deleted file mode 100644 index 8dc492eab99ad..0000000000000 --- a/pglite-wasm/pglite-modpython.c +++ /dev/null @@ -1,218 +0,0 @@ -#include -#include -#include - -#ifdef __wii__ -# include -# include -# include -#endif - -#include "w2c2/w2c2_base.h" -#include "wasi/wasi.h" -#include "${WASM2C}.h" - -void -trap(Trap trap) { - fprintf(stderr, "TRAP: %s\n", trapDescription(trap)); -#ifdef __wii__ - VIDEO_WaitVSync(); -#endif - abort(); -} - -U32 -wasi_snapshot_preview1__fd_renumber(void* ctx, U32 from, U32 to) { - return -1; -} - -wasmMemory* -wasiMemory(void* instance) { - return ${WASM2C}_memory((${WASM2C}Instance*)instance); -} - -extern char **environ; -volatile int skip_main = 0; -//char** __environ = NULL; -${WASM2C}Instance instance; - -#include "${WASM2C}.c" - -#define WASM_PREFIX "/tmp/pglite" -#define WASM_USERNAME "postgres" -#define PGDB WASM_PREFIX "/base" - - -char **g_argv; -int g_argc; - -char **copy_argv(int argc, char *argv[]) { - // calculate the contiguous argv buffer size - int length=0; - size_t ptr_args = argc + 1; - for (int i = 0; i < argc; i++) { - length += (strlen(argv[i]) + 1); - } - char** new_argv = (char**)malloc((ptr_args) * sizeof(char*) + length); - - // copy argv into the contiguous buffer - length = 0; - for (int i = 0; i < argc; i++) { - new_argv[i] = &(((char*)new_argv)[(ptr_args * sizeof(char*)) + length]); - strcpy(new_argv[i], argv[i]); - length += (strlen(argv[i]) + 1); - } - - // insert NULL terminating ptr at the end of the ptr array - new_argv[ptr_args-1] = NULL; - return (new_argv); -} - -int pre_main(int tmp_argc, char* tmp_argv[]) { - puts("# 84: pre_main"); - // __environ = environ; - - setenv("EMBED", "wasi", 1); - setenv("REPL", "N", 1); - - setenv("PGSYSCONFDIR", WASM_PREFIX, 1); - setenv("PGCLIENTENCODING", "UTF8", 1); - - setenv("TZ", "UTC", 1); - setenv("PGTZ", "UTC", 1); - setenv("PGUSER", WASM_USERNAME , 1); - setenv("PGDATA", PGDB , 1); - setenv("PGDATABASE", "template1" , 1); - setenv("PG_COLOR", "always", 0); - puts("# 87: global argc/argv"); - g_argv = copy_argv(tmp_argc, tmp_argv); - g_argc = sizeof(g_argv) / sizeof(char*) - 1; - - -# if defined(__MWERKS__) && defined(macintosh) - MaxApplZone(); - MoreMasters(); - MoreMasters(); - argc = ccommand(&argv); -# elif defined(__wii__) - /* TODO: get interactive console working */ - wiiInitVideo(); - fatInitDefault(); -# endif - - /* Initialize WASI */ - if (!wasiInit(g_argc, g_argv, environ)) { - fprintf(stderr, "failed to init WASI\n"); - return 1; - } - - if (!wasiFileDescriptorAdd(-1, "/", NULL)) { - fprintf(stderr, "failed to add preopen\n"); - return 1; - } - -# ifdef __MSL__ - SIOUXSetTitle("\p${WASM2C}"); -# endif - - ${WASM2C}Instantiate(&instance, NULL); - - return 0; -} - -// #define REACTOR - -#if defined(REACTOR) - #define STARTPROC(i) ${WASM2C}_setup(i) - // #define STARTPROC(i) ${WASM2C}_pg_initdb(i) -#else - #define STARTPROC(i) ${WASM2C}__start(&instance); -#endif - - -int main(int argc, char* argv[]); - -void do_main() { - puts("# 128: do_main Begin"); - STARTPROC(&instance); - puts("# 134: do_main End"); -} - -#if defined(__PYDK__) -# include "Python.h" - - static PyObject * ${WASM2C}_info(PyObject *self, PyObject *args, PyObject *kwds) - { - puts("${WASM2C} test function : return 42"); - return Py_BuildValue("i", 42); - } - - static PyObject * Begin(PyObject *self, PyObject *args, PyObject *kwds) { - puts("PyInit_${WASM2C}"); - - char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; - int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; -puts("167"); - pre_main(tmp_argc, tmp_argv); -puts("169"); - -//${WASM2C}__start(&instance); - Py_RETURN_NONE; - } - - static PyObject * End(PyObject *self, PyObject *args, PyObject *kwds) { -puts("164: do main"); - do_main(); -puts("FREE"); - ${WASM2C}FreeInstance(&instance); - Py_RETURN_NONE; - } - -# include "${WASM2C}.pymod" - - static PyMethodDef mod_${WASM2C}_methods[] = { -#include "${WASM2C}.def" - {NULL, NULL, 0, NULL} - }; - - static struct PyModuleDef mod_${WASM2C} = { - PyModuleDef_HEAD_INIT, - "${WASM2C}", - NULL, - -1, - mod_${WASM2C}_methods, - NULL, // m_slots - NULL, // m_traverse - NULL, // m_clear - NULL, // m_free - }; - - - PyMODINIT_FUNC - PyInit_${WASM2C}(void) { - PyObject *${WASM2C}_mod = PyModule_Create(&mod_${WASM2C}); -# ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(${WASM2C}_mod, Py_MOD_GIL_NOT_USED); -# endif - return ${WASM2C}_mod; - } -#endif - - -int main(int argc, char* argv[]) { - if (!skip_main) { -puts("216"); - char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; - int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; - skip_main = pre_main(tmp_argc, tmp_argv); -puts("220"); - if (!skip_main) { - do_main(); -puts("FREE"); - ${WASM2C}FreeInstance(&instance); - } - } else { - puts(" -- main skipped --"); - } - return skip_main; -} diff --git a/pglite-wasm/repl.html b/pglite-wasm/repl.html deleted file mode 100644 index 15e9ed2d20e1c..0000000000000 --- a/pglite-wasm/repl.html +++ /dev/null @@ -1,1226 +0,0 @@ - - - - - - - PG SHELL TEST - - - - - - - - - - - - - - - - - - -
-
emscripten
-
Downloading...
-
- -
- - - -
- - -
-pglite.wasi: Download file
-
-pglite.wasi + filesystem: Download file
-
-pg_dump.wasm: Download file
-
-
- -
- -
- -
- - - -#if MODULARIZE - - -#endif - - - - - diff --git a/pglite/Makefile b/pglite/Makefile new file mode 100644 index 0000000000000..fc1f84c45bdc1 --- /dev/null +++ b/pglite/Makefile @@ -0,0 +1,39 @@ +# Packages other extensions into individual tar.gz archives + +top_builddir = .. +include $(top_builddir)/src/Makefile.global + +SUBDIRS = \ + pg_ivm \ + vector + +prefix ?= /install/pglite +EXTENSIONS_BUILD_ROOT := /tmp/extensions/build +ARCHIVE_DIR := /install/pglite/extensions + +EXTENSIONS := $(SUBDIRS) + +# Default target: build tarballs for all contribs +dist: $(addsuffix .tar.gz,$(EXTENSIONS)) + +# Pattern rule: build $(EXT).tar.gz for each extra extension +%.tar.gz: + @echo "=== Staging $* ===" + rm -rf $(EXTENSIONS_BUILD_ROOT)/$* + bash -c 'mkdir -p $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix)/{bin,lib,share/extension,share/doc,share/postgresql/extension,share/postgresql/tsearch_data,include}' + $(MAKE) -C $* install DESTDIR=$(EXTENSIONS_BUILD_ROOT)/$* + @echo "=== Packaging $* ===" + mkdir -p $(ARCHIVE_DIR) + cd $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix) && \ + files=$$(find . -type f -o -type l | sed 's|^\./||') && \ + tar -czf $(ARCHIVE_DIR)/$*.tar.gz $$files +# tar -C $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix) -czf $(ARCHIVE_DIR)/$*.tar.gz . + +.PHONY: dist + +archive_clean: + @echo "=== Cleaning $* ===" + rm -rf $(EXTENSIONS_BUILD_ROOT)/$* + rm -rf $(ARCHIVE_DIR) + +$(recurse) diff --git a/pglite/pg_ivm b/pglite/pg_ivm new file mode 160000 index 0000000000000..f4b40e93a6047 --- /dev/null +++ b/pglite/pg_ivm @@ -0,0 +1 @@ +Subproject commit f4b40e93a60478a1ea9d69f0fd305452e5c00690 diff --git a/pglite/vector b/pglite/vector new file mode 160000 index 0000000000000..2627c5ff775ae --- /dev/null +++ b/pglite/vector @@ -0,0 +1 @@ +Subproject commit 2627c5ff775ae6d7aef0c430121ccf857842d2f2 diff --git a/src/Makefile.shlib b/src/Makefile.shlib index aed880df4b876..befb131d977c4 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -225,13 +225,14 @@ ifeq ($(SHLIB_EXPORTS),) endif ifeq ($(PORTNAME), emscripten) - LINK.shared = emsdk-shared + LINK.shared = $(COMPILER) -shared -sSIDE_MODULE=1 -Wno-unused-function ifdef soname # emscripten uses unversioned shared libraries shlib = $(shlib_bare) soname = $(shlib_bare) endif - BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ + BUILD.exports = ( $(AWK) '/^[^\#]/ {printf "%s\n",$$1}' $< ) | sort -u >$@ +# BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ exports_file = $(SHLIB_EXPORTS:%.txt=%.list) # ifneq (,$(exports_file)) # LINK.shared += -Wl,--version-script=$(exports_file) diff --git a/src/backend/Makefile b/src/backend/Makefile index dbc1d4a148f3a..0bc6a84c3b10a 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -75,16 +75,53 @@ endif ifeq ($(PORTNAME), emscripten) AR ?= llvm-ar -LIBPGCORE ?= $(top_builddir)/libpgcore.a -LIBPG = $(top_builddir)/libpostgres.a +LIBPGCORE ?= libpgcore.a +LIBPG = libpostgres.a +LIBPGMAIN = libpgmain.a PGCORE = $(top_builddir)/src/common/libpgcommon_srv.a $(top_builddir)/src/port/libpgport_srv.a $(LIBPG) PGMAIN = main/main.o tcop/postgres.o +PGLITE_MAIN = pglite-wasm/pg_main.c +PGROOT=/install/pglite +PGPRELOAD=--preload-file ${PGROOT}/share/postgresql@/tmp/pglite/share/postgresql --preload-file ${PGROOT}/lib/postgresql@/tmp/pglite/lib/postgresql --preload-file $(top_builddir)/other/password@/tmp/pglite/password --preload-file $(top_builddir)/other/PGPASSFILE@/home/web_user/.pgpass --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/postgres --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/initdb postgres: $(OBJS) - $(AR) rcs $(top_builddir)/libpgmain.a $(PGMAIN) + $(AR) rcs $(LIBPGMAIN) $(PGMAIN) $(AR) rcs $(LIBPG) $(filter-out $(PGMAIN),$(call expand_subsys,$(ONLYOBJS))) $(CC) -r -o $(top_builddir)/libpgcore.o -Wl,--whole-archive $(PGCORE) $(AR) rcs $(LIBPGCORE) $(top_builddir)/libpgcore.o - COPTS="$(LOPTS)" $(CC) $(MAIN_MODULE) $(CFLAGS) $(LDFLAGS) -o $@ $(LIBPGCORE) $(top_builddir)/libpgmain.a $(LIBS) + COPTS="$(LOPTS)" $(CC) $(MAIN_MODULE) $(CFLAGS) $(LDFLAGS) -o $@$(X) $(LIBPGCORE) $(LIBPGMAIN) $(LIBS) + +# Extensions use functions from the core PG. These need to be exported by the emscripten compiler. +# The following target gathers all extension imports + the default ones (included.pglite.imports), +# excludes the one in excluded.pglite.imports and adds a leading _ to each. +pglite-exported-functions: + $(MKDIR_P) '$(emscripten_imports_dir)' + cat $(top_builddir)/pglite-wasm/excluded.*.imports $(top_builddir)/src/interfaces/libpq/exports.list $(LIB_EXPORTS_DIR)/libossp-uuid.exports | sort -u > '$(top_builddir)/pglite-wasm/excluded.imports' + cat $(DESTDIR)$(emscripten_extension_imports_dir)/*.imports '$(top_builddir)/pglite-wasm/included.pglite.imports' | \ + sort -u | \ + grep -Fvx -f '$(top_builddir)/pglite-wasm/excluded.imports' | \ + sed 's/^/_/' \ + > '$(emscripten_imports_dir)/exported_functions.txt' + +# -sDYLINK_DEBUG=2 use this for debugging missing exported symbols (ex when an extension calls a pgcore function that hasn't been exported) +# PGLITE_CFLAGS is something like "-O2" (aka release version) or "-g -gsource-map --no-wasm-opt" (aka debug version) +# PGLITE_EMSCRIPTEN_FLAGS are the emscripten flags to be passed to the build. Ideally they would be completely transparent, such that we +# could build a libpglite with or without those flags (to switch between native and wasm builds) +pglite: pglite-exported-functions + $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers + $(CC) \ + $(PGLITE_CFLAGS) \ + -fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html \ + $(PGPRELOAD) \ + -ferror-limit=1 \ + -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ + -I./src/include -I./src/ -I./src/interfaces/libpq \ + -L/install/libs/lib -L$(top_builddir) -L$(top_builddir)/src/interfaces/libpq/ -L. \ + pglite.o \ + -Ldict_snowball \ + -lxslt \ + -lpgcore \ + -lnodefs.js -lidbfs.js -lxml2 -lz endif ifeq ($(PORTNAME), wasi) @@ -214,6 +251,11 @@ ifeq ($(PORTNAME), win32) ifeq ($(MAKE_DLL), true) $(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a' endif +endif +ifeq ($(PORTNAME), emscripten) + $(INSTALL_DATA) $(LIBPG) '$(DESTDIR)$(libdir)/libpostgres.a' + $(INSTALL_DATA) $(LIBPGCORE) '$(DESTDIR)$(libdir)/libpgcore.a' + $(INSTALL_DATA) $(LIBPGMAIN) '$(DESTDIR)$(libdir)/libpgmain.a' endif $(MAKE) -C catalog install-data $(MAKE) -C tsearch install-data @@ -235,6 +277,9 @@ ifeq ($(MAKE_EXPORTS), true) $(INSTALL_DATA) $(POSTGRES_IMP) '$(DESTDIR)$(pkglibdir)/$(POSTGRES_IMP)' $(INSTALL_PROGRAM) $(MKLDEXPORT) '$(DESTDIR)$(pgxsdir)/$(MKLDEXPORT_DIR)/mkldexport.sh' endif +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) postgres.wasm '$(DESTDIR)$(bindir)/postgres.wasm' +endif .PHONY: install-bin @@ -255,6 +300,16 @@ ifeq ($(MAKE_EXPORTS), true) $(MKDIR_P) '$(DESTDIR)$(pgxsdir)/$(MKLDEXPORT_DIR)' endif +ifeq ($(PORTNAME), emscripten) +install-pglite: pglite + $(INSTALL_PROGRAM) pglite.html '$(DESTDIR)$(bindir)/pglite.html' + $(INSTALL_PROGRAM) pglite.wasm '$(DESTDIR)$(bindir)/pglite.wasm' + $(INSTALL_PROGRAM) pglite.js '$(DESTDIR)$(bindir)/pglite.js' + $(INSTALL_PROGRAM) pglite.data '$(DESTDIR)$(bindir)/pglite.data' +endif + +.PHONY: install-pglite + ########################################################################## @@ -273,6 +328,16 @@ ifeq ($(PORTNAME), win32) ifeq ($(MAKE_DLL), true) rm -f '$(DESTDIR)$(libdir)/libpostgres.a' endif +endif +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(libdir)/libpostgres.a' + rm -f '$(DESTDIR)$(libdir)/libpgcore.a' + rm -f '$(DESTDIR)$(libdir)/libpgmain.a' + rm -f '$(DESTDIR)$(bindir)/postgres.wasm' + rm -f '$(DESTDIR)$(bindir)/pglite.wasm' + rm -f '$(DESTDIR)$(bindir)/pglite.html' + rm -f '$(DESTDIR)$(bindir)/pglite.js' + rm -f '$(DESTDIR)$(bindir)/pglite.data' endif $(MAKE) -C catalog uninstall-data $(MAKE) -C tsearch uninstall-data @@ -295,6 +360,12 @@ endif ifeq ($(PORTNAME), win32) rm -f postgres.dll libpostgres.a $(WIN32RES) endif +ifeq ($(PORTNAME), emscripten) + rm -f postgres.wasm libpostgres.a libpgcore.a libpgmain.a pglite.o pglite.html pglite.js pglite.wasm pglite.data +endif +ifeq ($(PORTNAME), emscripten) + rm -f '$(emscripten_imports_dir)/exported_functions.txt' +endif distclean: clean # generated by configure diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index ff5277a682852..856e9d2a3623c 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -80,6 +80,10 @@ #include "utils/guc_hooks.h" #include "utils/memutils.h" +#if defined(__EMSCRIPTEN__) +#include +#endif + /* * Cope with the various platform-specific ways to spell TCP keepalive socket * options. This doesn't cover Windows, which as usual does its own thing. @@ -122,23 +126,56 @@ static char *PqSendBuffer; static int PqSendBufferSize; /* Size send buffer */ static size_t PqSendPointer; /* Next index to store a byte in PqSendBuffer */ static size_t PqSendStart; /* Next index to send a byte in PqSendBuffer */ -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE]; static int PqRecvPointer; /* Next index to read a byte from PqRecvBuffer */ static int PqRecvLength; /* End of data available in PqRecvBuffer */ -#else -static char PqRecvBuffer_static[PQ_RECV_BUFFER_SIZE]; -static char *PqRecvBuffer; -static int PqRecvPointer; -static int PqRecvLength; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) volatile int querylen = 0; volatile FILE* queryfp = NULL; -#endif -/* pglite specific */ -extern int cma_rsize; -extern bool sockfiles; +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +extern pglite_read_t pglite_read; + +typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); +extern pglite_write_t pglite_write; + +int EMSCRIPTEN_KEEPALIVE fcntl(int __fd, int __cmd, ...) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE setsockopt(int __fd, int __level, int __optname, + const void *__optval, socklen_t __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockopt(int __fd, int __level, int __optname, + void *__restrict __optval, + socklen_t *__restrict __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockname(int __fd, struct sockaddr * __addr, + socklen_t *__restrict __len) { + // dummy + return 0; + } + +ssize_t EMSCRIPTEN_KEEPALIVE + recv(int __fd, void *__buf, size_t __n, int __flags) { + ssize_t got = pglite_read(__buf, __n); + return got; + } +ssize_t EMSCRIPTEN_KEEPALIVE + send(int __fd, const void *__buf, size_t __n, int __flags) { + ssize_t wrote = pglite_write(__buf, __n); + return wrote; + } + +#endif /* * Message status @@ -148,7 +185,6 @@ static bool PqCommReadingMsg; /* in the middle of reading a message */ /* Internal functions */ - static void socket_comm_reset(void); static void socket_close(int code, Datum arg); static void socket_set_nonblocking(bool nonblocking); @@ -162,6 +198,9 @@ static inline int internal_flush(void); static pg_noinline int internal_flush_buffer(const char *buf, size_t *start, size_t *end); +static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); +static int Setup_AF_UNIX(const char *sock_path); + static const PQcommMethods PqCommSocketMethods = { .comm_reset = socket_comm_reset, .flush = socket_flush, @@ -171,10 +210,6 @@ static const PQcommMethods PqCommSocketMethods = { .putmessage_noblock = socket_putmessage_noblock }; -static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); -static int Setup_AF_UNIX(const char *sock_path); - - const PQcommMethods *PqCommMethods = &PqCommSocketMethods; WaitEventSet *FeBeWaitSet; @@ -196,7 +231,6 @@ pq_init(ClientSocket *client_sock) port->sock = client_sock->sock; memcpy(&port->raddr.addr, &client_sock->raddr.addr, client_sock->raddr.salen); port->raddr.salen = client_sock->raddr.salen; -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* fill in the server (local) address */ port->laddr.salen = sizeof(port->laddr.addr); if (getsockname(port->sock, @@ -288,7 +322,6 @@ pq_init(ClientSocket *client_sock) (void) pq_setkeepalivescount(tcp_keepalives_count, port); (void) pq_settcpusertimeout(tcp_user_timeout, port); } -#endif /* WASM */ /* initialize state variables */ PqSendBufferSize = PQ_SEND_BUFFER_SIZE; @@ -296,7 +329,6 @@ pq_init(ClientSocket *client_sock) PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0; PqCommBusy = false; PqCommReadingMsg = false; -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* set up process-exit hook to close the socket */ on_proc_exit(socket_close, 0); @@ -326,10 +358,6 @@ pq_init(ClientSocket *client_sock) MyLatch, NULL); AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); -#else /* WASM */ - /* because we may fill before starting reading message */ - PqRecvBuffer = &PqRecvBuffer_static[0]; -#endif /* WASM */ /* * The event positions match the order we added them, but let's sanity * check them to be sure. @@ -749,7 +777,7 @@ Setup_AF_UNIX(const char *sock_path) Assert(Unix_socket_group); if (Unix_socket_group[0] != '\0') { -#if defined(WIN32) || defined(__wasi__) +#ifdef WIN32 elog(WARNING, "configuration item \"unix_socket_group\" is not supported on this platform"); #else char *endptr; @@ -928,20 +956,6 @@ pq_recvbuf(void) else PqRecvLength = PqRecvPointer = 0; } -#if defined(__EMSCRIPTEN__) || defined(__wasi__) - if (queryfp && querylen) { - int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); - querylen -= got; - PqRecvLength += got; - if (querylen<=0) { - PDEBUG("# 931: could close fp early here " __FILE__); - queryfp = NULL; - } - if (got>0) - return 0; - } - return EOF; -#endif /* Ensure that we're in blocking mode */ socket_set_nonblocking(false); @@ -1044,9 +1058,7 @@ pq_getbyte_if_available(unsigned char *c) *c = PqRecvBuffer[PqRecvPointer++]; return 1; } -#if defined(__EMSCRIPTEN__) || (__wasi__) -puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); -#else + /* Put the socket into non-blocking mode */ socket_set_nonblocking(true); @@ -1083,7 +1095,7 @@ puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); /* EOF detected */ r = EOF; } -#endif + return r; } @@ -1150,7 +1162,6 @@ pq_discardbytes(size_t len) return 0; } - /* -------------------------------- * pq_buffer_remaining_data - return number of bytes in receive buffer * @@ -1172,26 +1183,6 @@ pq_buffer_remaining_data(void) * This must be called before any of the pq_get* functions. * -------------------------------- */ -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -EMSCRIPTEN_KEEPALIVE void -pq_recvbuf_fill(FILE* fp, int packetlen) { - if (packetlen>PQ_RECV_BUFFER_SIZE) { - int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); - queryfp = fp; - querylen = packetlen - got; - PqRecvLength = got; - } else { - fread( PqRecvBuffer, packetlen, 1, fp); - PqRecvLength = packetlen; - queryfp = NULL; - querylen = 0; - } - PqRecvPointer = 0; -} - -#endif - -static char * PqSendBuffer_save; void pq_startmsgread(void) { @@ -1203,29 +1194,7 @@ pq_startmsgread(void) ereport(FATAL, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("terminating connection because protocol synchronization was lost"))); -#if defined(__EMSCRIPTEN__) || defined(__wasi__) - if (!pq_buffer_remaining_data()) { - if (sockfiles) { - PqRecvBuffer = &PqRecvBuffer_static[0]; - if (PqSendBuffer_save) - PqSendBuffer=PqSendBuffer_save; - PqSendBufferSize = PQ_SEND_BUFFER_SIZE; - } else { - PqRecvPointer = 0; - PqRecvLength = cma_rsize; - PqRecvBuffer = (char*)0x1; - - PqSendPointer = 0; - PqSendBuffer_save = PqSendBuffer; - PqSendBuffer = 2 + (char*)(cma_rsize); - PqSendBufferSize = (CMA_MB*1024*1024) - (int)(&PqSendBuffer[0]); - } - } -#if PDEBUG - printf("# 1225: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); -#endif -#endif PqCommReadingMsg = true; } @@ -1348,56 +1317,7 @@ pq_getmessage(StringInfo s, int maxlen) return 0; } -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; -static int -internal_putbytes(const char *s, size_t len) { - size_t amount; - if (!sockfiles) { - while (len > 0) { - /* If buffer is full, then flush it out from cma to file and continue from there */ - if (PqSendPointer >= PqSendBufferSize) { - int redirected = fwrite(PqSendBuffer, 1, PqSendPointer, SOCKET_FILE); - sockfiles = true; -#if PGDEBUG - fprintf(stderr, "# 1364: overflow %zu >= %d redirect=%d cma_rsize=%d CMA_MB=%d \n", PqSendPointer, PqSendBufferSize, redirected, cma_rsize, CMA_MB); -#endif - break; - } - amount = PqSendBufferSize - PqSendPointer; - if (amount > len) - amount = len; - memcpy(PqSendBuffer + PqSendPointer, s, amount); - PqSendPointer += amount; - s += amount; - len -= amount; - SOCKET_DATA+=amount; - } - } - - if (sockfiles) { - int wc= fwrite(s, 1, len, SOCKET_FILE); - SOCKET_DATA+=wc; - } - return 0; -} - -static int -socket_flush(void) { - return internal_flush(); -} - -static int -internal_flush(void) { - /* no flush for raw wire */ - if (sockfiles) { - PqSendStart = PqSendPointer = 0; - } - return 0; -} -#else static inline int internal_putbytes(const char *s, size_t len) @@ -1548,7 +1468,7 @@ internal_flush_buffer(const char *buf, size_t *start, size_t *end) *start = *end = 0; return 0; } -#endif /* wasm */ + /* -------------------------------- * pq_flush_if_writable - flush pending output if writable without blocking * diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 72481df9bd16a..bbcf4112aead6 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -108,13 +108,13 @@ int client_connection_check_interval = 0; int restrict_nonsystem_relation_kind; #if (defined(__EMSCRIPTEN__) || defined(__wasi__)) -#if !defined(PGL_MAIN) - volatile int cma_rsize = 0; - volatile bool sockfiles = false; -#endif // PGL_MAIN bool quote_all_identifiers = false; -FILE* SOCKET_FILE = NULL; -int SOCKET_DATA = 0; + +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +pglite_read_t pglite_read = NULL; + +typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); +pglite_write_t pglite_write = NULL; #endif // WASM diff --git a/src/bin/initdb/Makefile b/src/bin/initdb/Makefile index 031cc77c9d61b..a960ce6580188 100644 --- a/src/bin/initdb/Makefile +++ b/src/bin/initdb/Makefile @@ -47,16 +47,25 @@ localtime.c: % : $(top_srcdir)/src/timezone/% install: all installdirs $(INSTALL_PROGRAM) initdb$(X) '$(DESTDIR)$(bindir)/initdb$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) initdb.wasm '$(DESTDIR)$(bindir)/initdb.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/initdb$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/initdb.wasm' +endif clean distclean: rm -f initdb$(X) $(OBJS) localtime.c rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f initdb.wasm +endif # ensure that changes in datadir propagate into object file initdb.o: initdb.c $(top_builddir)/src/Makefile.global diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 9aa9b48a79325..eb4699868b44e 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -1245,7 +1245,14 @@ test_specific_config_settings(int test_conns, int test_buffs) DEVNULL, DEVNULL); fflush(NULL); + #if defined(__EMSCRIPTEN__) + // emscripten supports (some?) syscalls, but that's not actually what we want, because we want to remain inside the wasm sandbox + // so just return a dummy value until we decide how to handle syscalls. + // TODO: https://github.com/electric-sql/pglite/issues/798 + status = 123; + #else status = system(cmd.data); + #endif // #if defined(__EMSCRIPTEN__) termPQExpBuffer(&cmd); diff --git a/src/bin/pg_amcheck/Makefile b/src/bin/pg_amcheck/Makefile index f9488c447a887..765032692b22a 100644 --- a/src/bin/pg_amcheck/Makefile +++ b/src/bin/pg_amcheck/Makefile @@ -33,16 +33,25 @@ pg_amcheck: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_amcheck$(X) '$(DESTDIR)$(bindir)/pg_amcheck$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_amcheck.wasm '$(DESTDIR)$(bindir)/pg_amcheck.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_amcheck$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_amcheck.wasm' +endif clean distclean: rm -f pg_amcheck$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_amcheck.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_archivecleanup/Makefile b/src/bin/pg_archivecleanup/Makefile index 93fd703f22591..aaeac7ffd3524 100644 --- a/src/bin/pg_archivecleanup/Makefile +++ b/src/bin/pg_archivecleanup/Makefile @@ -18,16 +18,25 @@ pg_archivecleanup: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_archivecleanup$(X) '$(DESTDIR)$(bindir)/pg_archivecleanup$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_archivecleanup.wasm '$(DESTDIR)$(bindir)/pg_archivecleanup.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_archivecleanup$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_archivecleanup.wasm' +endif clean distclean: rm -f pg_archivecleanup$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_archivecleanup.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile index 26c53e473f560..9b5df3ca2ecd3 100644 --- a/src/bin/pg_basebackup/Makefile +++ b/src/bin/pg_basebackup/Makefile @@ -63,6 +63,12 @@ install: all installdirs $(INSTALL_PROGRAM) pg_createsubscriber$(X) '$(DESTDIR)$(bindir)/pg_createsubscriber$(X)' $(INSTALL_PROGRAM) pg_receivewal$(X) '$(DESTDIR)$(bindir)/pg_receivewal$(X)' $(INSTALL_PROGRAM) pg_recvlogical$(X) '$(DESTDIR)$(bindir)/pg_recvlogical$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_basebackup.wasm '$(DESTDIR)$(bindir)/pg_basebackup.wasm' + $(INSTALL_PROGRAM) pg_createsubscriber.wasm '$(DESTDIR)$(bindir)/pg_createsubscriber.wasm' + $(INSTALL_PROGRAM) pg_receivewal.wasm '$(DESTDIR)$(bindir)/pg_receivewal.wasm' + $(INSTALL_PROGRAM) pg_recvlogical.wasm '$(DESTDIR)$(bindir)/pg_recvlogical.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -72,12 +78,21 @@ uninstall: rm -f '$(DESTDIR)$(bindir)/pg_createsubscriber$(X)' rm -f '$(DESTDIR)$(bindir)/pg_receivewal$(X)' rm -f '$(DESTDIR)$(bindir)/pg_recvlogical$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_basebackup.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_createsubscriber.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_receivewal.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_recvlogical.wasm' +endif clean distclean: rm -f pg_basebackup$(X) pg_createsubscriber$(X) pg_receivewal$(X) pg_recvlogical$(X) \ $(BBOBJS) pg_createsubscriber.o pg_receivewal.o pg_recvlogical.o \ $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_basebackup.wasm pg_createsubscriber.wasm pg_receivewal.wasm pg_recvlogical.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_checksums/Makefile b/src/bin/pg_checksums/Makefile index 31de5fb467344..f524f483fe2a7 100644 --- a/src/bin/pg_checksums/Makefile +++ b/src/bin/pg_checksums/Makefile @@ -29,16 +29,25 @@ pg_checksums: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_checksums$(X) '$(DESTDIR)$(bindir)/pg_checksums$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_checksums.wasm '$(DESTDIR)$(bindir)/pg_checksums.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_checksums$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_checksums.wasm' +endif clean distclean: rm -f pg_checksums$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_checksums.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile index c3729755ba4ba..167ec4be960f0 100644 --- a/src/bin/pg_combinebackup/Makefile +++ b/src/bin/pg_combinebackup/Makefile @@ -35,16 +35,25 @@ pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_combinebackup$(X) '$(DESTDIR)$(bindir)/pg_combinebackup$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_combinebackup.wasm '$(DESTDIR)$(bindir)/pg_combinebackup.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_combinebackup$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_combinebackup.wasm' +endif clean distclean maintainer-clean: rm -f pg_combinebackup$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_combinebackup.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_config/Makefile b/src/bin/pg_config/Makefile index c54f68c9daddd..3d3c44fa2b4b8 100644 --- a/src/bin/pg_config/Makefile +++ b/src/bin/pg_config/Makefile @@ -19,23 +19,50 @@ OBJS = \ $(WIN32RES) \ pg_config.o +all: +ifeq ($(PORTNAME), emscripten) +all: pg_config pg_config_sh +else all: pg_config +endif pg_config: $(OBJS) | submake-libpgport $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +# the emscripten build generates multiple "runnable" files (.cjs/.mjs + .wasm) +# but other postgres tools expect exacly 'pg_config' as an executable +# the following rule creates a node script that calls the pg_config.cjs/.mjs +# pg_config.wasm is implicitly expected to be in the same folder +pg_config_sh: pg_config + echo "#!/usr/bin/env node" > pg_config + echo 'require("./pg_config$(X)")' >> pg_config + chmod +x pg_config + install: all installdirs $(INSTALL_SCRIPT) pg_config$(X) '$(DESTDIR)$(bindir)/pg_config$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_config.wasm '$(DESTDIR)$(bindir)/pg_config.wasm' + $(INSTALL_PROGRAM) pg_config '$(DESTDIR)$(bindir)/pg_config' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_config$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_config.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_config' +endif + clean distclean: rm -f pg_config$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_config.wasm + rm -f pg_config +endif check: $(prove_check) diff --git a/src/bin/pg_controldata/Makefile b/src/bin/pg_controldata/Makefile index c3f64e189690a..0be4e65656b07 100644 --- a/src/bin/pg_controldata/Makefile +++ b/src/bin/pg_controldata/Makefile @@ -26,16 +26,25 @@ pg_controldata: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_controldata$(X) '$(DESTDIR)$(bindir)/pg_controldata$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_controldata.wasm '$(DESTDIR)$(bindir)/pg_controldata.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_controldata$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_controldata.wasm' +endif clean distclean: rm -f pg_controldata$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_controldata.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_ctl/Makefile b/src/bin/pg_ctl/Makefile index a3fbad7a3a5a5..514cd8cf032f9 100644 --- a/src/bin/pg_ctl/Makefile +++ b/src/bin/pg_ctl/Makefile @@ -35,16 +35,25 @@ pg_ctl: $(OBJS) | submake-libpgport $(SUBMAKE_LIBPQ) install: all installdirs $(INSTALL_PROGRAM) pg_ctl$(X) '$(DESTDIR)$(bindir)/pg_ctl$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_ctl.wasm '$(DESTDIR)$(bindir)/pg_ctl.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_ctl$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_ctl.wasm' +endif clean distclean: rm -f pg_ctl$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_ctl.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 930c741c95d85..cdd9cfcc5de96 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -57,6 +57,11 @@ install: all installdirs $(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X) $(INSTALL_PROGRAM) pg_restore$(X) '$(DESTDIR)$(bindir)'/pg_restore$(X) $(INSTALL_PROGRAM) pg_dumpall$(X) '$(DESTDIR)$(bindir)'/pg_dumpall$(X) +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_dump.wasm '$(DESTDIR)$(bindir)/pg_dump.wasm' + $(INSTALL_PROGRAM) pg_restore.wasm '$(DESTDIR)$(bindir)/pg_restore.wasm' + $(INSTALL_PROGRAM) pg_dumpall.wasm '$(DESTDIR)$(bindir)/pg_dumpall.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -69,7 +74,13 @@ installcheck: uninstall: rm -f $(addprefix '$(DESTDIR)$(bindir)'/, pg_dump$(X) pg_restore$(X) pg_dumpall$(X)) +ifeq ($(PORTNAME), emscripten) + rm -f $(addprefix '$(DESTDIR)$(bindir)'/, pg_dump.wasm pg_restore.wasm pg_dumpall.wasm) +endif clean distclean: rm -f pg_dump$(X) pg_restore$(X) pg_dumpall$(X) $(OBJS) pg_dump.o common.o pg_dump_sort.o pg_restore.o pg_dumpall.o rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_dump.wasm pg_restore.wasm pg_dumpall.wasm +endif \ No newline at end of file diff --git a/src/bin/pg_resetwal/Makefile b/src/bin/pg_resetwal/Makefile index 4228a5a772a9f..ff8e5d642a1f6 100644 --- a/src/bin/pg_resetwal/Makefile +++ b/src/bin/pg_resetwal/Makefile @@ -28,16 +28,25 @@ pg_resetwal: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_resetwal$(X) '$(DESTDIR)$(bindir)/pg_resetwal$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_resetwal.wasm '$(DESTDIR)$(bindir)/pg_resetwal.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_resetwal$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_resetwal.wasm' +endif clean distclean: rm -f pg_resetwal$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_resetwal.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefile index 12b138b2f2ce3..e0a5f5dd2c38f 100644 --- a/src/bin/pg_rewind/Makefile +++ b/src/bin/pg_rewind/Makefile @@ -42,16 +42,25 @@ xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% install: all installdirs $(INSTALL_PROGRAM) pg_rewind$(X) '$(DESTDIR)$(bindir)/pg_rewind$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_rewind.wasm '$(DESTDIR)$(bindir)/pg_rewind.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_rewind$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_rewind.wasm' +endif clean distclean: rm -f pg_rewind$(X) $(OBJS) xlogreader.c rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_rewind.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_test_fsync/Makefile b/src/bin/pg_test_fsync/Makefile index 4c5e518125033..e3bb3527a8858 100644 --- a/src/bin/pg_test_fsync/Makefile +++ b/src/bin/pg_test_fsync/Makefile @@ -18,6 +18,9 @@ pg_test_fsync: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_test_fsync$(X) '$(DESTDIR)$(bindir)/pg_test_fsync$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_test_fsync.wasm '$(DESTDIR)$(bindir)/pg_test_fsync.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -30,7 +33,13 @@ installcheck: uninstall: rm -f '$(DESTDIR)$(bindir)/pg_test_fsync$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_test_fsync.wasm' +endif clean distclean: rm -f pg_test_fsync$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_test_fsync.wasm +endif \ No newline at end of file diff --git a/src/bin/pg_test_timing/Makefile b/src/bin/pg_test_timing/Makefile index 7f677edadb30f..eb4f95c2bf088 100644 --- a/src/bin/pg_test_timing/Makefile +++ b/src/bin/pg_test_timing/Makefile @@ -18,6 +18,9 @@ pg_test_timing: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_test_timing$(X) '$(DESTDIR)$(bindir)/pg_test_timing$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_test_timing.wasm '$(DESTDIR)$(bindir)/pg_test_timing.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -30,7 +33,13 @@ installcheck: uninstall: rm -f '$(DESTDIR)$(bindir)/pg_test_timing$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_test_timing.wasm' +endif clean distclean: rm -f pg_test_timing$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_test_timing.wasm +endif \ No newline at end of file diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile index bde91e2beb82c..faa309ac31691 100644 --- a/src/bin/pg_upgrade/Makefile +++ b/src/bin/pg_upgrade/Makefile @@ -42,17 +42,26 @@ pg_upgrade: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_upgrade$(X) '$(DESTDIR)$(bindir)/pg_upgrade$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_upgrade.wasm '$(DESTDIR)$(bindir)/pg_upgrade.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_upgrade$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_upgrade.wasm' +endif clean distclean: rm -f pg_upgrade$(X) $(OBJS) rm -rf delete_old_cluster.sh log/ tmp_check/ \ reindex_hash.sql +ifeq ($(PORTNAME), emscripten) + rm -f pg_upgrade.wasm +endif export with_icu diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile index 7c045f142e8d7..e49278f32d28c 100644 --- a/src/bin/pg_verifybackup/Makefile +++ b/src/bin/pg_verifybackup/Makefile @@ -30,16 +30,25 @@ pg_verifybackup: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_verifybackup$(X) '$(DESTDIR)$(bindir)/pg_verifybackup$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_verifybackup.wasm '$(DESTDIR)$(bindir)/pg_verifybackup.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_verifybackup$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_verifybackup.wasm' +endif clean distclean: rm -f pg_verifybackup$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_verifybackup.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile index 4c1ee649501f4..1f08027439af9 100644 --- a/src/bin/pg_waldump/Makefile +++ b/src/bin/pg_waldump/Makefile @@ -38,16 +38,25 @@ $(RMGRDESCSOURCES): % : $(top_srcdir)/src/backend/access/rmgrdesc/% install: all installdirs $(INSTALL_PROGRAM) pg_waldump$(X) '$(DESTDIR)$(bindir)/pg_waldump$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_waldump.wasm '$(DESTDIR)$(bindir)/pg_waldump.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_waldump$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_waldump.wasm' +endif clean distclean: rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c xlogstats.c rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_waldump.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_walsummary/Makefile b/src/bin/pg_walsummary/Makefile index 1886c39e98b19..c686d9450641a 100644 --- a/src/bin/pg_walsummary/Makefile +++ b/src/bin/pg_walsummary/Makefile @@ -31,16 +31,25 @@ pg_walsummary: $(OBJS) | submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_walsummary$(X) '$(DESTDIR)$(bindir)/pg_walsummary$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_walsummary.wasm '$(DESTDIR)$(bindir)/pg_walsummary.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_walsummary$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_walsummary.wasm' +endif clean distclean maintainer-clean: rm -f pg_walsummary$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_walsummary.wasm +endif check: $(prove_check) diff --git a/src/bin/pgbench/Makefile b/src/bin/pgbench/Makefile index 987bf64df9de0..604f9493b35af 100644 --- a/src/bin/pgbench/Makefile +++ b/src/bin/pgbench/Makefile @@ -40,17 +40,26 @@ exprparse.o exprscan.o: exprparse.h install: all installdirs $(INSTALL_PROGRAM) pgbench$(X) '$(DESTDIR)$(bindir)/pgbench$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pgbench.wasm '$(DESTDIR)$(bindir)/pgbench.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pgbench$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pgbench.wasm' +endif clean distclean: rm -f pgbench$(X) $(OBJS) rm -rf tmp_check rm -f exprparse.h exprparse.c exprscan.c +ifeq ($(PORTNAME), emscripten) + rm -f pgbench.wasm +endif check: $(prove_check) diff --git a/src/bin/psql/Makefile b/src/bin/psql/Makefile index 374c4c3ab8f83..9c17566be2f2f 100644 --- a/src/bin/psql/Makefile +++ b/src/bin/psql/Makefile @@ -65,17 +65,26 @@ psqlscanslash.c: FLEX_FIX_WARNING=yes install: all installdirs $(INSTALL_PROGRAM) psql$(X) '$(DESTDIR)$(bindir)/psql$(X)' $(INSTALL_DATA) $(srcdir)/psqlrc.sample '$(DESTDIR)$(datadir)/psqlrc.sample' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) psql.wasm '$(DESTDIR)$(bindir)/psql.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' '$(DESTDIR)$(datadir)' uninstall: rm -f '$(DESTDIR)$(bindir)/psql$(X)' '$(DESTDIR)$(datadir)/psqlrc.sample' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/psql.wasm' +endif clean distclean: rm -f psql$(X) $(OBJS) lex.backup rm -rf tmp_check rm -f sql_help.h sql_help.c psqlscanslash.c +ifeq ($(PORTNAME), emscripten) + rm -f psql.wasm +endif check: $(prove_check) diff --git a/src/bin/scripts/Makefile b/src/bin/scripts/Makefile index 9633c99136880..f155e24f0c395 100644 --- a/src/bin/scripts/Makefile +++ b/src/bin/scripts/Makefile @@ -41,17 +41,33 @@ install: all installdirs $(INSTALL_PROGRAM) vacuumdb$(X) '$(DESTDIR)$(bindir)'/vacuumdb$(X) $(INSTALL_PROGRAM) reindexdb$(X) '$(DESTDIR)$(bindir)'/reindexdb$(X) $(INSTALL_PROGRAM) pg_isready$(X) '$(DESTDIR)$(bindir)'/pg_isready$(X) +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) createdb.wasm '$(DESTDIR)$(bindir)'/createdb.wasm + $(INSTALL_PROGRAM) dropdb.wasm '$(DESTDIR)$(bindir)'/dropdb.wasm + $(INSTALL_PROGRAM) createuser.wasm '$(DESTDIR)$(bindir)'/createuser.wasm + $(INSTALL_PROGRAM) dropuser.wasm '$(DESTDIR)$(bindir)'/dropuser.wasm + $(INSTALL_PROGRAM) clusterdb.wasm '$(DESTDIR)$(bindir)'/clusterdb.wasm + $(INSTALL_PROGRAM) vacuumdb.wasm '$(DESTDIR)$(bindir)'/vacuumdb.wasm + $(INSTALL_PROGRAM) reindexdb.wasm '$(DESTDIR)$(bindir)'/reindexdb.wasm + $(INSTALL_PROGRAM) pg_isready.wasm '$(DESTDIR)$(bindir)'/pg_isready.wasm +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f $(addprefix '$(DESTDIR)$(bindir)'/, $(addsuffix $(X), $(PROGRAMS))) +ifeq ($(PORTNAME), emscripten) + rm -f $(addprefix '$(DESTDIR)$(bindir)'/, $(addsuffix .wasm, $(PROGRAMS))) +endif clean distclean: rm -f $(addsuffix $(X), $(PROGRAMS)) $(addsuffix .o, $(PROGRAMS)) rm -f common.o $(WIN32RES) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f $(addsuffix .wasm, $(PROGRAMS)) +endif export with_icu diff --git a/src/include/port/emscripten.h b/src/include/port/emscripten.h index db4903aa48f78..fb647c0d45199 100644 --- a/src/include/port/emscripten.h +++ b/src/include/port/emscripten.h @@ -4,10 +4,13 @@ #define I_WASM #if !defined(__cplusplus) -#include +// #include +// #include "/tmp/pglite/include/sdk_port.h" +// #include #endif -#include "/tmp/pglite/include/wasm_common.h" +// #include "/tmp/pglite/include/wasm_common.h" +#include "port/wasm_common.h" #define BOOT_END_MARK "build indices" diff --git a/src/include/port/pg_debug.h b/src/include/port/pg_debug.h new file mode 100644 index 0000000000000..73da07ddb9747 --- /dev/null +++ b/src/include/port/pg_debug.h @@ -0,0 +1,8 @@ +#ifndef I_PGDEBUG +#define I_PGDEBUG +#define WASM_USERNAME "postgres" +#define PDEBUG(string) +#define JSDEBUG(string) +#define ADEBUG(string) +#define PGDEBUG 0 +#endif diff --git a/src/include/port/sdk_port.h b/src/include/port/sdk_port.h new file mode 100644 index 0000000000000..7e6b178dce493 --- /dev/null +++ b/src/include/port/sdk_port.h @@ -0,0 +1,262 @@ +#if defined(__EMSCRIPTEN__) +#include + +#elif defined(__wasi__) + + +#ifndef I_WASI +#define I_WASI + +#undef HAVE_PTHREAD + +#if defined(HAVE_SETSID) +#undef HAVE_SETSID +#endif + +#if defined(HAVE_GETRLIMIT) +#undef HAVE_GETRLIMIT +#endif + + +#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC + +#define EMSCRIPTEN_KEEPALIVE __attribute__((used)) +#define __declspec( dllimport ) __attribute__((used)) + +#define em_callback_func void +#define emscripten_set_main_loop(...) +#define emscripten_force_exit(...) +#define EM_JS(...) + +#include "wasm_common.h" + + +static pid_t +fork(void) { + puts("# 31: fork -1"); + return -1; +} + +// ======== signal ======================== +#define SA_RESTART 4 +#define SIG_SETMASK 2 + +#define SIG_BLOCK 0 +#define SIG_UNBLOCK 1 + +/* A signal handler. */ +typedef void (*handler_t) (int signal); +typedef unsigned char sigset_t; +typedef void (*__sighandler_t) (int); + +struct sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; +#ifdef SA_RESTORER + __sigrestore_t sa_restorer; +#endif + sigset_t sa_mask; +}; +extern int sigemptyset(sigset_t *set); +extern int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); +extern int sigdelset (sigset_t *set, int sig); +extern int sigfillset (sigset_t *set); +extern int sigprocmask (int operation, const sigset_t *set, sigset_t *old_set); +extern int sigaddset (sigset_t *set, int sig); + +// STUBS +extern int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); +extern int sigismember(const sigset_t *set, int signum); +extern int sigpending(sigset_t *set); +extern int sigwait(const sigset_t *restrict set, int *restrict sig); + +// ==================================================== +// unistd +extern unsigned int alarm(unsigned int seconds); + +// ==================================================== + + +// WIP : + +#include +static uid_t +getuid(void) { + return 1000; +} + +static int +dup(int fd) { + puts("# 128: dup"); + return fd; +} +static int +dup2(int old, int new) { + puts("# 140: dup2"); + return -1; +} +static int +pipe(int fd[2]) { + puts("# 145: pipe"); + abort(); + return -1; +} + +#include +#define RLIMIT_NOFILE 7 +#define RLIMIT_STACK 3 +#define RLIM_INFINITY ((unsigned long int)(~0UL)) + +struct rlimit { + unsigned long rlim_cur; + unsigned long rlim_max; +}; +static int +getrlimit(int resource, struct rlimit *rlim) { + return -1; +} + + +static const char *gai_strerror_msg = "# 165: gai_strerror_msg"; +static const char * +gai_strerror(int errcode) { + return gai_strerror_msg; +} + + + + +// WIP: semaphores here +// ================================================================== +#include + +static int +semctl(int semid, int semnum, int cmd, ...) { + return 0; // -1; +} + +static int +semget(key_t key, int nsems, int semflg) { +#if 0 // PGDEBUG + printf("# 213: semget(key_t key = %d, int nsems=%d, int semflg=%d)\n", key, nsems, semflg); +#endif + return 1; +} + +static int +semop(int semid, struct sembuf *sops, size_t nsops) { + return 0; // -1; +} + + + +#include +#if defined(PYDK) +extern int shm_open(const char *name, int oflag, mode_t mode); +extern int shm_unlink(const char *name); +#else +static int +shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + snprintf(tmpnam, 128, "/tmp%s", name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 212: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} + +static int +shm_unlink(const char *name) { + char tmpnam[128]; + snprintf(tmpnam, 128, "/tmp%s", name); + fprintf(stderr, "# 220: shm_unlink(%s) STUB\n", tmpnam); + return remove(tmpnam); // -1 +} + +#endif + + + +#define system(command) system_wasi(command) +extern int system_wasi(const char *command); + + + +// time.h + +static void +tzset(void) { + puts("# 241: tzset(void) STUB"); +} + +#if defined(PG_INITDB) || defined(FE_UTILS_PRINT) || defined(PG_DUMP_PARALLEL) +static void +__SIG_IGN(int param) { +} +#endif + + +extern void sock_flush(); + + +// TODO: socket here +// ================================================================== + +/* +#include + +extern ssize_t sdk_recv(int sockfd, void *buf, size_t len, int flags); +extern ssize_t recvfrom(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + +extern ssize_t sdk_send(int sockfd, const void *buf, size_t len, int flags); +extern ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, void *dest_addr, socklen_t addrlen); + +#define recv(sockfd, buf, len, flags) sdk_recv(sockfd, buf, len, flags) + + + + +static int +listen(int sockfd, int backlog) { + return 0; +} + +static struct group *_Nullable +getgrnam(const char *_Nullable name) { + return NULL; +} + + +static int +getaddrinfo(const char *restrict node, + const char *restrict service, + void *restrict hints, + void **restrict res) { + puts("# 60: getaddrinfo"); + return -1; +} +static void +freeaddrinfo(void *res) { + puts("# 65: freeaddrinfo"); +} + +extern ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + +*/ + + +//#define pthread_mutex_lock(mut) sdk_pthread_mutex_lock(mut) +//extern int sdk_pthread_mutex_lock(void *mutex); + +/* + int pthread_mutex_lock(pthread_mutex_t *mutex); + int pthread_mutex_trylock(pthread_mutex_t *mutex); + int pthread_mutex_unlock(pthread_mutex_t *mutex); +*/ + + +#endif // I_WASI + +#else + #error "unknown port mode should be __EMSCRIPTEN__ or __wasi__" +#endif // __EMSCRIPTEN__ diff --git a/src/include/port/wasm_common.h b/src/include/port/wasm_common.h index 6f53f0fc5bb92..1801fb0468c01 100644 --- a/src/include/port/wasm_common.h +++ b/src/include/port/wasm_common.h @@ -45,7 +45,7 @@ # define PG_PLUGIN_INCLUDE "/pgdata/pg_plugin.h" #endif -#include "pg_debug.h" +#include "port/pg_debug.h" // #define COPY_INTERNAL #define COPY_OFF @@ -102,30 +102,7 @@ extern int pg_valid_server_encoding_id_private(int encoding); #define proc_exit(arg) pg_proc_exit(arg) -/* -extern FILE* IDB_PIPE_FP; -extern int IDB_STAGE; -*/ -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; -extern int pgl_pclose(FILE *stream); -/* -#if !defined(PGL_MAIN) && !defined(PGL_INITDB_MAIN) -# if !defined(PG_EXEC) extern int pgl_pclose(FILE *stream); -#define pclose(stream) pg_pclose(stream) -# else -#if 1 -#include -int -pg_pclose(FILE *stream) { - puts("# 118:" __FILE__ " int pg_pclose(FILE *stream) STUB"); - return 0; -} -#endif -# endif // PG_EXEC -#endif // pgl -*/ /* * OpenPipeStream : another kind of pipe open in fd.c diff --git a/src/interfaces/ecpg/preproc/Makefile b/src/interfaces/ecpg/preproc/Makefile index 934b7cef1b0f5..ea5cbb1594caa 100644 --- a/src/interfaces/ecpg/preproc/Makefile +++ b/src/interfaces/ecpg/preproc/Makefile @@ -83,14 +83,23 @@ keywords.o: $(top_srcdir)/src/include/parser/kwlist.h install: all installdirs $(INSTALL_PROGRAM) ecpg$(X) '$(DESTDIR)$(bindir)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) ecpg.wasm '$(DESTDIR)$(bindir)/ecpg.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/ecpg$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/ecpg.wasm' +endif clean distclean: rm -f *.o ecpg$(X) rm -f typename.c rm -f preproc.y preproc.c preproc.h pgc.c c_kwlist_d.h ecpg_kwlist_d.h +ifeq ($(PORTNAME), emscripten) + rm -f ecpg.wasm +endif \ No newline at end of file diff --git a/src/makefiles/Makefile.emscripten b/src/makefiles/Makefile.emscripten index 3d37d043ad8c4..72e4c10a9fba4 100644 --- a/src/makefiles/Makefile.emscripten +++ b/src/makefiles/Makefile.emscripten @@ -14,3 +14,9 @@ AROPT = crs # Rule for building a shared library from a single .o file %.so: %.o $(CC) $(CFLAGS) $< $(LDFLAGS) $(LDFLAGS_SL) -shared -o $@ + +emscripten_include_dir := $(pkgincludedir)/emscripten +emscripten_base_dir := $(emscripten_include_dir)/base +emscripten_imports_dir := $(emscripten_base_dir)/imports +emscripten_extension_dir := $(emscripten_include_dir)/extension +emscripten_extension_imports_dir := $(emscripten_extension_dir)/imports \ No newline at end of file diff --git a/src/makefiles/pgxs.mk b/src/makefiles/pgxs.mk index 0de3737e789b4..bf4916166cba1 100644 --- a/src/makefiles/pgxs.mk +++ b/src/makefiles/pgxs.mk @@ -249,6 +249,12 @@ ifdef MODULES ifeq ($(with_llvm), yes) $(foreach mod, $(MODULES), $(call install_llvm_module,$(mod),$(mod).bc)) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + $(LLVM_NM) -u *.o | awk '{print $$2}' | sed '/^$$/d' | sort -u > undef.txt + $(LLVM_NM) *.o | awk '$$2 ~ /^[TDB]$$/ {print $$3}' | sed '/^$$/d' | sort -u > defs.txt + comm -23 undef.txt defs.txt > '$(emscripten_extension_imports_dir)/$(MODULES).imports' + rm -f undef.txt defs.txt +endif # PORTNAME=emscripten endif # MODULES ifdef DOCS ifdef docdir @@ -271,6 +277,12 @@ ifdef MODULE_big ifeq ($(with_llvm), yes) $(call install_llvm_module,$(MODULE_big),$(OBJS)) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + $(LLVM_NM) -u $(OBJS) | awk '{print $$2}' | sed '/^$$/d' | sort -u > undef.txt + $(LLVM_NM) $(OBJS) | awk '$$2 ~ /^[TDB]$$/ {print $$3}' | sed '/^$$/d' | sort -u > defs.txt + comm -23 undef.txt defs.txt > '$(emscripten_extension_imports_dir)/$(MODULE_big).imports' + rm -f undef.txt defs.txt +endif # PORTNAME=emscripten install: install-lib endif # MODULE_big @@ -302,6 +314,9 @@ ifdef MODULE_big installdirs: installdirs-lib endif # MODULE_big +ifeq ($(PORTNAME), emscripten) + $(MKDIR_P) '$(DESTDIR)$(emscripten_extension_imports_dir)' +endif uninstall: ifneq (,$(EXTENSION)) @@ -318,6 +333,9 @@ ifdef MODULES ifeq ($(with_llvm), yes) $(foreach mod, $(MODULES), $(call uninstall_llvm_module,$(mod))) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(emscripten_extension_imports_dir)/$(MODULES).imports' +endif endif # MODULES ifdef DOCS rm -f $(addprefix '$(DESTDIR)$(docdir)/$(docmoduledir)'/, $(DOCS)) @@ -340,6 +358,10 @@ ifeq ($(with_llvm), yes) $(call uninstall_llvm_module,$(MODULE_big)) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(emscripten_extension_imports_dir)/$(MODULE_big).imports' +endif + uninstall: uninstall-lib endif # MODULE_big diff --git a/src/pl/plpgsql/src/Makefile b/src/pl/plpgsql/src/Makefile index 63cb96fae3efc..7bdf2cd3f1e51 100644 --- a/src/pl/plpgsql/src/Makefile +++ b/src/pl/plpgsql/src/Makefile @@ -53,11 +53,20 @@ install: all install-lib install-data install-headers installdirs: installdirs-lib $(MKDIR_P) '$(DESTDIR)$(datadir)/extension' $(MKDIR_P) '$(DESTDIR)$(includedir_server)' +ifeq ($(PORTNAME), emscripten) + $(MKDIR_P) '$(DESTDIR)$(emscripten_extension_imports_dir)' +endif uninstall: uninstall-lib uninstall-data uninstall-headers install-data: installdirs $(INSTALL_DATA) $(addprefix $(srcdir)/, $(DATA)) '$(DESTDIR)$(datadir)/extension/' +ifeq ($(PORTNAME), emscripten) + $(LLVM_NM) -u $(OBJS) | awk '{print $$2}' | sed '/^$$/d' | sort -u > undef.txt + $(LLVM_NM) $(OBJS) | awk '$$2 ~ /^[TDB]$$/ {print $$3}' | sed '/^$$/d' | sort -u > defs.txt + comm -23 undef.txt defs.txt > '$(DESTDIR)$(emscripten_extension_imports_dir)/$(NAME).imports' + rm -f undef.txt defs.txt +endif # The plpgsql.h header file is needed by instrumentation plugins install-headers: installdirs @@ -65,6 +74,9 @@ install-headers: installdirs uninstall-data: rm -f $(addprefix '$(DESTDIR)$(datadir)/extension'/, $(notdir $(DATA))) +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(emscripten_extension_imports_dir)/$(NAME).imports' +endif uninstall-headers: rm -f '$(DESTDIR)$(includedir_server)/plpgsql.h' diff --git a/src/test/Makefile b/src/test/Makefile index dbd3192874d33..03bb5ccd34dfe 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -7,6 +7,12 @@ # src/test/Makefile # #------------------------------------------------------------------------- +ifeq ($(PORTNAME), emscripten) +# edited for pglite +all: $(echo src/test and src/test/isolation skipped) +clean check installcheck all-src-recurse: all +install: all +else subdir = src/test top_builddir = ../.. @@ -49,3 +55,5 @@ $(call recurse,installcheck, $(installable_dirs)) $(call recurse,install, $(installable_dirs)) $(recurse_always) + +endif \ No newline at end of file diff --git a/src/test/isolation/Makefile b/src/test/isolation/Makefile index ade2256ed3aa7..e0094bcb662ff 100644 --- a/src/test/isolation/Makefile +++ b/src/test/isolation/Makefile @@ -1,6 +1,12 @@ # # Makefile for isolation tests # +ifeq ($(PORTNAME), emscripten) +# edited for pglite +all: $(echo src/test and src/test/isolation skipped) +clean check installcheck all-src-recurse: all +install: all +else PGFILEDESC = "pg_isolation_regress/isolationtester - multi-client test driver" PGAPPICON = win32 @@ -72,3 +78,5 @@ installcheck-prepared-txns: all temp-install check-prepared-txns: all temp-install $(pg_isolation_regress_check) --schedule=$(srcdir)/isolation_schedule prepared-transactions prepared-transactions-cic + +endif \ No newline at end of file diff --git a/src/timezone/Makefile b/src/timezone/Makefile index c85e831247a5a..d871b67582378 100644 --- a/src/timezone/Makefile +++ b/src/timezone/Makefile @@ -48,8 +48,12 @@ endif # but GNU make versions <= 3.78.1 or perhaps later have a bug # that causes a segfault; GNU make 3.81 or later fixes this. ifeq (,$(ZIC)) +ifeq ($(PORTNAME), emscripten) +ZIC= /usr/sbin/zic +else ZIC= ./zic endif +endif zic: $(ZICOBJS) | submake-libpgport $(CC) $(CFLAGS) $(ZICOBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) @@ -77,3 +81,6 @@ endif clean distclean: rm -f zic$(X) $(ZICOBJS) abbrevs.txt +ifeq ($(PORTNAME), emscripten) + rm -f zic.wasm +endif \ No newline at end of file diff --git a/wasm-build/sdk_port.h b/wasm-build/include/sdk_port.h similarity index 100% rename from wasm-build/sdk_port.h rename to wasm-build/include/sdk_port.h diff --git a/wasm-build/sdk_port-wasi/sdk_port-wasi.c b/wasm-build/sdk_port-wasi/sdk_port-wasi.c index 5dbab909264d7..f01d519d91075 100644 --- a/wasm-build/sdk_port-wasi/sdk_port-wasi.c +++ b/wasm-build/sdk_port-wasi/sdk_port-wasi.c @@ -316,11 +316,8 @@ int shm_unlink(const char *name) { #include // FILE+fprintf extern FILE* IDB_PIPE_FP; -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; extern int IDB_STAGE; - static inline int ends_with(const char *str, const char *suffix) { @@ -404,106 +401,4 @@ volatile int fd_sock = 0; volatile int fd_out = 2; volatile int fd_queue = 0; - - - -#if 0 -#define AF_UNIX 1 // PF_LOCAL - - - - -volatile bool web_warned = false; - -void sock_flush() { - if (fd_queue) { - printf(" -- 441 sockflush : AIO YIELD, expecting %s filled on return --\n", PGS_OUT); - if (!fd_FILE) { - if (!web_warned) { - puts("# 444: WARNING: fd_FILE not set but queue not empty, assuming web"); - web_warned = true; - } - - } else { - printf("# 449: SENT=%ld/%d fd_out=%d fno=%d\n", ftell(fd_FILE), fd_queue, fd_out, fileno(fd_FILE)); - fclose(fd_FILE); - rename(PGS_ILOCK, PGS_IN); -//freopen(PGS_ILOCK, "w+", fd_FILE); - fd_FILE = fopen(PGS_ILOCK, "w"); - fd_out = fileno(fd_FILE); - printf("# 455: fd_out=%d fno=%d\n", fd_out, fileno(fd_FILE)); - } - fd_queue = 0; - sched_yield(); - return; - } - - printf(" -- 462 sockflush[%d] : NO YIELD --\n",stage); - - // limit inf loops - if (stage++ > 1024) { - puts("# 466 sock_flush : busy looping ?"); - abort(); - } -} - - -volatile int fd_current_pos = 0; -volatile int fd_filesize = 0; - - -ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len) { -// int busy = 0; - int rcv = -1; - sock_flush(); -/* - while (access(PGS_OUT, F_OK) != 0) { - if (!(++busy % 555111)) { - printf("# 471: FIXME: busy wait (%d) for input stream %s\n", busy, PGS_OUT); - } - if (busy>1665334) { - errno = EINTR; - return -1; - } - } -*/ - FILE *sock_in = fopen(PGS_OUT,"r"); - if (sock_in) { - if (!fd_filesize) { - fseek(sock_in, 0L, SEEK_END); - fd_filesize = ftell(sock_in); - } - fseek(sock_in, fd_current_pos, SEEK_SET); - - char *buf = buffer; - buf[0] = 0; - rcv = fread(buf, 1, length, sock_in); - - if (rcv