Modular vfs support

build-wasm-dbg.sh

@@ -4,8 +4,8 @@
 
 mkdir -p packages/crsqlite-wasm/dist
 cd deps/emsdk
-./emsdk install 3.1.45
-./emsdk activate 3.1.45
+./emsdk install 3.1.61
+./emsdk activate 3.1.61
 source ./emsdk_env.sh
 cd ../wa-sqlite
 make debug

build-wasm.sh

@@ -4,8 +4,8 @@
 
 mkdir -p packages/crsqlite-wasm/dist
 cd deps/emsdk
-./emsdk install 3.1.45
-./emsdk activate 3.1.45
+./emsdk install 3.1.61
+./emsdk activate 3.1.61
 source ./emsdk_env.sh
 cd ../wa-sqlite
 make

packages/crsqlite-wasm/src/index.ts

@@ -6,23 +6,25 @@ import { serialize, topLevelMutex } from "./serialize.js";
 import { DB } from "./DB.js";
 export { DB } from "./DB.js";
 
-let api: SQLite3 | null = null;
+const apiCache = new Map<string, SQLite3>();
 type SQLiteAPI = ReturnType<typeof SQLite.Factory>;
 
 export class SQLite3 {
   constructor(private base: SQLiteAPI) {}
 
-  open(filename?: string, mode: string = "c") {
+  open(filename: string = ":memory:", mode: string = "c", vfs_name?: string) {
     return serialize(
       null,
       undefined,
       () => {
         return this.base.open_v2(
-          filename || ":memory:",
+          filename,
           SQLite.SQLITE_OPEN_CREATE |
             SQLite.SQLITE_OPEN_READWRITE |
             SQLite.SQLITE_OPEN_URI,
-          filename != null ? "idb-batch-atomic" : undefined
+          // My understanding is that filename = ":memory:" case doesn't care about the vfs_name, whereas not specifying
+          // the vfs_name will use the default vfs (which is the desired VFS if set up using 'iniwWasm')
+          vfs_name
         );
       },
       topLevelMutex
@@ -47,14 +49,33 @@ export class SQLite3 {
   }
 }
 
-export default async function initWasm(
+/**
+ * @param module The WASM module (glue code) instance (not to be confused with SQLiteAPI - wrapping the module)
+ */
+export type VFSFactory = (module: any) => Promise<SQLiteVFS>;
+export type ModuleFactory = (moduleArg?: Record<string, any>) => any;
+
+export type WasmInitializerConfig = {
+  ModuleFactory: ModuleFactory;
+  vfsFactory: VFSFactory;
+  cacheKey?: string;
+};
+
+export type WasmInitializer = (
   locateWasm?: (file: string) => string
+) => Promise<SQLite3>;
+
+async function initWasmInternal(
+  ModuleFactory: ModuleFactory,
+  vfsFactory: VFSFactory,
+  locateWasm?: (file: string) => string,
+  cacheKey?: string
 ): Promise<SQLite3> {
-  if (api != null) {
-    return api;
+  if (cacheKey && apiCache.has(cacheKey)) {
+    return apiCache.get(cacheKey)!;
   }
 
-  const wasmModule = await SQLiteAsyncESMFactory({
+  const wasmModule = await ModuleFactory({
     locateFile(file: string) {
       if (locateWasm) {
         return locateWasm(file);
@@ -63,10 +84,42 @@ export default async function initWasm(
     },
   });
   const sqlite3 = SQLite.Factory(wasmModule);
-  sqlite3.vfs_register(
-    new IDBBatchAtomicVFS("idb-batch-atomic", { durability: "relaxed" })
-  );
+  const vfs = await vfsFactory(wasmModule);
+  sqlite3.vfs_register(vfs, true);
+
+  const api = new SQLite3(sqlite3);
+  if (cacheKey) {
+    apiCache.set(cacheKey, api);
+  }
 
-  api = new SQLite3(sqlite3);
   return api;
 }
+
+/**
+ * Default initializer (for backwards compatibility), uses:
+ * - asyncify WASM build
+ * - IDB Batch Atomic VFS
+ */
+export default async function initWasm(
+  locateWasm?: (file: string) => string
+): Promise<SQLite3> {
+  const ModuleFactory = SQLiteAsyncESMFactory;
+  const vfsFactory: VFSFactory = (module) =>
+    IDBBatchAtomicVFS.create("idb-batch-atomic", module);
+  return initWasmInternal(ModuleFactory, vfsFactory, locateWasm, "default");
+}
+
+/**
+ * Creates a custom wasm initializer (same signature as default `initWasm`) allowing
+ * consumer to provide:
+ * - custom WASM glue factory
+ * - custom VFS implementation.
+ */
+export function createWasmInitializer({
+  ModuleFactory,
+  vfsFactory,
+  cacheKey,
+}: WasmInitializerConfig): WasmInitializer {
+  return (locateWasm?: (file: string) => string) =>
+    initWasmInternal(ModuleFactory, vfsFactory, locateWasm, cacheKey);
+}

packages/direct-connect-browser/src/common/DB.ts

@@ -1,4 +1,4 @@
-import initWasm from "@vlcn.io/crsqlite-wasm";
+import initWasm, { WasmInitializer } from "@vlcn.io/crsqlite-wasm";
 import { DBAsync, StmtAsync } from "@vlcn.io/xplat-api";
 import { TXAsync } from "@vlcn.io/xplat-api";
 import { DBID } from "@vlcn.io/xplat-api";
@@ -100,8 +100,12 @@ export class DB {
   }
 }
 
-export default async function getDB(wasmUri: string | undefined, dbid: DBID) {
-  const sqlite = await initWasm(wasmUri ? () => wasmUri : undefined);
+export default async function getDB(
+  wasmUri: string | undefined,
+  dbid: DBID,
+  initializer: WasmInitializer = initWasm
+) {
+  const sqlite = await initializer(wasmUri ? () => wasmUri : undefined);
   const db = await sqlite.open(dbid);
 
   const [pullChangesetStmt, applyChangesetStmt, updatePeerTrackerStmt] =

packages/react/src/db/DBFactory.ts

@@ -1,4 +1,4 @@
-import initWasm, { SQLite3 } from "@vlcn.io/crsqlite-wasm";
+import initWasm, { SQLite3, WasmInitializer } from "@vlcn.io/crsqlite-wasm";
 import tblrx from "@vlcn.io/rx-tbl";
 import { CtxAsync } from "../context.js";
 import { Mutex } from "async-mutex";
@@ -14,12 +14,13 @@ const dbMap = new Map<DBID, [string, CtxAsync]>();
 const hooks = new Map<DBID, () => CtxAsync | null>();
 
 let initPromise: Promise<SQLite3> | null = null;
-function init(wasmUri?: string) {
+// TODO: the initializer logic isn't wired in (see call to init() below)
+function init(wasmUri?: string, initializer: WasmInitializer = initWasm) {
   if (initPromise) {
     return initPromise;
   }
 
-  initPromise = initWasm(wasmUri ? () => wasmUri : undefined);
+  initPromise = initializer(wasmUri ? () => wasmUri : undefined);
   return initPromise;
 }
 

packages/ws-browserdb/src/index.ts

@@ -1,5 +1,8 @@
 import { DB } from "@vlcn.io/ws-client";
-import initWasm, { DB as WasmDB } from "@vlcn.io/crsqlite-wasm";
+import initWasm, {
+  DB as WasmDB,
+  WasmInitializer,
+} from "@vlcn.io/crsqlite-wasm";
 import { Change } from "@vlcn.io/ws-common";
 import { StmtAsync, firstPick } from "@vlcn.io/xplat-api";
 import tblrx from "@vlcn.io/rx-tbl";
@@ -138,10 +141,11 @@ class WrappedDB implements DB {
  * @returns
  */
 export function createDbProvider(
-  wasmUri?: string
+  wasmUri?: string,
+  initializer = initWasm
 ): (dbname: string) => PromiseLike<DB> {
   return async (dbname: string): Promise<DB> => {
-    const sqlite = await initWasm(wasmUri ? () => wasmUri : undefined);
+    const sqlite = await initializer(wasmUri ? () => wasmUri : undefined);
     const db = await sqlite.open(dbname);
 
     const [pullChangesetStmt, applyChangesetStmt, updatePeerTrackerStmt] =