interpreters (#935)

* Add julia ext and command

* Added php support

* interpreters

* {root, interpreters}

* java support (from #846)

* fix test

* document

* pass interpreters to resolvers for file paths

* Update docs/config.md

Co-authored-by: Mike Bostock <[email protected]>

* Update docs/config.md

* missing line

* add line

Co-authored-by: Mike Bostock <[email protected]>

* fix tests

* revert order

* docs; order

* restore “in order”

* sync describe callback

* describe must be sync

* LoaderResolver

* extract mocks to prevent duplicate tests

* shorter

* minimize diff

* LoaderResolver.getWatchPath

* remove todo

---------

Co-authored-by: Vikas Negi <[email protected]>
Co-authored-by: Michal Štefaňák <[email protected]>
Co-authored-by: Mike Bostock <[email protected]>

Author: 4 people

docs/config.md

@@ -200,6 +200,48 @@ toc: false
 
 Whether to enable [search](./search) on the project; defaults to false.
 
+## interpreters
+
+The **interpreters** option specifies additional interpreted languages for data loaders, indicating the file extension and associated interpreter. (See [loader routing](./loaders#routing) for more.) The default list of interpreters is:
+
+```js run=false
+{
+  ".js": ["node", "--no-warnings=ExperimentalWarning"],
+  ".ts": ["tsx"],
+  ".py": ["python3"],
+  ".r": ["Rscript"],
+  ".R": ["Rscript"],
+  ".rs": ["rust-script"]
+  ".go": ["go", "run"],
+  ".java": ["java"],
+  ".jl": ["julia"],
+  ".php": ["php"],
+  ".sh": ["sh"],
+  ".exe": []
+}
+```
+
+Keys specify the file extension and values the associated command and arguments. For example, to add Perl (extension `.pl`) and AppleScript (`.scpt`) to the list above:
+
+```js run=false
+export default {
+  interpreters: {
+    ".pl": ["perl"],
+    ".scpt": ["osascript"]
+  }
+};
+```
+
+To disable an interpreter, set its value to null. For example, to disable Rust:
+
+```js run=false
+export default {
+  interpreters: {
+    ".rs": null
+  }
+};
+```
+
 ## markdownIt <a href="https://github.com/observablehq/framework/releases/tag/v1.1.0" target="_blank" class="observablehq-version-badge" data-version="^1.1.0" title="Added in v1.1.0"></a>
 
 A hook for registering additional [markdown-it](https://github.com/markdown-it/markdown-it) plugins. For example, to use [markdown-it-footnote](https://github.com/markdown-it/markdown-it-footnote):

docs/loaders.md

@@ -126,29 +126,49 @@ Like with any other file, these files from generated archives are live in previe
 
 ## Routing
 
-Data loaders live in the source root (typically `docs`) alongside your other source files. When a file is referenced from JavaScript via `FileAttachment`, if the file does not exist, Observable Framework will look for a file of the same name with a double extension to see if there is a corresponding data loader. The following second extensions are checked, in order, with the corresponding language and interpreter:
+Data loaders live in the source root (typically `docs`) alongside your other source files. When a file is referenced from JavaScript via `FileAttachment`, if the file does not exist, Observable Framework will look for a file of the same name with a double extension to see if there is a corresponding data loader. By default, the following second extensions are checked, in order, with the corresponding language and interpreter:
 
 - `.js` - JavaScript (`node`)
 - `.ts` - TypeScript (`tsx`)
 - `.py` - Python (`python3`)
 - `.R` - R (`Rscript`)
 - `.rs` - Rust (`rust-script`)
 - `.go` - Go (`go run`)
+- `.java` — Java (`java`; requires Java 11+ and [single-file programs](https://openjdk.org/jeps/330))
+- `.jl` - Julia (`julia`)
+- `.php` - PHP (`php`)
 - `.sh` - shell script (`sh`)
 - `.exe` - arbitrary executable
 
-For example, for the file `quakes.csv`, the following data loaders are considered:
+For example, for the file `quakes.csv`, the following data loaders are considered: `quakes.csv.js`, `quakes.csv.ts`, `quakes.csv.py`, _etc._ The first match is used.
 
-- `quakes.csv.js`
-- `quakes.csv.ts`
-- `quakes.csv.py`
-- `quakes.csv.R`
-- `quakes.csv.rs`
-- `quakes.csv.go`
-- `quakes.csv.sh`
-- `quakes.csv.exe`
+<div class="tip">The <b>interpreters</b> <a href="./config#interpreters">configuration option</a> can be used to extend the list of supported extensions.</div>
 
-If you use `.py`, `.R`, `.rs`, or `.go`, the corresponding interpreter (`python3`, `Rscript`, `rust-script`, or `go run`, respectively) must be installed and available on your `$PATH`. Any additional modules, packages, libraries, _etc._, must also be installed before you can use them.
+To use an interpreted data loader (anything other than `.exe`), the corresponding interpreter must be installed and available on your `$PATH`. Any additional modules, packages, libraries, _etc._, must also be installed. Some interpreters are not available on all platforms; for example `sh` is only available on Unix-like systems.
+
+<div class="tip">
+
+You can use a virtual environment in Python, such as [uv](https://github.com/astral-sh/uv), to install libraries locally to the project. This is useful when working in multiple projects, and when collaborating; you can also track dependencies in a `requirements.txt` file. To create a virtual environment with uv, run:
+
+```sh
+uv venv # Create a virtual environment at .venv.
+```
+
+To activate the virtual environment on macOS or Linux:
+
+```sh
+source .venv/bin/activate
+```
+
+Or on Windows:
+
+```sh
+.venv\Scripts\activate
+```
+
+You can then run the `observable preview` or `observable build` commands as usual; data loaders will run within the virtual environment. Run the `deactivate` command to exit the virtual environment.
+
+</div>
 
 Data loaders are run in the same working directory in which you run the `observable build` or `observable preview` command, which is typically the project root. In Node, you can access the current working directory by calling `process.cwd()`, and the data loader’s source location with [`import.meta.url`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/import.meta). To compute the path of a file relative to the data loader source (rather than relative to the current working directory), use [`import.meta.resolve`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/import.meta/resolve). For example, a data loader in `docs/summary.txt.js` could read the file `docs/table.txt` as:
 
@@ -159,18 +179,18 @@ import {fileURLToPath} from "node:url";
 const table = await readFile(fileURLToPath(import.meta.resolve("./table.txt")), "utf-8");
 ```
 
-Whereas `.js`, `.ts`, `.py`, `.R`, `.rs`, `.go`, and `.sh` data loaders are run via interpreters, `.exe` data loaders are run directly and must have the executable bit set. This is typically done via [`chmod`](https://en.wikipedia.org/wiki/Chmod). For example:
+Executable (`.exe`) data loaders are run directly and must have the executable bit set. This is typically done via [`chmod`](https://en.wikipedia.org/wiki/Chmod). For example:
 
 ```sh
 chmod +x docs/quakes.csv.exe
 ```
 
-While a `.exe` data loader may be any binary executable (_e.g.,_ compiled from C), it is often convenient to specify another interpreter using a [shebang](<https://en.wikipedia.org/wiki/Shebang_(Unix)>). For example, to write a data loader in Julia:
+While a `.exe` data loader may be any binary executable (_e.g.,_ compiled from C), it is often convenient to specify another interpreter using a [shebang](<https://en.wikipedia.org/wiki/Shebang_(Unix)>). For example, to write a data loader in Perl:
 
-```julia
-#!/usr/bin/env julia
+```perl
+#!/usr/bin/env perl
 
-println("hello world")
+print("Hello World\n");
 ```
 
 If multiple requests are made concurrently for the same data loader, the data loader will only run once; each concurrent request will receive the same response.

examples/eia/README.md

@@ -29,14 +29,14 @@ The base map is created in the `us-states.json.js` data loader, which uses [Topo
 
 ### Static files
 
-Several static files used in the dashboard were downloaded from the EIA Hourly Electric Grid Monitor [About page](https://www.eia.gov/electricity/gridmonitor/about) (EIA-930 data reference tables), and not created or processed in data loaders. These files contain reference information that expect to remain unchanged, including: 
+Several static files used in the dashboard were downloaded from the EIA Hourly Electric Grid Monitor [About page](https://www.eia.gov/electricity/gridmonitor/about) (EIA-930 data reference tables), and not created or processed in data loaders. These files contain reference information that expect to remain unchanged, including:
 
-- `eia-bia-reference.csv`: Reference information about balancing authority name, time zone, region, country, etc.
+- `eia-bia-reference.csv`: Reference information about balancing authority name, time zone, region, country, _etc._
 - `eia-connections-reference.csv`: Reference information about connections between balancing authorities
 
 ## Charts
 
-The charts and map are drawn with [Observable Plot](https://observablehq.com/plot/), and saved as components in `components/charts.js` and `components/map.js` to simplify our layout code in `index.md`. 
+The charts and map are drawn with [Observable Plot](https://observablehq.com/plot/), and saved as components in `components/charts.js` and `components/map.js` to simplify our layout code in `index.md`.
 
 ## Thanks
 

examples/plot/README.md

@@ -28,7 +28,7 @@ The [`npm-downloads.csv.ts`](./docs/data/npm-downloads.csv.ts) loader retrieves
 
 ## Big numbers
 
-Key performance indicators are displayed as “big numbers” with, in some cases, a trend indicating growth over one week. Their layout is using the convenience CSS classes _big_, _red_ etc.
+Key performance indicators are displayed as “big numbers” with, in some cases, a trend indicating growth over one week. Their layout is using the convenience CSS classes _big_, _red_ _etc._
 
 ## Charts
 

src/build.ts

@@ -3,7 +3,6 @@ import {existsSync} from "node:fs";
 import {access, constants, copyFile, readFile, writeFile} from "node:fs/promises";
 import {basename, dirname, extname, join} from "node:path/posix";
 import type {Config} from "./config.js";
-import {Loader} from "./dataloader.js";
 import {CliError, isEnoent} from "./error.js";
 import {getClientPath, prepareOutput, visitMarkdownFiles} from "./files.js";
 import {getModuleHash} from "./javascript/module.js";
@@ -16,7 +15,7 @@ import {isPathImport, relativePath, resolvePath} from "./path.js";
 import {renderPage} from "./render.js";
 import type {Resolvers} from "./resolvers.js";
 import {getModuleResolver, getResolvers} from "./resolvers.js";
-import {resolveFilePath, resolveImportPath, resolveStylesheetPath} from "./resolvers.js";
+import {resolveImportPath, resolveStylesheetPath} from "./resolvers.js";
 import {bundleStyles, rollupClient} from "./rollup.js";
 import {searchIndex} from "./search.js";
 import {Telemetry} from "./telemetry.js";
@@ -48,7 +47,7 @@ export async function build(
   {config, addPublic = true}: BuildOptions,
   effects: BuildEffects = new FileBuildEffects(config.output)
 ): Promise<void> {
-  const {root} = config;
+  const {root, loaders} = config;
   Telemetry.record({event: "build", step: "start"});
 
   // Make sure all files are readable before starting to write output files.
@@ -78,7 +77,7 @@ export async function build(
       effects.logger.log(faint("(skipped)"));
       continue;
     }
-    const resolvers = await getResolvers(page, {root, path: sourceFile});
+    const resolvers = await getResolvers(page, {root, path: sourceFile, loaders});
     const elapsed = Math.floor(performance.now() - start);
     for (const f of resolvers.assets) files.add(resolvePath(sourceFile, f));
     for (const f of resolvers.files) files.add(resolvePath(sourceFile, f));
@@ -151,7 +150,7 @@ export async function build(
   for (const file of files) {
     let sourcePath = join(root, file);
     if (!existsSync(sourcePath)) {
-      const loader = Loader.find(root, join("/", file), {useStale: true});
+      const loader = loaders.find(join("/", file), {useStale: true});
       if (!loader) {
         effects.logger.error("missing referenced file", sourcePath);
         continue;
@@ -168,7 +167,7 @@ export async function build(
     const hash = createHash("sha256").update(contents).digest("hex").slice(0, 8);
     const ext = extname(file);
     const alias = `/${join("_file", dirname(file), `${basename(file, ext)}.${hash}${ext}`)}`;
-    aliases.set(resolveFilePath(root, file), alias);
+    aliases.set(loaders.resolveFilePath(file), alias);
     await effects.writeFile(alias, contents);
   }
 

src/config.ts

@@ -5,6 +5,7 @@ import {basename, dirname, join} from "node:path/posix";
 import {cwd} from "node:process";
 import {pathToFileURL} from "node:url";
 import type MarkdownIt from "markdown-it";
+import {LoaderResolver} from "./dataloader.js";
 import {visitMarkdownFiles} from "./files.js";
 import {formatIsoDate, formatLocaleDate} from "./format.js";
 import {createMarkdownIt, parseMarkdown} from "./markdown.js";
@@ -54,6 +55,7 @@ export interface Config {
   deploy: null | {workspace: string; project: string};
   search: boolean; // default to false
   md: MarkdownIt;
+  loaders: LoaderResolver;
 }
 
 /**
@@ -115,7 +117,8 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
     header = "",
     footer = `Built with <a href="https://observablehq.com/" target="_blank">Observable</a> on <a title="${formatIsoDate(
       currentDate
-    )}">${formatLocaleDate(currentDate)}</a>.`
+    )}">${formatLocaleDate(currentDate)}</a>.`,
+    interpreters
   } = spec;
   root = String(root);
   output = String(output);
@@ -136,6 +139,7 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
   toc = normalizeToc(toc);
   deploy = deploy ? {workspace: String(deploy.workspace).replace(/^@+/, ""), project: String(deploy.project)} : null;
   search = Boolean(search);
+  interpreters = normalizeInterpreters(interpreters);
   return {
     root,
     output,
@@ -152,7 +156,8 @@ export async function normalizeConfig(spec: any = {}, defaultRoot = "docs"): Pro
     style,
     deploy,
     search,
-    md
+    md,
+    loaders: new LoaderResolver({root, interpreters})
   };
 }
 
@@ -196,6 +201,14 @@ function normalizePage(spec: any): Page {
   return {name, path};
 }
 
+function normalizeInterpreters(spec: any): Record<string, string[] | null> {
+  return Object.fromEntries(
+    Object.entries<any>(spec ?? {}).map(([key, value]): [string, string[] | null] => {
+      return [String(key), value == null ? null : Array.from(value, String)];
+    })
+  );
+}
+
 function normalizeToc(spec: any): TableOfContents {
   if (typeof spec === "boolean") spec = {show: spec};
   let {label = "Contents", show = true} = spec;