feat: json, json-all generators

.c8rc.json

@@ -5,6 +5,7 @@
     "**/fixtures",
     "src/generators/legacy-html/assets",
     "src/generators/web/ui",
-    "**/*.d.ts"
+    "**/*.d.ts",
+    "tools/"
   ]
 }

package.json

@@ -34,6 +34,8 @@
     "eslint-plugin-import-x": "^4.16.1",
     "eslint-plugin-jsdoc": "^60.5.0",
     "husky": "^9.1.7",
+    "json-schema-to-typescript": "^15.0.4",
+    "jsonschema": "^1.5.0",
     "lint-staged": "^16.2.3",
     "prettier": "3.6.2"
   },
@@ -58,6 +60,7 @@
     "globals": "^16.4.0",
     "hast-util-to-string": "^3.0.1",
     "hastscript": "^9.0.1",
+    "jsonc-parser": "^3.3.1",
     "lightningcss": "^1.30.1",
     "mdast-util-slice-markdown": "^2.0.1",
     "preact": "^10.27.2",

src/generators/index.mjs

@@ -3,6 +3,8 @@
 import addonVerify from './addon-verify/index.mjs';
 import apiLinks from './api-links/index.mjs';
 import astJs from './ast-js/index.mjs';
+import json from './json/index.mjs';
+import jsonAll from './json-all/index.mjs';
 import jsonSimple from './json-simple/index.mjs';
 import jsxAst from './jsx-ast/index.mjs';
 import legacyHtml from './legacy-html/index.mjs';
@@ -27,6 +29,10 @@ export const publicGenerators = {
   'orama-db': oramaDb,
   'llms-txt': llmsTxt,
   web,
+  'jsx-ast': jsxAst,
+  metadata,
+  json,
+  'json-all': jsonAll,
 };
 
 // These ones are special since they don't produce standard output,

src/generators/json-all/__tests__/index.test.mjs

@@ -0,0 +1,10 @@
+import { describe } from 'node:test';
+
+// TODO add test w/
+
+describe.todo('generator output complies with json schema', () => {
+  // TODO:
+  //  - create fixtures that cover various different types of docs we see
+  //  - run them through the generator
+  //  - validate the response with https://www.npmjs.com/package/jsonschema
+});

src/generators/json-all/__tests__/version.test.mjs

@@ -0,0 +1,18 @@
+'use strict';
+
+import assert from 'node:assert';
+import test from 'node:test';
+
+import json from '../../json/index.mjs';
+import jsonAll from '../index.mjs';
+import { generateJsonSchema } from '../util/generateJsonSchema.mjs';
+
+test('json-all generator matches json generator version match', () => {
+  assert.strictEqual(jsonAll.version, json.version);
+});
+
+test('schema version matches generator version', () => {
+  const schema = generateJsonSchema();
+
+  assert.strictEqual(schema.$id, `nodejs-api-doc-all@v${jsonAll.version}`);
+});

src/generators/json-all/index.mjs

@@ -0,0 +1,79 @@
+// @ts-check
+'use strict';
+
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { DOC_NODE_VERSION } from '../../constants.mjs';
+import { generateJsonSchema } from './util/generateJsonSchema.mjs';
+
+// TODO add test w/ https://www.npmjs.com/package/jsonschema
+
+/**
+ * TODO docs
+ *
+ * @typedef {Array<ApiDocMetadataEntry>} Input
+ *
+ * @type {GeneratorMetadata<Input, object>}
+ */
+export default {
+  name: 'json-all',
+
+  // This should be kept in sync with the JSON schema version for this
+  // generator AND the `json` generator
+  version: '2.0.0',
+
+  description: 'TODO',
+
+  dependsOn: 'json',
+
+  /**
+   * Generates a JSON file.
+   *
+   * @param {Input} input
+   * @param {Partial<GeneratorOptions>} param1
+   * @returns {Promise<object>}
+   */
+  async generate(input, { output }) {
+    const generatedValue = {
+      $schema: `https://nodejs.org/docs/${DOC_NODE_VERSION}/api/node-doc-all-schema.jsonc`,
+      modules: [],
+      text: [],
+    };
+
+    const propertiesToIgnore = ['$schema', 'source'];
+
+    input.forEach(section => {
+      const copiedSection = {};
+
+      Object.keys(section).forEach(key => {
+        if (!propertiesToIgnore.includes(key)) {
+          copiedSection[key] = section[key];
+        }
+      });
+
+      switch (section.type) {
+        case 'module':
+          generatedValue.modules.push(copiedSection);
+          break;
+        case 'text':
+          generatedValue.text.push(copiedSection);
+          break;
+        default:
+          throw new TypeError(`unsupported root section type ${section.type}`);
+      }
+    });
+
+    if (output) {
+      const schema = generateJsonSchema();
+
+      // Write the parsed JSON schema to the output directory
+      await writeFile(
+        join(output, 'node-doc-schema.json'),
+        JSON.stringify(schema)
+      );
+    }
+
+    return generatedValue;
+  },
+};

src/generators/json-all/util/generateJsonSchema.mjs

@@ -0,0 +1,27 @@
+'use strict';
+
+import { DOC_NODE_VERSION } from '../../../constants.mjs';
+import jsonAll from '../index.mjs';
+
+const JSON_SCHEMA_URL = `https://nodejs.org/docs/${DOC_NODE_VERSION}/api/node-doc-schema.json`;
+
+/**
+ *
+ */
+export const generateJsonSchema = () => ({
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  $id: `nodejs-api-doc-all@v${jsonAll.version}`,
+  title: 'Node.js API Documentation Schema (All)',
+  readOnly: true,
+
+  properties: {
+    modules: {
+      type: 'array',
+      items: { $ref: `${JSON_SCHEMA_URL}/#/definitions/Module` },
+    },
+    text: {
+      type: 'array',
+      items: { $ref: `${JSON_SCHEMA_URL}/#/definitions/Text` },
+    },
+  },
+});

src/generators/json/__tests__/fixtures/module.md

@@ -0,0 +1,73 @@
+# Module
+
+<!-- introduced_in=v20.0.0 -->
+<!-- source_link=lib/module.js -->
+
+> Stability: 2 - Stable
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+Nulla sagittis pulvinar lacus et venenatis. Morbi eget varius sem, a rutrum velit. Vestibulum placerat vehicula felis. Proin et porttitor augue. Sed feugiat urna sit amet euismod posuere. Phasellus quis pretium velit. Integer porta ut velit ac consectetur. Pellentesque venenatis leo et volutpat dapibus. Nam consectetur quam venenatis feugiat tempus. Curabitur semper fringilla felis.
+
+Donec sed est at nulla lacinia suscipit id id neque. Cras quis risus a risus mattis consectetur ut ut ex. Cras non metus a orci facilisis laoreet. Sed eu nisi sit amet mauris iaculis placerat. Nullam interdum, lectus sed convallis tristique, nibh nibh ullamcorper leo, non vehicula metus elit a nisl. Donec eros felis, ornare eget tempor nec, egestas mattis elit. Integer semper purus at risus pretium, in tristique ipsum tristique.
+
+```mjs
+import { something } from 'node:something';
+```
+
+## text section
+
+<!-- YAML
+changes:
+  - version: v3.0.0
+    pr-url: https://github.com/nodejs/node/pull/1234
+    description: asdf
+-->
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+Nulla sagittis pulvinar lacus et venenatis. Morbi eget varius sem, a rutrum velit. Vestibulum placerat vehicula felis. Proin et porttitor augue. Sed feugiat urna sit amet euismod posuere. Phasellus quis pretium velit. Integer porta ut velit ac consectetur. Pellentesque venenatis leo et volutpat dapibus. Nam consectetur quam venenatis feugiat tempus. Curabitur semper fringilla felis.
+
+Donec sed est at nulla lacinia suscipit id id neque. Cras quis risus a risus mattis consectetur ut ut ex. Cras non metus a orci facilisis laoreet. Sed eu nisi sit amet mauris iaculis placerat. Nullam interdum, lectus sed convallis tristique, nibh nibh ullamcorper leo, non vehicula metus elit a nisl. Donec eros felis, ornare eget tempor nec, egestas mattis elit. Integer semper purus at risus pretium, in tristique ipsum tristique.
+
+## Class: `Something`
+
+<!-- YAML
+added:
+  - v15.7.0
+  - v14.18.0
+changes:
+  - version:
+    - v18.0.0
+    - v16.17.0
+    pr-url: https://github.com/nodejs/node/pull/1234
+    description: No longer experimental.
+-->
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+### `new something.Something([sources[, options]])`
+
+- `sources` {string\[]|object} something something bla bla bla
+- `options` {object} something something bla bla bla
+  - `optionA` {boolean} asdf
+  - `bla` qwerty
+
+### `something.doThing()`
+
+- Returns: {Promise}
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+### Static method: `something.doStaticThing()`
+
+- Returns: {Promise}
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+### Event: `somethingHappened`
+
+- `thing` {string} thing that happened
+- `how` {string|object} bla
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.

src/generators/json/__tests__/fixtures/text-doc.md

@@ -0,0 +1,43 @@
+# Hello world
+
+<!-- type=misc -->
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+## abc123
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+Nulla sagittis pulvinar lacus et venenatis. Morbi eget varius sem, a rutrum velit. Vestibulum placerat vehicula felis. Proin et porttitor augue. Sed feugiat urna sit amet euismod posuere. Phasellus quis pretium velit. Integer porta ut velit ac consectetur. Pellentesque venenatis leo et volutpat dapibus. Nam consectetur quam venenatis feugiat tempus. Curabitur semper fringilla felis.
+
+Donec sed est at nulla lacinia suscipit id id neque. Cras quis risus a risus mattis consectetur ut ut ex. Cras non metus a orci facilisis laoreet. Sed eu nisi sit amet mauris iaculis placerat. Nullam interdum, lectus sed convallis tristique, nibh nibh ullamcorper leo, non vehicula metus elit a nisl. Donec eros felis, ornare eget tempor nec, egestas mattis elit. Integer semper purus at risus pretium, in tristique ipsum tristique.
+
+Nulla ultrices venenatis ex, vitae ullamcorper dui pulvinar non. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Vivamus fermentum, metus in blandit hendrerit, lorem urna interdum lacus, nec pretium turpis justo eget est. Cras tincidunt urna vel massa ultrices ultrices. Vivamus ullamcorper leo at nulla semper, ac vehicula nisl fringilla. Curabitur viverra vulputate tristique. Donec lorem erat, ornare sit amet faucibus id, mattis vel diam. Ut posuere sem laoreet augue tristique, eget volutpat leo malesuada. Nullam non velit mattis, pulvinar sem eu, vehicula purus. Proin placerat accumsan porttitor. Duis pharetra facilisis aliquet. Suspendisse blandit a nulla eget gravida. Etiam molestie porttitor sodales. Donec lobortis pretium sollicitudin. [asd](https://github.com/nodejs/node)
+
+### asd
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+Nulla sagittis pulvinar lacus et venenatis. Morbi eget varius sem, a rutrum velit. Vestibulum placerat vehicula felis. Proin et porttitor augue. Sed feugiat urna sit amet euismod posuere. Phasellus quis pretium velit. Integer porta ut velit ac consectetur. Pellentesque venenatis leo et volutpat dapibus. Nam consectetur quam venenatis feugiat tempus. Curabitur semper fringilla felis.
+
+- lorem
+- ipsum
+
+```c++
+int main() {
+    std::cout << "blahaj\n";
+    return 0;
+}
+```
+
+## lorem
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla aliquet est sit amet nisi iaculis ornare. Donec eu orci condimentum, semper quam ut, luctus nisl. Aliquam vel erat efficitur, egestas justo a, accumsan mauris. Quisque scelerisque enim rhoncus ornare iaculis. Sed commodo, sem ac accumsan posuere, sapien magna eleifend massa, sit amet rhoncus sem diam quis nulla. Donec feugiat elit a tellus hendrerit congue quis bibendum urna. Pellentesque mollis tincidunt tortor, in lobortis ex fringilla a.
+
+Nulla sagittis pulvinar lacus et venenatis. Morbi eget varius sem, a rutrum velit. Vestibulum placerat vehicula felis. Proin et porttitor augue. Sed feugiat urna sit amet euismod posuere. Phasellus quis pretium velit. Integer porta ut velit ac consectetur. Pellentesque venenatis leo et volutpat dapibus. Nam consectetur quam venenatis feugiat tempus. Curabitur semper fringilla felis.
+
+Donec sed est at nulla lacinia suscipit id id neque. Cras quis risus a risus mattis consectetur ut ut ex. Cras non metus a orci facilisis laoreet. Sed eu nisi sit amet mauris iaculis placerat. Nullam interdum, lectus sed convallis tristique, nibh nibh ullamcorper leo, non vehicula metus elit a nisl. Donec eros felis, ornare eget tempor nec, egestas mattis elit. Integer semper purus at risus pretium, in tristique ipsum tristique.
+
+Fusce pretium tincidunt dolor sit amet dignissim. Aliquam egestas purus sed lorem feugiat faucibus. Aenean eget augue non velit maximus tempor cursus eget augue. Suspendisse id laoreet felis. Integer consectetur scelerisque turpis, eu dapibus diam. Etiam ut tempus massa, vel euismod ipsum. Suspendisse quis nisl sapien. Etiam sed euismod dui. Nulla fermentum fringilla dignissim. In mauris ex, iaculis at arcu et, gravida sodales erat.
+
+Nulla ultrices venenatis ex, vitae ullamcorper dui pulvinar non. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Vivamus fermentum, metus in blandit hendrerit, lorem urna interdum lacus, nec pretium turpis justo eget est. Cras tincidunt urna vel massa ultrices ultrices. Vivamus ullamcorper leo at nulla semper, ac vehicula nisl fringilla. Curabitur viverra vulputate tristique. Donec lorem erat, ornare sit amet faucibus id, mattis vel diam. Ut posuere sem laoreet augue tristique, eget volutpat leo malesuada. Nullam non velit mattis, pulvinar sem eu, vehicula purus. Proin placerat accumsan porttitor. Duis pharetra facilisis aliquet. Suspendisse blandit a nulla eget gravida. Etiam molestie porttitor sodales. Donec lobortis pretium sollicitudin.

src/generators/json/__tests__/index.test.mjs

@@ -0,0 +1,54 @@
+// @ts-check
+import assert from 'node:assert';
+import { join } from 'node:path';
+import { describe, test, before } from 'node:test';
+
+import { Validator } from 'jsonschema';
+import { SemVer } from 'semver';
+
+import createGenerator from '../../../generators.mjs';
+import createMarkdownLoader from '../../../loaders/markdown.mjs';
+import createMarkdownParser from '../../../parsers/markdown.mjs';
+import { parseSchema } from '../utils/parseSchema.mjs';
+
+const FIXTURES_DIR = join(import.meta.dirname, 'fixtures');
+
+const loader = createMarkdownLoader();
+const parser = createMarkdownParser();
+
+describe('generator output complies with json schema', () => {
+  const validator = new Validator();
+
+  /**
+   * @type {object}
+   */
+  let schema;
+
+  before(async () => {
+    schema = await parseSchema();
+  });
+
+  for (const fixture of ['text-doc', 'module']) {
+    const input = join(FIXTURES_DIR, `${fixture}.md`);
+
+    test(`${fixture}.md`, async () => {
+      const files = await loader.loadFiles([input]);
+      const docs = await parser.parseApiDocs(files);
+
+      const { runGenerators } = createGenerator(docs);
+
+      const result = await runGenerators({
+        generators: ['json'],
+        input,
+        output: undefined,
+        version: new SemVer('v1.2.3'),
+        releases: [],
+        gitRef: 'a'.repeat(40),
+        threads: 1,
+        typeMap: {},
+      });
+
+      assert.ok(validator.validate(result[0], schema).valid);
+    });
+  }
+});

src/generators/json/__tests__/version.test.mjs

@@ -0,0 +1,20 @@
+'use strict';
+
+import assert from 'node:assert';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import test from 'node:test';
+
+import { parse as jsoncParse } from 'jsonc-parser';
+
+import json from '../index.mjs';
+
+test('schema version matches generator version', async () => {
+  const schemaString = await readFile(
+    join(import.meta.dirname, '..', 'schema.jsonc'),
+    'utf8'
+  );
+  const schema = await jsoncParse(schemaString);
+
+  assert.strictEqual(schema.$id, `nodejs-api-doc@v${json.version}`);
+});

src/generators/json/constants.mjs

@@ -0,0 +1,21 @@
+'use strict';
+
+// Grabs the default value if present
+export const DEFAULT_EXPRESSION = /^(D|d)efault(s|):$/;
+
+// Grabs the type and description of one of the formats for event types
+export const EVENT_TYPE_DESCRIPTION_EXTRACTOR = /{(.*)}(.*)/;
+
+// Grabs type and optional description for a method's parameter signature
+export const METHOD_TYPE_EXTRACTOR = /^{(.*)}( .*)?$/;
+
+// Grabs return type and optional description for a method signature
+// Accepts the following:
+// Returns: {string}
+// Returns {string}
+// Returns: {string} bla bla bla
+export const METHOD_RETURN_TYPE_EXTRACTOR = /^Returns(:?) {(.*)}( .*)?$/;
+
+// Grabs the parameters from a method's signature
+//  ex/ 'new buffer.Blob([sources[, options]])'.match(PARAM_EXPRESSION) === ['([sources[, options]])', '[sources[, options]]']
+export const METHOD_PARAM_EXPRESSION = /\((.+)\);?$/;