Merge pull request #163 from code-hike/python-comments

Python comments support

Author: pomber

examples/bundle-test/package.json

@@ -3,6 +3,7 @@
   "private": true,
   "version": "0.0.0",
   "scripts": {
+    "dev": "next",
     "build": "next build"
   },
   "dependencies": {

examples/mdx-bundler/package.json

@@ -3,6 +3,7 @@
   "private": true,
   "version": "0.0.0",
   "scripts": {
+    "dev": "next",
     "build": "next build"
   },
   "dependencies": {

examples/nextjs/next.config.js

@@ -10,4 +10,5 @@ const withMDX = require("@next/mdx")({
 
 module.exports = withMDX({
   pageExtensions: ["ts", "tsx", "js", "jsx", "md", "mdx"],
+  eslint: { ignoreDuringBuilds: true },
 })

examples/vite/vite.config.js

@@ -6,14 +6,12 @@ import theme from "shiki/themes/github-dark.json"
 export default defineConfig(async () => {
   const mdx = await import("@mdx-js/rollup")
   return {
+    optimizeDeps: {
+      include: ["react/jsx-runtime"],
+    },
     plugins: [
       react(),
       mdx.default({ remarkPlugins: [[remarkCodeHike, { theme }]] }),
     ],
-    resolve: {
-      alias: {
-        "react/jsx-runtime": "react/jsx-runtime.js",
-      },
-    },
   }
 })

package.json

@@ -12,7 +12,6 @@
     "build:lib": "lerna run --stream --scope @*/mdx build",
     "test": "lerna run --stream test",
     "dev": "lerna run --stream --scope @*/mdx dev",
-    "storybook": "lerna run --scope storybook start --stream",
     "release": "auto shipit"
   },
   "devDependencies": {

packages/mdx/dev/content/comment-annotations.mdx

@@ -68,3 +68,27 @@ function adipiscing(...elit) {
   return elit.map(ipsum => ipsum.sit)
 }
 ```
+
+Bash like comments
+
+```python
+def lorem(ipsum, dolor = 1):
+  sit = ipsum == None and 0
+  # this isn't an annotation
+  # 你好
+  dolor = sit - amet(dolor)
+  # focus
+  return sit and consectetur(ipsum) or []
+```
+
+```bash
+function lorem(ipsum, dolor = 1) {
+  # focus(1:3)
+  # box[3:7]
+  local sit=0
+  # label something something
+  dolor=$((sit - amet(dolor)))
+  # link[19:29] https://github.com/code-hike/codehike
+  return $sit and consectetur(ipsum) or []
+}
+```

packages/mdx/src/mdx-plugin/__snapshots__/comment-data.test.ts.snap

@@ -0,0 +1,73 @@
+// Vitest Snapshot v1
+
+exports[`js |        // focus 1`] = `
+{
+  "data": "",
+  "focusString": "",
+  "key": "focus",
+}
+`;
+
+exports[`js |   // bg(1:3) #222 1`] = `
+{
+  "data": "#222",
+  "focusString": "(1:3)",
+  "key": "bg",
+}
+`;
+
+exports[`js |   // box[6:9] 1`] = `
+{
+  "data": "",
+  "focusString": "[6:9]",
+  "key": "box",
+}
+`;
+
+exports[`js | // focus 1`] = `
+{
+  "data": "",
+  "focusString": "",
+  "key": "focus",
+}
+`;
+
+exports[`js | const x = 1 1`] = `{}`;
+
+exports[`js | foo // bar 1`] = `{}`;
+
+exports[`python |        # focus 1`] = `
+{
+  "data": "",
+  "focusString": "",
+  "key": "focus",
+}
+`;
+
+exports[`python |   # bg(1:3) #222 1`] = `
+{
+  "data": "#222",
+  "focusString": "(1:3)",
+  "key": "bg",
+}
+`;
+
+exports[`python |  # box[6:9] 1`] = `
+{
+  "data": "",
+  "focusString": "[6:9]",
+  "key": "box",
+}
+`;
+
+exports[`python |  x = '#333' 1`] = `{}`;
+
+exports[`python | # focus 1`] = `
+{
+  "data": "",
+  "focusString": "",
+  "key": "focus",
+}
+`;
+
+exports[`python | foo # bar 1`] = `{}`;

packages/mdx/src/mdx-plugin/annotations.ts

@@ -3,6 +3,7 @@ import { CodeAnnotation } from "../smooth-code"
 import { wrapChildren } from "./to-estree"
 import { annotationsMap } from "../mdx-client/annotations"
 import { JsxNode as JsxNode, SuperNode } from "./nodes"
+import { getCommentData } from "./comment-data"
 
 export function getAnnotationsFromMetastring(
   options: Record<string, string>
@@ -24,8 +25,10 @@ export function extractAnnotationsFromCode(code: Code) {
   const focusList = [] as string[]
   while (lineNumber <= lines.length) {
     const line = lines[lineNumber - 1]
-    const { key, focusString, data } = getCommentData(line)
-    // console.log({ key, focusString, data })
+    const { key, focusString, data } = getCommentData(
+      line,
+      code.lang
+    )
 
     const Component = annotationsMap[key!]
 
@@ -50,31 +53,6 @@ export function extractAnnotationsFromCode(code: Code) {
   return [annotations, focusList.join(",")] as const
 }
 
-const commentRegex = /\/\/\s+(\w+)(\S*)\s*(.*)/
-function getCommentData(line: Code["lines"][0]) {
-  const comment = line.tokens.find(t =>
-    t.content.trim().startsWith("//")
-  )?.content
-
-  if (!comment) {
-    return {}
-  }
-
-  const result = commentRegex.exec(comment)
-
-  if (!result) {
-    return {}
-  }
-
-  const [, key, focusString, data] = result
-
-  return {
-    key,
-    focusString,
-    data,
-  }
-}
-
 export function extractJSXAnnotations(
   node: SuperNode,
   index: number,
@@ -108,7 +86,6 @@ export function extractJSXAnnotations(
       focus,
       data: isEmpty(data) ? undefined : data,
     })
-    // console.log(jsxAnnotation)
     parent.children.splice(nextIndex, 1)
   }
   return annotations

packages/mdx/src/mdx-plugin/comment-data.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect, test } from "vitest"
+import { highlight } from "../highlighter"
+import { getCommentData } from "./comment-data"
+
+test.each([
+  "// focus",
+  "       // focus",
+  "foo // bar",
+  "const x = 1",
+  "  // box[6:9]",
+  "  // bg(1:3) #222",
+])("js | %s", async code => {
+  expect(await getLineData("js", code)).toMatchSnapshot()
+})
+
+test.each([
+  "# focus",
+  "       # focus",
+  "foo # bar",
+  " x = '#333'",
+  " # box[6:9]",
+  "  # bg(1:3) #222",
+])("python | %s", async code => {
+  expect(
+    await getLineData("python", code)
+  ).toMatchSnapshot()
+})
+
+async function getLineData(lang: string, line: string) {
+  return getCommentData(await asLine(lang, line), lang)
+}
+
+async function asLine(lang: string, line: string) {
+  const code = await highlight({
+    code: line,
+    lang,
+    theme: {},
+  })
+  return code.lines[0]
+}

packages/mdx/src/mdx-plugin/comment-data.ts

@@ -0,0 +1,74 @@
+import { Code } from "../utils"
+import { annotationsMap } from "../mdx-client/annotations"
+
+const validKeys = ["focus", ...Object.keys(annotationsMap)]
+
+export function getCommentData(
+  line: Code["lines"][0],
+  lang: string
+) {
+  const result = bashLikeLangs.includes(lang)
+    ? bashLikeComment(line)
+    : otherComment(line)
+
+  if (!result) {
+    return {}
+  }
+
+  const [, key, focusString, data] = result
+
+  if (!validKeys.includes(key)) {
+    return {}
+  }
+
+  return {
+    key,
+    focusString,
+    data,
+  }
+}
+
+const commentRegex = /\/\/\s+(\w+)(\S*)\s*(.*)/
+function otherComment(line: Code["lines"][0]) {
+  const comment = line.tokens.find(t =>
+    t.content.trim().startsWith("//")
+  )?.content
+
+  if (!comment) {
+    return []
+  }
+
+  const result = commentRegex.exec(comment)
+
+  if (!result) {
+    return []
+  }
+
+  return result
+}
+
+const bashLikeLangs = [
+  "bash",
+  "sh",
+  "shell",
+  "python",
+  "py",
+]
+const bashLikeCommentRegex = /#\s+(\w+)(\S*)\s*(.*)/
+function bashLikeComment(line: Code["lines"][0]) {
+  const comment = line.tokens.find(t =>
+    t.content.trim().startsWith("#")
+  )?.content
+
+  if (!comment) {
+    return []
+  }
+
+  const result = bashLikeCommentRegex.exec(comment)
+
+  if (!result) {
+    return []
+  }
+
+  return result
+}