Merge pull request #146 from Annotation-Garden/feature/issue-144-persistent-lsp

feat: persistent hed-lsp client (closes #144)

Author: neuromechanist

.env.example

@@ -101,18 +101,27 @@ HED_SCHEMA_VERSION=8.4.0
 # ============================================================================
 # HED-LSP Configuration (Recommended)
 # ============================================================================
-# HEDit can use hed-lsp CLI for HED tag suggestions.
-# Install: git clone https://github.com/hed-standard/hed-lsp.git
-#          cd hed-lsp/server && npm install && npm run compile && npm link
+# HEDit holds one persistent connection to the hed-lsp Node server for the
+# lifetime of the process (server mode) or the command (standalone CLI).
+# Install hed-lsp: git clone https://github.com/hed-standard/hed-lsp.git
+#                  cd hed-lsp/server && npm install && npm run compile
 #
-# Once installed, the 'hed-suggest' command will be available in PATH.
-# HEDit auto-detects hed-lsp availability - no configuration needed!
-
-# Enable semantic search for better tag suggestions (requires embeddings)
-# HED_LSP_USE_SEMANTIC=false
-
-# Maximum number of tag suggestions to return
-# HED_LSP_MAX_RESULTS=10
+# Path to the built server.js. Inside the official Docker image this is
+# baked in at /app/hed-lsp/server/out/server.js. Outside Docker, point it
+# at your local checkout.
+# HED_LSP_SERVER_JS=/path/to/hed-lsp/server/out/server.js
+
+# Set to 1 to disable the persistent LSP connection entirely (the workflow
+# will run with keyword-only preprocessing, no LSP tag enrichment).
+# HED_LSP_DISABLE=0
+
+# Override the per-user runtime directory used by `hedit lsp start` to
+# place its socket and PID file.
+# HEDIT_LSP_RUNTIME_DIR=/path/to/runtime/dir
+
+# Set HEDIT_LSP=0 to make `hedit annotate` ignore an already-running
+# daemon (useful when debugging without the LSP path).
+# HEDIT_LSP=1
 
 # ============================================================================
 # Legacy JavaScript Validator (Deprecated)

.github/workflows/test.yml

@@ -47,6 +47,24 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set up Node.js for hed-lsp
+        uses: actions/setup-node@v6
+        with:
+          node-version: '22'
+
+      - name: Build hed-lsp (for tests/lsp/ real-LSP tests)
+        env:
+          HED_LSP_REF: 36447afb5e48bfecc45af0464b652614ba571414
+        run: |
+          git clone https://github.com/hed-standard/hed-lsp.git "$RUNNER_TEMP/hed-lsp"
+          git -C "$RUNNER_TEMP/hed-lsp" checkout --quiet "${HED_LSP_REF}"
+          cd "$RUNNER_TEMP/hed-lsp"
+          corepack enable
+          corepack prepare pnpm@10.33.4 --activate
+          pnpm install --frozen-lockfile
+          pnpm run -r compile
+          echo "HED_LSP_SERVER_JS=$RUNNER_TEMP/hed-lsp/server/out/server.js" >> "$GITHUB_ENV"
+
       - name: Install uv
         uses: astral-sh/setup-uv@v7
 

Dockerfile

@@ -41,19 +41,27 @@ WORKDIR /app
 
 # Clone HED repositories (self-contained)
 # Using schemas_latest_json from official hed-schemas repo (JSON inheritance fix now merged)
+# hed-lsp is pinned to the branch that adds the `hed/suggest` JSON-RPC
+# request handler; once that lands upstream this can switch back to a
+# tag-based clone of main.
+ARG HED_LSP_REF=feat/hed-suggest-request
 RUN git clone --depth 1 https://github.com/hed-standard/hed-schemas.git /app/hed-schemas && \
     git clone --depth 1 https://github.com/hed-standard/hed-javascript.git /app/hed-javascript && \
-    git clone --depth 1 https://github.com/hed-standard/hed-lsp.git /app/hed-lsp
+    git clone --depth 1 --branch "${HED_LSP_REF}" https://github.com/hed-standard/hed-lsp.git /app/hed-lsp
 
 # Build HED JavaScript validator
 WORKDIR /app/hed-javascript
 RUN npm install && npm run build
 
-# Build and install hed-suggest CLI from hed-lsp server
+# Build the hed-lsp server. As of the npm-to-pnpm migration in hed-lsp,
+# the repo uses pnpm workspaces; install pnpm via corepack so the
+# workspace's lockfile and server/node_modules resolve correctly.
+# The compiled server.js at /app/hed-lsp/server/out/server.js is what
+# the FastAPI lifespan spawns via HED_LSP_SERVER_JS.
 WORKDIR /app/hed-lsp
-RUN npm install && npm run compile
-WORKDIR /app/hed-lsp/server
-RUN npm install && npm link
+RUN corepack enable && corepack prepare pnpm@10.33.4 --activate && \
+    pnpm install --frozen-lockfile && \
+    pnpm run -r compile
 
 # Return to app directory
 WORKDIR /app
@@ -74,6 +82,7 @@ RUN pip install uv && \
 # Set environment variables for HED resources (internal paths)
 ENV HED_SCHEMA_DIR=/app/hed-schemas/schemas_latest_json \
     HED_VALIDATOR_PATH=/app/hed-javascript \
+    HED_LSP_SERVER_JS=/app/hed-lsp/server/out/server.js \
     USE_JS_VALIDATOR=true
 
 # Expose port

deploy/Dockerfile

@@ -23,20 +23,28 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \
     rm -rf /var/lib/apt/lists/*
 
 # Clone HED repositories (self-contained)
-# Using schemas_latest_json from official hed-schemas repo (JSON inheritance fix now merged)
+# hed-lsp is pinned to the merge commit on main that landed the
+# `hed/suggest` JSON-RPC request (hed-standard/hed-lsp#31, v0.4.0).
+# Pinning to a SHA (not a tag/branch) so the image is reproducible.
+ARG HED_LSP_REF=36447afb5e48bfecc45af0464b652614ba571414
 RUN git clone --depth 1 https://github.com/hed-standard/hed-schemas.git /app/hed-schemas && \
     git clone --depth 1 https://github.com/hed-standard/hed-javascript.git /app/hed-javascript && \
-    git clone --depth 1 https://github.com/hed-standard/hed-lsp.git /app/hed-lsp
+    git clone https://github.com/hed-standard/hed-lsp.git /app/hed-lsp && \
+    git -C /app/hed-lsp checkout --quiet "${HED_LSP_REF}"
 
 # Build HED JavaScript validator
 WORKDIR /app/hed-javascript
 RUN npm install && npm run build
 
-# Build and install hed-suggest CLI from hed-lsp server
+# Build the hed-lsp server. As of the npm-to-pnpm migration in hed-lsp,
+# the repo uses pnpm workspaces; install pnpm via corepack so the
+# workspace's lockfile and server/node_modules resolve correctly.
+# The compiled server.js at /app/hed-lsp/server/out/server.js is what
+# the FastAPI lifespan spawns via HED_LSP_SERVER_JS.
 WORKDIR /app/hed-lsp
-RUN npm install && npm run compile
-WORKDIR /app/hed-lsp/server
-RUN npm install && npm link
+RUN corepack enable && corepack prepare pnpm@10.33.4 --activate && \
+    pnpm install --frozen-lockfile && \
+    pnpm run -r compile
 
 # Return to app directory
 WORKDIR /app
@@ -56,6 +64,7 @@ COPY src /app/src
 # Set environment variables for HED resources (internal paths)
 ENV HED_SCHEMA_DIR=/app/hed-schemas/schemas_latest_json \
     HED_VALIDATOR_PATH=/app/hed-javascript \
+    HED_LSP_SERVER_JS=/app/hed-lsp/server/out/server.js \
     USE_JS_VALIDATOR=true
 
 # Expose the application port

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hedit"
-version = "0.7.9a2"
+version = "0.7.10a0"
 description = "Multi-agent system for HED annotation generation and validation"
 readme = "PKG_README.md"
 requires-python = ">=3.12"
@@ -39,6 +39,7 @@ dependencies = [
 
 [project.scripts]
 hedit = "src.cli.main:cli"
+hedit-lspd = "src.lsp.daemon:main"
 
 [project.urls]
 Homepage = "https://annotation.garden/hedit"

src/agents/validation_agent.py

@@ -12,9 +12,9 @@
 from pathlib import Path
 
 from src.agents.state import HedAnnotationState
+from src.lsp import HedLspClient
 from src.utils.error_remediation import get_remediator
 from src.utils.schema_loader import HedSchemaLoader
-from src.validation.hed_lsp import is_hed_lsp_available, suggest_tags_for_keywords
 from src.validation.hed_validator import (
     HedJavaScriptValidator,
     HedPythonValidator,
@@ -81,7 +81,7 @@ def __init__(
         use_javascript: bool = True,
         validator_path: Path | None = None,
         tests_json_path: Path | str | None = None,
-        use_hed_lsp: bool = True,
+        lsp_client: HedLspClient | None = None,
     ) -> None:
         """Initialize the validation agent.
 
@@ -90,13 +90,14 @@ def __init__(
             use_javascript: Whether to use JavaScript validator (more detailed)
             validator_path: Path to hed-javascript repository (required if use_javascript=True)
             tests_json_path: Optional path to javascriptTests.json for error remediation
-            use_hed_lsp: Whether to use hed-lsp for tag suggestions (auto-detected)
+            lsp_client: Pre-built HedLspClient for tag-replacement suggestions
+                when validation finds invalid tags. None disables suggestions.
         """
         self.schema_loader = schema_loader
         self.use_javascript = use_javascript
         self.validator_path = validator_path
         self.error_remediator = get_remediator(tests_json_path)
-        self.use_hed_lsp = use_hed_lsp and is_hed_lsp_available()
+        self.lsp_client = lsp_client
 
         # Validator is lazily initialized on first use via _get_or_create_validator
         self._validator: HedJavaScriptValidator | HedPythonValidator | None = None
@@ -134,34 +135,37 @@ def _extract_problematic_tags(self, errors: list, warnings: list) -> list[str]:
 
         return problematic_tags
 
-    def _get_tag_suggestions(
-        self, problematic_tags: list[str], schema_version: str
-    ) -> dict[str, list[str]]:
-        """Get suggested valid tags for problematic tags using hed-lsp.
+    async def _get_tag_suggestions(self, problematic_tags: list[str]) -> dict[str, list[str]]:
+        """Get suggested valid tags for problematic tags via persistent LSP.
 
         Args:
             problematic_tags: List of problematic tag names
-            schema_version: HED schema version
 
         Returns:
-            Dictionary mapping problematic tags to suggested alternatives
+            Dictionary mapping each problematic tag to a list of suggested
+            valid alternatives. Empty if no LSP client is configured or the
+            server returned no matches.
         """
-        if not self.use_hed_lsp or not problematic_tags:
+        if self.lsp_client is None or not problematic_tags:
             return {}
 
         try:
-            return suggest_tags_for_keywords(
-                problematic_tags,
-                schema_version=schema_version,
-                max_results=5,  # Limit suggestions for clarity
-            )
-        except (RuntimeError, OSError) as e:
+            result = await self.lsp_client.suggest(*problematic_tags)
+        except Exception as exc:
+            # `suggest()` already converts transport errors to a failure
+            # result; anything that lands here is a programming error and
+            # deserves a traceback.
             logger.warning(
-                "Failed to get tag suggestions from hed-lsp for tags %s: %s",
+                "hed-lsp suggest call failed for tags %s: %s",
                 problematic_tags,
-                e,
+                exc,
+                exc_info=True,
             )
             return {}
+        if not result.success:
+            logger.debug("hed-lsp suggest returned failure: %s", result.error)
+            return {}
+        return result.raw
 
     async def validate(self, state: HedAnnotationState) -> dict:
         """Validate the current HED annotation.
@@ -202,10 +206,10 @@ async def validate(self, state: HedAnnotationState) -> dict:
 
         # Extract problematic tags and get suggestions from hed-lsp
         tag_suggestions: dict[str, list[str]] = {}
-        if not result.is_valid and self.use_hed_lsp:
+        if not result.is_valid and self.lsp_client is not None:
             problematic_tags = self._extract_problematic_tags(result.errors, result.warnings)
             if problematic_tags:
-                tag_suggestions = self._get_tag_suggestions(problematic_tags, schema_version)
+                tag_suggestions = await self._get_tag_suggestions(problematic_tags)
 
         # Determine validation status
         validation_attempts = state["validation_attempts"] + 1