diff --git a/backlog/tasks/task-1 - Implement-TypeScript-schema-extension-with-FunctionDefinition-export-name-filtering.md b/backlog/tasks/task-1 - Implement-TypeScript-schema-extension-with-FunctionDefinition-export-name-filtering.md index 8320ae827..ea90e62b7 100644 --- a/backlog/tasks/task-1 - Implement-TypeScript-schema-extension-with-FunctionDefinition-export-name-filtering.md +++ b/backlog/tasks/task-1 - Implement-TypeScript-schema-extension-with-FunctionDefinition-export-name-filtering.md @@ -6,7 +6,7 @@ title: >- status: In Progress assignee: [] created_date: '2026-02-25 06:16' -updated_date: '2026-02-25 11:49' +updated_date: '2026-02-28 07:01' labels: - typescript - schema @@ -22,7 +22,13 @@ references: CallExpression-Confidence-Scoring-and-Validation-Agent.md documentation: - 'backlog://workflow/overview' + - 'https://nextjs.org/docs/app/building-your-application/routing/route-handlers' + - 'https://nextjs.org/docs/app/api-reference/file-conventions/route' + - 'https://tree-sitter.github.io/py-tree-sitter/classes/tree_sitter.Query.html' + - >- + https://tree-sitter.github.io/py-tree-sitter/classes/tree_sitter.QueryCursor.html priority: high +ordinal: 1000 --- ## Description @@ -49,19 +55,121 @@ Out of scope - [ ] #4 Prisma-style call expression detection remains based on absolute imports + `CallExpression`, with validator handoff for ambiguous cases. - [ ] #5 No additional TypeScript concept enums are introduced beyond `FunctionDefinition` in this change. - [ ] #6 Implementation docs and examples remain aligned with the updated plan and validator assumptions. +- [ ] #7 TypeScript language pipeline wires framework detection service so `.ts` files can emit `custom_features_list` detections for Next.js route handlers. +- [ ] #8 Import-gated matching is enforced for TypeScript: exported HTTP handlers are only matched when `next/server` absolute-path imports are present. +- [ ] #9 A canonical `framework-definitions/typescript/nextjs.json` definition is added and passes schema validation with existing v3 schema. ## Implementation Plan -1. Update shared contracts in commons and query-engine for FunctionDefinition and new construct_query regex keys. -2. Extend canonical and public schema mirrors with FunctionDefinition and function/export regex fields. -3. Relax absolute_paths schema pattern for TypeScript package/module forms while preserving terminal symbol requirement. -4. Update flow-bridge loader to discover framework definition files across all language directories. -5. Keep payload normalization compatible with FunctionDefinition values during ingestion. -6. Update validation workflow commands and framework-definition contributor docs. -7. Add targeted unit tests in commons/query-engine and loader-focused tests in flow-bridge. -8. Run targeted schema validation, lint/type checks, and pytest suites. +Objective +Implement end-to-end TypeScript framework detection for Next.js route handlers by reusing the existing schema/DB contracts and mirroring the Python detection pipeline architecture. + +Scope Boundary +- In scope: TypeScript `FunctionDefinition` detection for Next.js App Router route handlers in `.ts` files (e.g., `route.ts` exports like `GET`, `POST`). +- Out of scope: validator-agent workflow integration (covered by TASK-1.2), TSX-specific route detection, additional TypeScript concepts beyond `FunctionDefinition`. + +Phase 1 - Framework Definition Catalog (ingestion input) +1) Create `framework-definitions/typescript/nextjs.json`. +2) Add library `nextjs` with feature `route_handler_export`: + - `concept`: `FunctionDefinition` + - `target_level`: `function` + - `startpoint`: `true` + - `construct_query.function_name_regex`: `^(GET|HEAD|POST|PUT|DELETE|PATCH|OPTIONS)$` + - `construct_query.export_name_regex`: `^(GET|HEAD|POST|PUT|DELETE|PATCH|OPTIONS)$` + - `absolute_paths`: import gates from `next/server` symbols (initially `next/server.NextRequest`, `next/server.NextResponse`) + - `base_confidence`: set explicitly for deterministic downstream confidence seeding. +3) Keep schema unchanged unless validation reveals a gap; current v3 contract already includes needed fields. + +Phase 2 - TypeScript Source Context + Import Alias Model +Algorithm (deterministic import extraction) +1) Parse TypeScript source once using tree-sitter; keep a shared context object (`source_bytes`, `tree`, `root_node`). +2) Walk `import_statement` nodes and extract: + - module source string (e.g., `next/server`) + - default import alias (`import x from 'm'` -> `m -> x`) + - namespace alias (`import * as ns from 'm'` -> `m -> ns`) + - named imports (`import { A, B as C } from 'm'` -> `m.A -> A`, `m.B -> C`) + - type-only named imports handled same as named imports for import gating. +3) Build `import_aliases: dict[str, str]` in canonical form compatible with DB `absolute_paths` (e.g., `next/server.NextResponse -> NextResponse`, `next/server -> nextServer`). +4) Deduplicate + preserve deterministic ordering for reproducible downstream matching/debug logs. + +Phase 3 - TypeScript Query Builder (`FunctionDefinition`) +Algorithm (query generation) +1) Add a TypeScript query template for exported function declarations: + - capture export statement (`@export_statement`) + - capture declaration node (`@function_definition`) + - capture function name (`@function_name`) + - capture export name (`@export_name`) +2) Render predicates from `construct_query`: + - apply `#match? @function_name ...` when `function_name_regex` is present + - apply `#match? @export_name ...` when `export_name_regex` is present +3) Cache compiled query by hash(feature definition payload) for performance parity with Python. + +Phase 4 - TypeScript Tree-Sitter Framework Detector +Algorithm (feature matching) +1) For each `FeatureSpec`, enforce import gating first: + - match if any `absolute_path` exists directly in `import_aliases` + - or module prefix strategy matches (symbol imported via module alias path). +2) Concept dispatch: + - implement `Concept.FUNCTION_DEFINITION` + - explicitly skip unsupported concepts for TypeScript detector with debug logging. +3) Execute query and create detections from `@function_definition` spans: + - `start_line`, `end_line`, `match_text` + - metadata: `concept='FunctionDefinition'`, `source='tree_sitter'`, `match_confidence=feature_spec.base_confidence`. +4) Deduplicate identical span+feature rows in-memory before return. + +Phase 5 - TypeScript Framework Detection Service (DB-backed) +Algorithm (service orchestration) +1) Guard language == `typescript`; return empty otherwise. +2) Build TypeScript source context. +3) Expand import paths for DB lookup (module path and hierarchical prefixes where applicable). +4) Query PostgreSQL using existing `get_framework_features_for_imports(session, 'typescript', imports)`. +5) Pass `FeatureSpec` set to TypeScript detector and return detections. +6) Error handling/logging should mirror Python service behavior. + +Phase 6 - Parser/Processor Wiring +1) `CodeConfluenceCodebaseParser`: + - instantiate TypeScript framework detection service when codebase language is TypeScript. +2) `TypeScriptLanguageProcessor`: + - extract imports (no longer `imports=None`) + - call injected framework detection service + - populate `custom_features_list` with detections + - keep existing data-model detection behavior intact. + +Phase 7 - Tests (unit + integration) +1) Definition/loader tests: + - ensure `framework-definitions/typescript/nextjs.json` is loaded and preserves `FunctionDefinition` regex config. +2) Source-context tests: + - validate alias extraction for default/named/aliased/namespace/type imports from `next/server`. +3) Detector tests: + - positive: detects `export async function GET(...)` when `next/server` import exists + - negative: no detection without import gate + - regex filter: `GET` matches; non-HTTP export does not + - alias variant (`NextResponse as Resp`) still satisfies import gate. +4) Processor tests: + - TypeScript processor returns `imports` and `custom_features_list` for Next.js route sample. +5) Optional Postgres integration: + - DB-seeded TypeScript definitions produce detections via service path. + +Phase 8 - Validation + Quality Gates +Execution order +1) `uv run --group dev basedpyright src/` +2) `uv run ruff check src/` +3) `uv run --group dev check-jsonschema --schemafile framework-definitions/schema.json framework-definitions/*/*.json` +4) `uv run --group test pytest` for targeted TypeScript framework-detection tests, then broader suite if needed. + +Key Risks + Mitigations +- Risk: import alias edge cases in TypeScript grammar (type-only imports, aliases). + Mitigation: dedicated source-context tests for each import form. +- Risk: false positives on exported helper functions. + Mitigation: strict HTTP-method regex + import gating on `next/server` symbols. +- Risk: regression in existing TypeScript processor assumptions (`imports=None`). + Mitigation: update processor tests and keep backward-compatible optional imports field in `UnoplatFile`. + +Handoff Notes +- Keep validator/agent-trigger logic untouched in this task. +- If additional concepts are requested during implementation, pause and route as explicit scope change against TASK-1 acceptance criteria. ## Implementation Notes @@ -72,6 +180,12 @@ Set TASK-1 status to In Progress and prepared a concrete implementation sequence Implemented contract changes, schema updates, loader scanning updates, docs updates, and focused tests across flow-bridge, commons, and query-engine. Validation executed: check-jsonschema (framework-definitions/*/*.json), flow-bridge targeted pytest + ruff + basedpyright, commons targeted pytest, query-engine targeted pytest, and compileall checks. + +Detailed implementation flow was expanded on 2026-02-28 after repository audit. Key finding: schema/contracts already support `FunctionDefinition`; primary gap is missing TypeScript runtime pipeline (source-context/query-builder/detector/service wiring) and missing TypeScript framework definitions catalog file. + +Design decision: mirror Python detector architecture for parity and maintainability, but restrict TypeScript detector scope in this task to `FunctionDefinition` route handlers for Next.js App Router. + +Design decision: preserve confidence contract from TASK-1.1 by stamping `match_confidence` from `FeatureSpec.base_confidence` into detection metadata, allowing downstream validator flow in TASK-1.2 to remain unchanged. ## Definition of Done diff --git a/backlog/tasks/task-1.1 - Add-CallExpression-confidence-fields-to-schema-ingestion-docs-contracts-no-validator-integration.md b/backlog/tasks/task-1.1 - Add-CallExpression-confidence-fields-to-schema-ingestion-docs-contracts-no-validator-integration.md index df106e023..5affb0667 100644 --- a/backlog/tasks/task-1.1 - Add-CallExpression-confidence-fields-to-schema-ingestion-docs-contracts-no-validator-integration.md +++ b/backlog/tasks/task-1.1 - Add-CallExpression-confidence-fields-to-schema-ingestion-docs-contracts-no-validator-integration.md @@ -3,10 +3,10 @@ id: TASK-1.1 title: >- Add CallExpression confidence fields to schema + ingestion docs/contracts (no validator integration) -status: In Progress +status: Done assignee: [] created_date: '2026-02-26 05:53' -updated_date: '2026-02-26 07:39' +updated_date: '2026-02-28 06:04' labels: - schema - ingestion @@ -40,6 +40,7 @@ documentation: unoplat-code-confluence-ingestion/code-confluence-flow-bridge/prisma-call-expression-simplification-failure-cases.md parent_task_id: TASK-1 priority: high +ordinal: 2000 --- ## Description diff --git a/backlog/tasks/task-1.2 - Integrate-low-confidence-CallExpression-validator-agent-into-query-engine-workflow.md b/backlog/tasks/task-1.2 - Integrate-low-confidence-CallExpression-validator-agent-into-query-engine-workflow.md index 3799b3bcd..71834daea 100644 --- a/backlog/tasks/task-1.2 - Integrate-low-confidence-CallExpression-validator-agent-into-query-engine-workflow.md +++ b/backlog/tasks/task-1.2 - Integrate-low-confidence-CallExpression-validator-agent-into-query-engine-workflow.md @@ -3,10 +3,10 @@ id: TASK-1.2 title: >- Integrate low-confidence CallExpression validator agent into query-engine workflow -status: In Progress +status: Done assignee: [] created_date: '2026-02-26 10:41' -updated_date: '2026-02-27 12:43' +updated_date: '2026-02-28 06:04' labels: - query-engine - validator @@ -46,6 +46,7 @@ documentation: - 'backlog://workflow/overview' parent_task_id: TASK-1 priority: high +ordinal: 3000 --- ## Description diff --git a/backlog/tasks/task-1.2.1 - Fix-low-confidence-validation-regressions-in-candidate-hydration-and-mapper-gating.md b/backlog/tasks/task-1.2.1 - Fix-low-confidence-validation-regressions-in-candidate-hydration-and-mapper-gating.md index c28a2405a..3a4400b74 100644 --- a/backlog/tasks/task-1.2.1 - Fix-low-confidence-validation-regressions-in-candidate-hydration-and-mapper-gating.md +++ b/backlog/tasks/task-1.2.1 - Fix-low-confidence-validation-regressions-in-candidate-hydration-and-mapper-gating.md @@ -3,10 +3,10 @@ id: TASK-1.2.1 title: >- Fix low-confidence validation regressions in candidate hydration and mapper gating -status: In Progress +status: Done assignee: [] created_date: '2026-02-27 13:12' -updated_date: '2026-02-28 05:09' +updated_date: '2026-02-28 06:04' labels: - query-engine - validator @@ -32,6 +32,7 @@ documentation: CallExpression-Confidence-Scoring-and-Validation-Agent.md parent_task_id: TASK-1.2 priority: high +ordinal: 4000 --- ## Description diff --git a/backlog/tasks/task-2 - Remaining-fixes-for-typescript-detection.md b/backlog/tasks/task-2 - Remaining-fixes-for-typescript-detection.md new file mode 100644 index 000000000..f1ec7155e --- /dev/null +++ b/backlog/tasks/task-2 - Remaining-fixes-for-typescript-detection.md @@ -0,0 +1,29 @@ +--- +id: TASK-2 +title: Remaining fixes for typescript detection +status: To Do +assignee: [] +created_date: '2026-02-28 06:59' +labels: [] +dependencies: [] +--- + +## Description + + +11. You missed updating TypeScript processor v1 contract comments/tests. +- Gap: current file explicitly says framework/import extraction is deferred v1 in unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/language_processors/typescript_processor.py. +- Fix: update docstring/tests in tests/parser/language_processors/test_typescript_processor.py that currently assert imports are intentionally omitted. +12. You missed test impact from hardcoded framework counts. +- Gap: several ingestion tests hardcode 4/7/11 counts in tests/integration/test_framework_definitions_ingestion.py. +- Fix: if any test path starts loading all language dirs (not just python/), those assertions must become dynamic or updated. +13. README/docs update is missing from execution steps. +- Gap: framework-definitions/README.md still says TypeScript is “future framework definitions.” +- Fix: update unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/README.md once typescript/nextjs.json exists. +14. Query-engine retrieval verification is missing as a test step. +- Gap: plan validates ingestion/detection but not downstream retrieval behavior. +- Fix: add at least one assertion path for db_get_all_framework_features_for_codebase(..., programming_language="typescript") in query-engine tests, especially if you keep non-standard feature key. +15. Minor but important: route.ts file-convention gating is not represented. +- Gap: your detector may match any exported GET/POST in any .ts importing next/server. +- Fix: either explicitly accept this heuristic in task notes, or extend interface to pass file path and gate by filename in a follow-up. + diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/README.md b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/README.md index d4c4980d3..c348829e0 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/README.md +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/README.md @@ -56,6 +56,39 @@ Detection is **import-gated** and **regex-based**: When `base_confidence` is below `0.70`, include disambiguation guidance in feature `notes` so downstream validation can apply framework-specific checks consistently. +## Import Style For Detection Accuracy + +Framework matching depends on imports. Use explicit import style to improve recall and reduce false positives. + +- Python (enforced with Ruff): + - Prefer explicit imports (`from package import Symbol`) over wildcard imports. + - Keep absolute imports only (no relative imports). + - Relevant Ruff rules in this repo: `F403`, `F405`, `F406`, `TID`, `I`. + +- TypeScript/JavaScript (recommended with ESLint): + - Prefer named imports for framework entry modules (for example `next/server`). + - Avoid namespace imports for detection-critical modules. + - Avoid `require(...)` in ESM-first TypeScript codebases. + - Suggested rules: `import/no-namespace`, `@typescript-eslint/no-require-imports`, `import/no-commonjs`. + +Examples: + +```python +# preferred +from fastapi import APIRouter, FastAPI + +# avoid +from fastapi import * +``` + +```typescript +// preferred +import { NextRequest, NextResponse } from "next/server" + +// avoid for detection-critical imports +import * as nextServer from "next/server" +``` + ## Validation ### Validate All Framework Definitions diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/typescript/nextjs.json b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/typescript/nextjs.json new file mode 100644 index 000000000..03c74fff4 --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/framework-definitions/typescript/nextjs.json @@ -0,0 +1,22 @@ +{ + "typescript": { + "nextjs": { + "docs_url": "https://nextjs.org/docs/app/building-your-application/routing/route-handlers", + "description": "Next.js App Router HTTP route handler exports in route.ts files.", + "features": { + "http_endpoint": { + "description": "App Router HTTP method handler: named export function (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS). Requires next/server import to gate false positives.", + "absolute_paths": ["next/server.NextRequest", "next/server.NextResponse"], + "target_level": "function", + "concept": "FunctionDefinition", + "construct_query": { + "function_name_regex": "^(GET|HEAD|POST|PUT|DELETE|PATCH|OPTIONS)$", + "export_name_regex": "^(GET|HEAD|POST|PUT|DELETE|PATCH|OPTIONS)$" + }, + "base_confidence": 0.95, + "startpoint": true + } + } + } + } +} diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/pyproject.toml b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/pyproject.toml index 8b8a82f68..293793678 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/pyproject.toml +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/pyproject.toml @@ -63,7 +63,7 @@ dev = [ [tool.uv.sources] #NOTE: use it for local development to instantly see changes in the schema and its impact #unoplat-code-confluence-commons = { path = "../../unoplat-code-confluence-commons" } -unoplat-code-confluence-commons = { git = "https://github.com/unoplat/unoplat-code-confluence.git", subdirectory = "unoplat-code-confluence-commons", rev = "unoplat-code-confluence-commons-v0.43.1" } +unoplat-code-confluence-commons = { git = "https://github.com/unoplat/unoplat-code-confluence.git", subdirectory = "unoplat-code-confluence-commons", rev = "unoplat-code-confluence-commons-v0.44.0" } [tool.basedpyright] typeCheckingMode = "strict" diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/framework_detection_service.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/framework_detection_service.py index bed628ccf..bde7f7591 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/framework_detection_service.py +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/framework_detection_service.py @@ -2,14 +2,25 @@ Abstract base class for language-specific framework detection services. """ +from __future__ import annotations + from abc import ABC, abstractmethod -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from unoplat_code_confluence_commons.base_models import ( Detection, PythonStructuralSignature, + TypeScriptStructuralSignature, ) +if TYPE_CHECKING: + from src.code_confluence_flow_bridge.engine.programming_language.python.python_source_context import ( + PythonSourceContext, + ) + from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_source_context import ( + TypeScriptSourceContext, + ) + class FrameworkDetectionService(ABC): """Abstract base for language-specific framework detection services.""" @@ -19,8 +30,9 @@ async def detect_features( self, source_code: Optional[str], imports: List[str], - structural_signature: PythonStructuralSignature | None, + structural_signature: PythonStructuralSignature | TypeScriptStructuralSignature | None, programming_language: str, + source_context: PythonSourceContext | TypeScriptSourceContext | None = None, ) -> List[Detection]: """ Detect framework features in source code using imports and structural signature. @@ -29,7 +41,8 @@ async def detect_features( source_code: Source code to analyze imports: List of imports in the file structural_signature: Structural signature of the file (optional) - programming_language: Programming language (e.g., "python") + programming_language: Programming language (e.g., "python", "typescript") + source_context: Pre-parsed source context to avoid double-parsing (optional) Returns: List of Detection objects for framework features found diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_detection_service.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_detection_service.py index b884a2083..56da572ec 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_detection_service.py +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_detection_service.py @@ -2,12 +2,15 @@ Python-specific framework detection service implementation. """ -from typing import List, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional from loguru import logger from unoplat_code_confluence_commons.base_models import ( Detection, PythonStructuralSignature, + TypeScriptStructuralSignature, ) from src.code_confluence_flow_bridge.engine.framework_detection_service import ( @@ -24,6 +27,11 @@ get_framework_features_for_imports, ) +if TYPE_CHECKING: + from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_source_context import ( + TypeScriptSourceContext, + ) + def _expand_import_paths(import_paths: List[str]) -> List[str]: expanded: List[str] = [] @@ -46,8 +54,9 @@ async def detect_features( self, source_code: Optional[str], imports: List[str], - structural_signature: PythonStructuralSignature | None, + structural_signature: PythonStructuralSignature | TypeScriptStructuralSignature | None, programming_language: str, + source_context: PythonSourceContext | TypeScriptSourceContext | None = None, ) -> List[Detection]: """ Detect framework features in Python source code using tree-sitter queries. @@ -57,10 +66,12 @@ async def detect_features( imports: List of imports in the file (unused - kept for interface compatibility) structural_signature: Structural signature of the file (unused) programming_language: Programming language (should be "python") + source_context: Pre-parsed source context (unused in Python path) Returns: List of Detection objects for framework features found """ + _ = source_context # unused in Python path if programming_language.lower() != "python": logger.warning( "PythonFrameworkDetectionService called with language: {}", diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_query_builder.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_query_builder.py index 8c72a44c7..615482831 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_query_builder.py +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/python/python_framework_query_builder.py @@ -16,7 +16,6 @@ TargetLevel, ) - _TEMPLATE_DIR = Path(__file__).resolve().parent / "queries" _TEMPLATE_PATHS = { "annotation_function": _TEMPLATE_DIR / "annotation_function_like.scm", @@ -113,11 +112,13 @@ def _render_query(self, template: str, feature_spec: FeatureSpec) -> str: return _render_template(template, replacements) - def _construct_query_config(self, feature_spec: FeatureSpec) -> ConstructQueryConfig: + def _construct_query_config( + self, feature_spec: FeatureSpec + ) -> ConstructQueryConfig: construct_query = feature_spec.construct_query_typed if construct_query is not None: return construct_query - return ConstructQueryConfig() + return ConstructQueryConfig.model_validate({}) def _build_annotation_call_block(self, method_regex: Optional[str]) -> str: predicate = _render_predicate("@decorator_method", method_regex) diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/queries/function_definition.scm b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/queries/function_definition.scm new file mode 100644 index 000000000..80bb9efae --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/queries/function_definition.scm @@ -0,0 +1,7 @@ +(export_statement + declaration: (function_declaration + name: (identifier) @export_name @function_name + ) @function_definition + {{EXPORT_NAME_PREDICATE}} + {{FUNCTION_NAME_PREDICATE}} +) @export_statement diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/queries/imports.scm b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/queries/imports.scm new file mode 100644 index 000000000..1eec33042 --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/queries/imports.scm @@ -0,0 +1 @@ +(import_statement) @import diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_framework_detection_service.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_framework_detection_service.py new file mode 100644 index 000000000..570b4fa67 --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_framework_detection_service.py @@ -0,0 +1,117 @@ +"""TypeScript-specific framework detection service implementation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +from loguru import logger +from unoplat_code_confluence_commons.base_models import ( + Detection, + PythonStructuralSignature, + TypeScriptStructuralSignature, +) + +from src.code_confluence_flow_bridge.engine.framework_detection_service import ( + FrameworkDetectionService, +) +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_source_context import ( + TypeScriptSourceContext, +) +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_tree_sitter_framework_detector import ( + TypeScriptTreeSitterFrameworkDetector, +) +from src.code_confluence_flow_bridge.processor.db.postgres.db import get_session_cm +from src.code_confluence_flow_bridge.processor.db.postgres.framework_query_service import ( + get_framework_features_for_imports, +) + +if TYPE_CHECKING: + from src.code_confluence_flow_bridge.engine.programming_language.python.python_source_context import ( + PythonSourceContext, + ) + + +def _expand_import_paths(import_paths: List[str]) -> List[str]: + expanded: List[str] = [] + for path in import_paths: + expanded.append(path) + parts = path.split(".") + for idx in range(1, len(parts)): + expanded.append(".".join(parts[:idx])) + return sorted(set(expanded)) + + +class TypeScriptFrameworkDetectionService(FrameworkDetectionService): + """TypeScript-specific framework detection service.""" + + def __init__(self) -> None: + self.detector = TypeScriptTreeSitterFrameworkDetector() + + async def detect_features( + self, + source_code: Optional[str], + imports: List[str], + structural_signature: PythonStructuralSignature | TypeScriptStructuralSignature | None, + programming_language: str, + source_context: PythonSourceContext | TypeScriptSourceContext | None = None, + ) -> List[Detection]: + """ + Detect framework features in TypeScript source code using tree-sitter queries. + + Reuses pre-parsed source context when provided to avoid double-parsing. + """ + if programming_language.lower() != "typescript": + logger.warning( + "TypeScriptFrameworkDetectionService called with language: {}", + programming_language, + ) + return [] + + # Reuse already-parsed context if caller provides it (avoids double parse) + if isinstance(source_context, TypeScriptSourceContext): + context = source_context + else: + if not source_code: + logger.debug("No source code provided for TypeScript framework detection") + return [] + context = TypeScriptSourceContext.from_source(source_code) + + if not context.import_aliases: + logger.debug("No import aliases found in TypeScript source") + return [] + + try: + async with get_session_cm() as session: + absolute_paths = _expand_import_paths( + list(context.import_aliases.keys()) + ) + feature_specs = await get_framework_features_for_imports( + session, "typescript", absolute_paths + ) + + logger.opt(lazy=True).debug( + "TypeScript framework feature specs loaded | count={} | imports={}", + lambda: len(feature_specs), + lambda: absolute_paths, + ) + + if not feature_specs: + logger.debug( + "No TypeScript framework features found for imports: {}", + absolute_paths, + ) + return [] + + detections = self.detector.detect(context, feature_specs) + + logger.opt(lazy=True).debug( + "Detected {} TypeScript framework features from {} feature specs", + lambda: len(detections), + lambda: len(feature_specs), + ) + + return detections + + except Exception as e: + logger.error("Error in TypeScript framework detection: {}", e) + return [] diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_framework_query_builder.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_framework_query_builder.py new file mode 100644 index 000000000..6ad21f161 --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_framework_query_builder.py @@ -0,0 +1,108 @@ +"""Dynamic tree-sitter query builder for TypeScript framework detection.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Dict, Optional + +import tree_sitter +from tree_sitter_language_pack import get_language +from unoplat_code_confluence_commons.base_models import ( + Concept, + ConstructQueryConfig, + FeatureSpec, +) + +_TEMPLATE_DIR = Path(__file__).resolve().parent / "queries" +_TEMPLATE_PATHS = { + "function_definition": _TEMPLATE_DIR / "function_definition.scm", +} + +_QUERY_CACHE: Dict[str, tree_sitter.Query] = {} + + +def _escape_query_regex(regex: str) -> str: + return regex.replace("\\", "\\\\").replace('"', '\\"') + + +def _render_predicate(capture_name: str, regex: Optional[str]) -> str: + if not regex: + return "" + safe_regex = _escape_query_regex(regex) + return f'(#match? {capture_name} "{safe_regex}")' + + +def _render_template(template: str, replacements: Dict[str, str]) -> str: + rendered = template + for key, value in replacements.items(): + rendered = rendered.replace(f"{{{{{key}}}}}", value) + return rendered + + +def _load_template(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _definition_hash(feature_spec: FeatureSpec) -> str: + payload = { + "feature_key": feature_spec.feature_key, + "library": feature_spec.library, + "absolute_paths": feature_spec.absolute_paths, + "target_level": feature_spec.target_level.value, + "concept": feature_spec.concept.value, + "locator_strategy": feature_spec.locator_strategy.value, + "construct_query": feature_spec.construct_query, + } + payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload_json.encode("utf-8")).hexdigest() + + +class TypeScriptFrameworkQueryBuilder: + """Builds tree-sitter queries for TypeScript framework detection.""" + + def __init__(self) -> None: + self._language = get_language("typescript") # type: ignore[arg-type] + + def build_query(self, feature_spec: FeatureSpec) -> tree_sitter.Query: + if feature_spec.concept != Concept.FUNCTION_DEFINITION: + raise ValueError( + f"TypeScriptFrameworkQueryBuilder only supports FunctionDefinition concept, " + f"got: {feature_spec.concept}" + ) + + template_path = _TEMPLATE_PATHS["function_definition"] + template = _load_template(template_path) + query_source = self._render_query(template, feature_spec) + cache_key = f"function_definition:{_definition_hash(feature_spec)}" + + if cache_key not in _QUERY_CACHE: + _QUERY_CACHE[cache_key] = tree_sitter.Query(self._language, query_source) + + return _QUERY_CACHE[cache_key] + + def _construct_query_config( + self, feature_spec: FeatureSpec + ) -> ConstructQueryConfig: + construct_query = feature_spec.construct_query_typed + if construct_query is not None: + return construct_query + return ConstructQueryConfig.model_validate({}) + + def _render_query(self, template: str, feature_spec: FeatureSpec) -> str: + construct_query = self._construct_query_config(feature_spec) + + export_name_predicate = _render_predicate( + "@export_name", construct_query.export_name_regex + ) + function_name_predicate = _render_predicate( + "@function_name", construct_query.function_name_regex + ) + + replacements = { + "EXPORT_NAME_PREDICATE": export_name_predicate, + "FUNCTION_NAME_PREDICATE": function_name_predicate, + } + + return _render_template(template, replacements) diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_source_context.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_source_context.py new file mode 100644 index 000000000..2cf24ec5b --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_source_context.py @@ -0,0 +1,169 @@ +"""Shared TypeScript source context for tree-sitter based detection.""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from typing import Dict, List + +from pydantic import BaseModel, ConfigDict +import tree_sitter +from tree_sitter_language_pack import get_language, get_parser + +_IMPORT_QUERY_PATH = Path(__file__).resolve().parent / "queries" / "imports.scm" + + +@lru_cache(maxsize=1) +def _get_import_query() -> tree_sitter.Query: + language = get_language("typescript") # type: ignore[arg-type] + query_source = _IMPORT_QUERY_PATH.read_text(encoding="utf-8") + return tree_sitter.Query(language, query_source) + + +def _extract_imports_from_tree( + root_node: tree_sitter.Node, source_bytes: bytes +) -> List[str]: + cursor = tree_sitter.QueryCursor(_get_import_query()) + captures: Dict[str, List[tree_sitter.Node]] = cursor.captures(root_node) + + imports: List[str] = [] + for nodes in captures.values(): + for node in nodes: + imports.append( + source_bytes[node.start_byte : node.end_byte].decode("utf-8") + ) + + return imports + + +def _record_import_alias( + mapping: Dict[str, str], full_path: str, alias: str +) -> None: + if full_path not in mapping: + mapping[full_path] = alias + + +def _get_string_value(node: tree_sitter.Node, source_bytes: bytes) -> str: + """Extract the string value from a TypeScript string node (strips quotes).""" + text = source_bytes[node.start_byte : node.end_byte].decode("utf-8") + return text.strip("'\"") + + +def build_import_aliases(imports: List[str]) -> Dict[str, str]: + """ + Return a mapping from fully-qualified import path to its local alias. + + Supported forms: + - Named: import { NextRequest } from 'next/server' + → {next/server.NextRequest: NextRequest} + - Named aliased: import { NextRequest as NR } from 'next/server' + → {next/server.NextRequest: NR} + - Default: import Next from 'next/server' + → {next/server: Next} + - Type-only named: import type { NextRequest } from 'next/server' + → same as named + - Namespace: import * as ns from 'next/server' → skipped (v1) + """ + parser = get_parser("typescript") # type: ignore[arg-type] + mapping: Dict[str, str] = {} + + for import_statement in imports: + if not import_statement.strip(): + continue + + tree = parser.parse(bytes(import_statement, "utf8")) + src_bytes = bytes(import_statement, "utf8") + + for node in tree.root_node.children: + if node.type != "import_statement": + continue + + # Find the module path (string node after 'from') + module_str_node = next( + (c for c in node.children if c.type == "string"), None + ) + if module_str_node is None: + continue + module_path = _get_string_value(module_str_node, src_bytes) + + # Find import_clause + import_clause = next( + (c for c in node.children if c.type == "import_clause"), None + ) + if import_clause is None: + continue + + for clause_child in import_clause.children: + if clause_child.type == "identifier": + # Default import: import Foo from 'mod' + alias = src_bytes[ + clause_child.start_byte : clause_child.end_byte + ].decode("utf-8") + _record_import_alias(mapping, module_path, alias) + + elif clause_child.type == "named_imports": + # Named imports: { Foo, Bar as B } + for specifier in clause_child.children: + if specifier.type != "import_specifier": + continue + identifiers = [ + c + for c in specifier.children + if c.type == "identifier" + ] + if len(identifiers) == 1: + # Simple named: { Foo } + name = src_bytes[ + identifiers[0].start_byte : identifiers[0].end_byte + ].decode("utf-8") + _record_import_alias( + mapping, f"{module_path}.{name}", name + ) + elif len(identifiers) >= 2: + # Aliased named: { Foo as Bar } + # identifiers[0] = original name, identifiers[-1] = alias + original = src_bytes[ + identifiers[0].start_byte : identifiers[0].end_byte + ].decode("utf-8") + alias = src_bytes[ + identifiers[-1].start_byte : identifiers[-1].end_byte + ].decode("utf-8") + _record_import_alias( + mapping, f"{module_path}.{original}", alias + ) + + elif clause_child.type == "namespace_import": + # Namespace import: import * as ns from 'mod' → skipped in v1 + pass + + return mapping + + +class TypeScriptSourceContext(BaseModel): + """Shared parsed source context for TypeScript detection paths.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + source_code: str + source_bytes: bytes + tree: tree_sitter.Tree + root_node: tree_sitter.Node + imports: List[str] + import_aliases: Dict[str, str] + + @classmethod + def from_source(cls, source_code: str) -> "TypeScriptSourceContext": + source_bytes = source_code.encode("utf-8", errors="ignore") + parser = get_parser("typescript") # type: ignore[arg-type] + tree = parser.parse(source_bytes) + root_node = tree.root_node + imports = _extract_imports_from_tree(root_node, source_bytes) + import_aliases = build_import_aliases(imports) + return cls( + source_code=source_code, + source_bytes=source_bytes, + tree=tree, + root_node=root_node, + imports=imports, + import_aliases=import_aliases, + ) diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_tree_sitter_framework_detector.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_tree_sitter_framework_detector.py new file mode 100644 index 000000000..990e3d6e8 --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/engine/programming_language/typescript/typescript_tree_sitter_framework_detector.py @@ -0,0 +1,151 @@ +"""Tree-sitter based TypeScript framework detector for FunctionDefinition detection.""" + +from __future__ import annotations + +from typing import Dict, List, Optional, Set, Tuple + +from loguru import logger +import tree_sitter +from unoplat_code_confluence_commons.base_models import ( + Concept, + Detection, + FeatureSpec, +) + +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_framework_query_builder import ( + TypeScriptFrameworkQueryBuilder, +) +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_source_context import ( + TypeScriptSourceContext, +) + + +def _extract_node_text(source_bytes: bytes, node: tree_sitter.Node) -> str: + return source_bytes[node.start_byte : node.end_byte].decode( + "utf-8", errors="ignore" + ) + + +def _is_feature_imported( + absolute_paths: List[str], import_aliases: Dict[str, str] +) -> bool: + for absolute_path in absolute_paths: + if absolute_path in import_aliases: + return True + parts = absolute_path.split(".") + for idx in range(1, len(parts)): + prefix = ".".join(parts[:idx]) + if prefix in import_aliases: + return True + return False + + +def _first_capture( + captures: Dict[str, List[tree_sitter.Node]], name: str +) -> Optional[tree_sitter.Node]: + nodes = captures.get(name) + if nodes: + return nodes[0] + return None + + +class TypeScriptTreeSitterFrameworkDetector: + """Detect TypeScript framework features using tree-sitter queries.""" + + def __init__(self) -> None: + self._query_builder = TypeScriptFrameworkQueryBuilder() + + def detect( + self, context: TypeScriptSourceContext, feature_specs: List[FeatureSpec] + ) -> List[Detection]: + detections: List[Detection] = [] + for spec in feature_specs: + try: + if not _is_feature_imported( + spec.absolute_paths, context.import_aliases + ): + logger.opt(lazy=True).debug( + "Skipping feature; import not found | library={} | feature_key={} | paths={} | aliases={}", + lambda: spec.library, + lambda: spec.feature_key, + lambda: spec.absolute_paths, + lambda: sorted(context.import_aliases.keys()), + ) + continue + feature_detections = self._detect_feature(context, spec) + if feature_detections: + logger.opt(lazy=True).debug( + "Feature detections | library={} | feature_key={} | count={}", + lambda: spec.library, + lambda: spec.feature_key, + lambda: len(feature_detections), + ) + detections.extend(feature_detections) + except Exception as exc: + logger.warning( + "TypeScript framework detection failed | feature_key={} | error={}", + spec.feature_key, + exc, + ) + return detections + + def _detect_feature( + self, context: TypeScriptSourceContext, spec: FeatureSpec + ) -> List[Detection]: + if spec.concept != Concept.FUNCTION_DEFINITION: + logger.debug( + "Skipping unsupported concept in TypeScript detector | concept={}", + spec.concept, + ) + return [] + + query = self._query_builder.build_query(spec) + cursor = tree_sitter.QueryCursor(query) + matches = cursor.matches(context.root_node) + return self._detect_function_definition(context, spec, matches) + + def _detect_function_definition( + self, + context: TypeScriptSourceContext, + spec: FeatureSpec, + matches: List[Tuple[int, Dict[str, List[tree_sitter.Node]]]], + ) -> List[Detection]: + detections: List[Detection] = [] + seen: Set[Tuple[str, str, int, int]] = set() + source_bytes = context.source_bytes + + for _pattern_index, captures in matches: + function_name_node = _first_capture(captures, "function_name") + export_statement_node = _first_capture(captures, "export_statement") + + if function_name_node is None or export_statement_node is None: + continue + + function_name_text = _extract_node_text(source_bytes, function_name_node) + match_text = _extract_node_text(source_bytes, export_statement_node) + start_line = export_statement_node.start_point[0] + 1 + end_line = export_statement_node.end_point[0] + 1 + + dedup_key = (spec.library, spec.feature_key, start_line, end_line) + if dedup_key in seen: + continue + seen.add(dedup_key) + + detections.append( + Detection( + feature_key=spec.feature_key, + library=spec.library, + match_text=match_text, + start_line=start_line, + end_line=end_line, + metadata={ + "concept": "FunctionDefinition", + "source": "tree_sitter", + "function_name": function_name_text, + "export_name": function_name_text, + "match_confidence": spec.base_confidence, + }, + ) + ) + + return detections diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/code_confluence_codebase_parser.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/code_confluence_codebase_parser.py index e64093970..23b8d9b4f 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/code_confluence_codebase_parser.py +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/code_confluence_codebase_parser.py @@ -18,6 +18,9 @@ from src.code_confluence_flow_bridge.engine.programming_language.python.python_framework_detection_service import ( PythonFrameworkDetectionService, ) +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_framework_detection_service import ( + TypeScriptFrameworkDetectionService, +) from src.code_confluence_flow_bridge.models.configuration.settings import ( EnvironmentSettings, ) @@ -128,6 +131,9 @@ def _initialize_components(self) -> None: if self.programming_language_metadata.language.value == "python": self.framework_detection_service = PythonFrameworkDetectionService() logger.debug("Initialized Python framework detection service") + elif self.programming_language_metadata.language.value == "typescript": + self.framework_detection_service = TypeScriptFrameworkDetectionService() + logger.debug("Initialized TypeScript framework detection service") else: logger.debug( "No framework detection service available for language: {}", diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/language_processors/typescript_processor.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/language_processors/typescript_processor.py index d94a58b39..712bd4e13 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/language_processors/typescript_processor.py +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/src/code_confluence_flow_bridge/parser/language_processors/typescript_processor.py @@ -1,16 +1,20 @@ # Standard Library import asyncio import hashlib -from typing import Optional, Set +from typing import List, Optional, Set # Third Party from aiofile import async_open from loguru import logger +from unoplat_code_confluence_commons.base_models import Detection # First Party from src.code_confluence_flow_bridge.engine.detector.data_model_detector import ( detect_data_model, ) +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_source_context import ( + TypeScriptSourceContext, +) from src.code_confluence_flow_bridge.models.code_confluence_parsing_models.unoplat_file import ( UnoplatFile, ) @@ -25,10 +29,9 @@ class TypeScriptLanguageProcessor(LanguageCodebaseProcessor): """Language processor responsible for parsing TypeScript source files. - Version 1 deliberately focuses on data-model detection only. Structural - signatures and import extraction will be wired once the dedicated - Tree-sitter extractor is ready. `.tsx` support is deferred because it - requires loading a distinct grammar and query set. + Version 1 focuses on data-model detection and framework feature detection via + tree-sitter. `.tsx` support is deferred because it requires loading a distinct + grammar and query set. """ _SUPPORTED_EXTENSIONS: Set[str] = {".ts"} @@ -57,7 +60,7 @@ def _calculate_file_checksum(content: bytes) -> str: return "" async def extract_file_data(self, file_path: str) -> Optional[UnoplatFile]: - """Read a TypeScript file and emit an `UnoplatFile` with data-model metadata.""" + """Read a TypeScript file and emit an `UnoplatFile` with data-model and framework metadata.""" metadata = self.context.programming_language_metadata try: @@ -70,8 +73,11 @@ async def extract_file_data(self, file_path: str) -> Optional[UnoplatFile]: content_text = content_bytes.decode("utf-8", errors="ignore") - # Imports intentionally omitted in v1. Field is optional, so set None. - imports = None + # Parse once with tree-sitter and reuse the context for all detection paths + ts_context = TypeScriptSourceContext.from_source(content_text) + + # Populate imports from parsed context + imports: Optional[List[str]] = ts_context.imports if ts_context.imports else None has_data_model, data_model_positions = detect_data_model( source_code=content_text, @@ -80,12 +86,23 @@ async def extract_file_data(self, file_path: str) -> Optional[UnoplatFile]: structural_signature=None, ) + custom_features_list: Optional[List[Detection]] = None + if self.context.framework_detection_service is not None: + detections = await self.context.framework_detection_service.detect_features( + source_code=content_text, + imports=imports or [], + structural_signature=None, + programming_language=metadata.language.value, + source_context=ts_context, + ) + custom_features_list = detections or None + return UnoplatFile( file_path=file_path, checksum=checksum, imports=imports, structural_signature=None, - custom_features_list=None, + custom_features_list=custom_features_list, has_data_model=has_data_model, data_model_positions=data_model_positions, ) diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/__init__.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/programming_language/__init__.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/programming_language/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/programming_language/typescript/__init__.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/programming_language/typescript/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/programming_language/typescript/test_nextjs_extraction.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/programming_language/typescript/test_nextjs_extraction.py new file mode 100644 index 000000000..fba302f4b --- /dev/null +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/engine/programming_language/typescript/test_nextjs_extraction.py @@ -0,0 +1,137 @@ +"""Unit tests for Next.js extraction in the TypeScript detector pipeline.""" + +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_source_context import ( + TypeScriptSourceContext, +) +from src.code_confluence_flow_bridge.engine.programming_language.typescript.typescript_tree_sitter_framework_detector import ( + TypeScriptTreeSitterFrameworkDetector, +) +from unoplat_code_confluence_commons.base_models import ( + Concept, + FeatureSpec, + LocatorStrategy, + TargetLevel, +) + + +def _build_nextjs_feature_spec() -> FeatureSpec: + return FeatureSpec( + feature_key="http_endpoint", + library="nextjs", + description="Next.js App Router HTTP method export", + absolute_paths=["next/server.NextRequest", "next/server.NextResponse"], + target_level=TargetLevel.FUNCTION, + concept=Concept.FUNCTION_DEFINITION, + locator_strategy=LocatorStrategy.VARIABLE_BOUND, + construct_query={ + "function_name_regex": "^(GET|HEAD|POST|PUT|DELETE|PATCH|OPTIONS)$", + "export_name_regex": "^(GET|HEAD|POST|PUT|DELETE|PATCH|OPTIONS)$", + }, + base_confidence=0.95, + startpoint=True, + ) + + +def test_source_context_extracts_nextjs_import_aliases() -> None: + source_code = """ +import { NextRequest, NextResponse as Resp } from "next/server" +import type { NextFetchEvent } from "next/server" +""" + + context = TypeScriptSourceContext.from_source(source_code) + + assert len(context.imports) == 2 + assert context.import_aliases["next/server.NextRequest"] == "NextRequest" + assert context.import_aliases["next/server.NextResponse"] == "Resp" + assert context.import_aliases["next/server.NextFetchEvent"] == "NextFetchEvent" + + +def test_detector_detects_exported_get_with_next_server_import() -> None: + source_code = """ +import { NextRequest, NextResponse } from "next/server" + +export async function GET(request: NextRequest) { + return NextResponse.json({ ok: true }) +} +""" + + context = TypeScriptSourceContext.from_source(source_code) + detector = TypeScriptTreeSitterFrameworkDetector() + + detections = detector.detect(context, [_build_nextjs_feature_spec()]) + + assert len(detections) == 1 + detection = detections[0] + assert detection.feature_key == "http_endpoint" + assert detection.library == "nextjs" + assert "export async function GET" in detection.match_text + assert detection.metadata["function_name"] == "GET" + assert detection.metadata["export_name"] == "GET" + assert detection.metadata["match_confidence"] == 0.95 + + +def test_detector_skips_exported_get_without_next_server_import() -> None: + source_code = """ +export async function GET() { + return Response.json({ ok: true }) +} +""" + + context = TypeScriptSourceContext.from_source(source_code) + detector = TypeScriptTreeSitterFrameworkDetector() + + detections = detector.detect(context, [_build_nextjs_feature_spec()]) + + assert detections == [] + + +def test_detector_skips_non_http_export_name_with_import_gate() -> None: + source_code = """ +import { NextRequest, NextResponse } from "next/server" + +export async function handler(request: NextRequest) { + return NextResponse.json({ ok: true }) +} +""" + + context = TypeScriptSourceContext.from_source(source_code) + detector = TypeScriptTreeSitterFrameworkDetector() + + detections = detector.detect(context, [_build_nextjs_feature_spec()]) + + assert detections == [] + + +def test_detector_allows_aliased_named_import_for_import_gate() -> None: + source_code = """ +import { NextResponse as Resp } from "next/server" + +export function POST() { + return Resp.json({ created: true }) +} +""" + + context = TypeScriptSourceContext.from_source(source_code) + detector = TypeScriptTreeSitterFrameworkDetector() + + detections = detector.detect(context, [_build_nextjs_feature_spec()]) + + assert len(detections) == 1 + assert detections[0].metadata["function_name"] == "POST" + + +def test_detector_namespace_import_does_not_satisfy_gate_in_v1() -> None: + source_code = """ +import * as nextServer from "next/server" + +export async function GET() { + return nextServer.NextResponse.json({ ok: true }) +} +""" + + context = TypeScriptSourceContext.from_source(source_code) + detector = TypeScriptTreeSitterFrameworkDetector() + + detections = detector.detect(context, [_build_nextjs_feature_spec()]) + + assert detections == [] diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/language_processors/__init__.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/language_processors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/language_processors/test_typescript_processor.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/language_processors/test_typescript_processor.py index 266caa74b..4bdac0ca0 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/language_processors/test_typescript_processor.py +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/language_processors/test_typescript_processor.py @@ -98,5 +98,7 @@ async def test_extract_file_data_detects_data_models( assert positions.get("ApiResponse") == (70, 73) assert positions.get("ApiError") == (75, 80) - # Imports are intentionally skipped in v1 - assert file_data.imports is None + # Imports are extracted in v1 for framework detection gating + assert file_data.imports is not None + assert len(file_data.imports) > 0 + assert any("from 'axios'" in import_stmt for import_stmt in file_data.imports) diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/typescript/__init__.py b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/tests/parser/typescript/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/uv.lock b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/uv.lock index ba50b30ca..694d15fc5 100644 --- a/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/uv.lock +++ b/unoplat-code-confluence-ingestion/code-confluence-flow-bridge/uv.lock @@ -284,7 +284,7 @@ wheels = [ [[package]] name = "code-confluence-flow-bridge" -version = "0.67.5" +version = "0.68.0" source = { virtual = "." } dependencies = [ { name = "aiofile" }, @@ -366,7 +366,7 @@ requires-dist = [ { name = "tomlkit", specifier = ">=0.13.2" }, { name = "tree-sitter-language-pack", specifier = ">=0.8.0" }, { name = "typing-extensions", specifier = ">=4.11" }, - { name = "unoplat-code-confluence-commons", directory = "../../unoplat-code-confluence-commons" }, + { name = "unoplat-code-confluence-commons", git = "https://github.com/unoplat/unoplat-code-confluence.git?subdirectory=unoplat-code-confluence-commons&rev=unoplat-code-confluence-commons-v0.44.0" }, { name = "validate-pyproject", extras = ["all"], specifier = ">=0.23" }, ] @@ -2062,8 +2062,8 @@ wheels = [ [[package]] name = "unoplat-code-confluence-commons" -version = "0.43.0" -source = { directory = "../../unoplat-code-confluence-commons" } +version = "0.44.0" +source = { git = "https://github.com/unoplat/unoplat-code-confluence.git?subdirectory=unoplat-code-confluence-commons&rev=unoplat-code-confluence-commons-v0.44.0#08aec555de7778bf33a6774c4fa451109d8403d0" } dependencies = [ { name = "cryptography" }, { name = "pydantic" }, @@ -2073,16 +2073,6 @@ dependencies = [ { name = "sqlmodel" }, ] -[package.metadata] -requires-dist = [ - { name = "cryptography", specifier = ">=45.0.6" }, - { name = "pydantic", specifier = ">=2.11.7" }, - { name = "pytest", specifier = "==8.3.4" }, - { name = "pytest-asyncio", specifier = ">=0.25.0,<0.26" }, - { name = "sqlalchemy", specifier = ">=2.0.41" }, - { name = "sqlmodel", specifier = ">=0.0.24" }, -] - [[package]] name = "urllib3" version = "2.4.0" diff --git a/unoplat-code-confluence-query-engine/pyproject.toml b/unoplat-code-confluence-query-engine/pyproject.toml index 3c422b88b..c85bed2c8 100644 --- a/unoplat-code-confluence-query-engine/pyproject.toml +++ b/unoplat-code-confluence-query-engine/pyproject.toml @@ -53,7 +53,7 @@ markers = [ [tool.uv.sources] #NOTE: use it for local development to instantly see changes in the schema and its impact #unoplat-code-confluence-commons = { path = "../unoplat-code-confluence-commons" } -unoplat-code-confluence-commons = { git = "https://github.com/unoplat/unoplat-code-confluence.git", subdirectory = "unoplat-code-confluence-commons", rev = "unoplat-code-confluence-commons-v0.43.1" } +unoplat-code-confluence-commons = { git = "https://github.com/unoplat/unoplat-code-confluence.git", subdirectory = "unoplat-code-confluence-commons", rev = "unoplat-code-confluence-commons-v0.44.0" } [tool.basedpyright] typeCheckingMode = "strict" diff --git a/unoplat-code-confluence-query-engine/uv.lock b/unoplat-code-confluence-query-engine/uv.lock index 9e2703598..7c4199785 100644 --- a/unoplat-code-confluence-query-engine/uv.lock +++ b/unoplat-code-confluence-query-engine/uv.lock @@ -3362,8 +3362,8 @@ wheels = [ [[package]] name = "unoplat-code-confluence-commons" -version = "0.43.1" -source = { git = "https://github.com/unoplat/unoplat-code-confluence.git?subdirectory=unoplat-code-confluence-commons&rev=unoplat-code-confluence-commons-v0.43.1#a836fbc878cb7732c2e510c023491dd30c1a8297" } +version = "0.44.0" +source = { git = "https://github.com/unoplat/unoplat-code-confluence.git?subdirectory=unoplat-code-confluence-commons&rev=unoplat-code-confluence-commons-v0.44.0#08aec555de7778bf33a6774c4fa451109d8403d0" } dependencies = [ { name = "cryptography" }, { name = "pydantic" }, @@ -3375,7 +3375,7 @@ dependencies = [ [[package]] name = "unoplat-code-confluence-query-engine" -version = "0.31.1" +version = "0.33.0" source = { virtual = "." } dependencies = [ { name = "aiofiles" }, @@ -3430,7 +3430,7 @@ requires-dist = [ { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2.0.0" }, { name = "sqlmodel", specifier = ">=0.0.24" }, { name = "sse-starlette", specifier = ">=3.0.0" }, - { name = "unoplat-code-confluence-commons", git = "https://github.com/unoplat/unoplat-code-confluence.git?subdirectory=unoplat-code-confluence-commons&rev=unoplat-code-confluence-commons-v0.43.1" }, + { name = "unoplat-code-confluence-commons", git = "https://github.com/unoplat/unoplat-code-confluence.git?subdirectory=unoplat-code-confluence-commons&rev=unoplat-code-confluence-commons-v0.44.0" }, ] [package.metadata.requires-dev]