feat(parsers): extract access modifiers and decorators via highlights.scm

codebase_rag/constants/ast_nodes.py

@@ -112,13 +112,46 @@
 QUERY_LOCALS = "locals"
 QUERY_CONFIG = "config"
 QUERY_LANGUAGE = "language"
+QUERY_HIGHLIGHTS = "highlights"
 
 # (H) Query capture names
 CAPTURE_FUNCTION = "function"
 CAPTURE_CLASS = "class"
 CAPTURE_CALL = "call"
 CAPTURE_IMPORT = "import"
 CAPTURE_IMPORT_FROM = "import_from"
+CAPTURE_KEYWORD_MODIFIER = "keyword.modifier"
+CAPTURE_KEYWORD = "keyword"
+CAPTURE_ATTRIBUTE = "attribute"
+CAPTURE_FUNCTION_DECORATOR = "function.decorator"
+
+# (H) Modifier extraction
+EXCLUDED_KEYWORDS = frozenset(
+    {
+        "def",
+        "class",
+        "fn",
+        "struct",
+        "impl",
+        "interface",
+        "enum",
+        "function",
+        "trait",
+        "type",
+        "void",
+        "None",
+        "True",
+        "False",
+        "null",
+        "true",
+        "false",
+        "return",
+        "import",
+        "from",
+        "as",
+        "where",
+    }
+)
 
 # (H) Tree-sitter Python import node types
 TS_IMPORT_STATEMENT = "import_statement"

codebase_rag/constants/graph.py

@@ -187,6 +187,7 @@ class AuditCheck(StrEnum):
 # (H) Property keys
 KEY_PARAMETERS = "parameters"
 KEY_DECORATORS = "decorators"
+KEY_MODIFIERS = "modifiers"
 KEY_DOCSTRING = "docstring"
 KEY_IS_EXPORTED = "is_exported"
 # (H) Marks a method that overrides a method of an EXTERNAL stdlib base class

codebase_rag/parser_loader.py

@@ -244,6 +244,48 @@ def _create_locals_query(
         return None
 
 
+def _create_highlights_query(
+    language: Language, lang_name: cs.SupportedLanguage
+) -> Query | None:
+    query_str = ""
+
+    # (H) TSX shares the TypeScript grammar for highlights
+    query_lang_name = (
+        cs.SupportedLanguage.TS if lang_name == cs.SupportedLanguage.TSX else lang_name
+    )
+
+    try:
+        module_name = (
+            f"{cs.TREE_SITTER_MODULE_PREFIX}{query_lang_name.replace('-', '_')}"
+        )
+        module = importlib.import_module(module_name)
+        if hasattr(module, "HIGHLIGHTS_QUERY"):
+            query_str = module.HIGHLIGHTS_QUERY
+    except Exception as e:
+        logger.debug(
+            f"Failed to load standard highlights query for {query_lang_name}: {e}"
+        )
+
+    try:
+        fallback_path = (
+            Path(__file__).parent / "queries" / "highlights" / f"{query_lang_name}.scm"
+        )
+        if fallback_path.exists():
+            custom_queries = fallback_path.read_text(encoding="utf-8")
+            query_str = (
+                query_str + "\n" + custom_queries if query_str else custom_queries
+            )
+
+        if query_str:
+            return Query(language, query_str)
+    except Exception as e:
+        logger.debug(
+            f"Failed to load fallback highlights query for {query_lang_name}: {e}"
+        )
+
+    return None
+
+
 COMBINED_FUNC_CLASS_QUERIES: dict[cs.SupportedLanguage, Query | None] = {}
 COMBINED_FUNC_CLASS_IMPORT_QUERIES: dict[cs.SupportedLanguage, Query | None] = {}
 
@@ -287,6 +329,7 @@ def _create_language_queries(
         calls=_create_optional_query(language, call_patterns),
         imports=_create_optional_query(language, combined_import_patterns),
         locals=_create_locals_query(language, lang_name),
+        highlights=_create_highlights_query(language, lang_name),
         config=lang_config,
         language=language,
         parser=parser,

codebase_rag/parsers/class_ingest/mixin.py

@@ -31,6 +31,7 @@
 from ..rs import RustTypeInferenceEngine
 from ..rs import utils as rs_utils
 from ..utils import (
+    extract_modifiers_and_decorators,
     function_span_key,
     ingest_method,
     record_cpp_definition_span,
@@ -686,10 +687,15 @@ def _process_class_node(
         )
         node_type = nt.determine_node_type(class_node, class_name, class_qn, language)
 
+        modifiers, decorators = extract_modifiers_and_decorators(
+            class_node, lang_queries
+        )
+
         class_props: PropertyDict = {
             cs.KEY_QUALIFIED_NAME: class_qn,
             cs.KEY_NAME: class_name,
-            cs.KEY_DECORATORS: self._extract_decorators(class_node),
+            cs.KEY_MODIFIERS: modifiers,
+            cs.KEY_DECORATORS: decorators,
             cs.KEY_START_LINE: class_node.start_point[0] + 1,
             cs.KEY_END_LINE: class_node.end_point[0] + 1,
             cs.KEY_DOCSTRING: self._get_docstring(class_node),
@@ -956,8 +962,8 @@ def _ingest_class_methods(
                 self.simple_name_lookup,
                 self._get_docstring,
                 language,
-                self._extract_decorators,
-                method_qualified_name,
+                lang_queries=lang_queries,
+                method_qualified_name=method_qualified_name,
                 file_path=file_path,
                 repo_path=self.repo_path,
                 external_override_names=external_override_names,

codebase_rag/parsers/cpp_frontend/frontend.py

@@ -115,6 +115,7 @@ def _node_props(self, cursor: Cursor, qn: str, name: str, rel: str) -> PropertyD
         return {
             cs.KEY_QUALIFIED_NAME: qn,
             cs.KEY_NAME: name,
+            cs.KEY_MODIFIERS: [],
             cs.KEY_DECORATORS: [],
             cs.KEY_START_LINE: cursor.location.line,
             cs.KEY_END_LINE: cursor.extent.end.line,

codebase_rag/parsers/function_ingest.py

@@ -30,6 +30,7 @@
 from .rs import utils as rs_utils
 from .utils import (
     callable_parameter_indices,
+    extract_modifiers_and_decorators,
     function_span_key,
     get_function_captures,
     ingest_method,
@@ -95,6 +96,7 @@ class _DeferredJsAnonymous(NamedTuple):
     module_qn: str
     language: cs.SupportedLanguage
     lang_config: LanguageSpec
+    lang_queries: LanguageQueries
 
 
 class _DeferredMethod(NamedTuple):
@@ -115,6 +117,7 @@ class resolves scope-first: two same-leaf classes (ast::Type and
     start_line: int
     start_col: int
     end_line: int
+    lang_queries: LanguageQueries | None = None
 
 
 class _DeferredGoMethod(NamedTuple):
@@ -124,6 +127,7 @@ class _DeferredGoMethod(NamedTuple):
     module_qn: str
     receiver_type: str
     file_path: Path | None
+    lang_queries: LanguageQueries | None = None
 
 
 class _DeferredCppContainment(NamedTuple):
@@ -177,9 +181,6 @@ class FunctionIngestMixin:
     @abstractmethod
     def _get_docstring(self, node: ASTNode) -> str | None: ...
 
-    @abstractmethod
-    def _extract_decorators(self, node: ASTNode) -> list[str]: ...
-
     def _ingest_all_functions(
         self,
         root_node: Node,
@@ -197,6 +198,7 @@ def _ingest_all_functions(
             if not result:
                 return
             lang_config, captures = result
+            lang_queries = queries[language]
         file_path = self.module_qn_to_file_path.get(module_qn)
         has_classes = bool(captures.get(cs.CAPTURE_CLASS))
 
@@ -205,7 +207,9 @@ def _ingest_all_functions(
                 continue
 
             if language == cs.SupportedLanguage.CPP:
-                if self._handle_cpp_out_of_class_method(func_node, module_qn):
+                if self._handle_cpp_out_of_class_method(
+                    func_node, module_qn, lang_queries
+                ):
                     continue
                 # (H) The query captures a templated function twice: the
                 # (H) template_declaration wrapper AND its inner definition.
@@ -220,7 +224,7 @@ def _ingest_all_functions(
                     continue
 
             if language == cs.SupportedLanguage.GO and self._defer_go_receiver_method(
-                func_node, module_qn
+                func_node, module_qn, lang_queries
             ):
                 continue
 
@@ -238,13 +242,18 @@ def _ingest_all_functions(
             if language in cs.JS_TS_LANGUAGES and resolution.is_anonymous:
                 self._deferred_js_anonymous.append(
                     _DeferredJsAnonymous(
-                        func_node, resolution, module_qn, language, lang_config
+                        func_node,
+                        resolution,
+                        module_qn,
+                        language,
+                        lang_config,
+                        lang_queries,
                     )
                 )
                 continue
 
             self._register_function(
-                func_node, resolution, module_qn, language, lang_config
+                func_node, resolution, module_qn, language, lang_config, lang_queries
             )
 
             # (H) Record a free C++ function's return type so a chained call off a
@@ -310,6 +319,7 @@ def _flush_deferred_js_anonymous(self) -> None:
                 entry.module_qn,
                 entry.language,
                 entry.lang_config,
+                entry.lang_queries,
             )
 
     def _resolve_function_identity(
@@ -431,7 +441,12 @@ def _is_cpp_defined(self, qn: str) -> bool:
             return suffix in cs.CPP_EXTENSIONS or suffix in cs.C_EXTENSIONS
         return False
 
-    def _handle_cpp_out_of_class_method(self, func_node: Node, module_qn: str) -> bool:
+    def _handle_cpp_out_of_class_method(
+        self,
+        func_node: Node,
+        module_qn: str,
+        lang_queries: LanguageQueries | None = None,
+    ) -> bool:
         if not cpp_utils.is_out_of_class_method_definition(func_node):
             return False
 
@@ -471,7 +486,7 @@ def _handle_cpp_out_of_class_method(self, func_node: Node, module_qn: str) -> bo
                 simple_name_lookup=self.simple_name_lookup,
                 get_docstring_func=self._get_docstring,
                 language=cs.SupportedLanguage.CPP,
-                extract_decorators_func=self._extract_decorators,
+                lang_queries=lang_queries,
                 file_path=file_path,
                 repo_path=self.repo_path,
             )
@@ -508,9 +523,15 @@ def _handle_cpp_out_of_class_method(self, func_node: Node, module_qn: str) -> bo
             method_name = cpp_utils.extract_function_name(func_node)
             if not method_name:
                 return True
-            decorators = self._extract_decorators(func_node)
+            decorators = []
+            modifiers = []
+            if lang_queries:
+                modifiers, decorators = extract_modifiers_and_decorators(
+                    func_node, lang_queries
+                )
             props: PropertyDict = {
                 cs.KEY_NAME: method_name,
+                cs.KEY_MODIFIERS: modifiers,
                 cs.KEY_DECORATORS: decorators,
                 cs.KEY_START_LINE: func_node.start_point[0] + 1,
                 cs.KEY_END_LINE: func_node.end_point[0] + 1,
@@ -537,6 +558,7 @@ def _handle_cpp_out_of_class_method(self, func_node: Node, module_qn: str) -> bo
                     start_line=func_node.start_point[0] + 1,
                     start_col=func_node.start_point[1],
                     end_line=func_node.end_point[0] + 1,
+                    lang_queries=lang_queries,
                 )
             )
 
@@ -625,7 +647,12 @@ def resolve_deferred_cpp_methods(self) -> int:
         self._deferred_cpp_methods = []
         return ingested
 
-    def _defer_go_receiver_method(self, func_node: Node, module_qn: str) -> bool:
+    def _defer_go_receiver_method(
+        self,
+        func_node: Node,
+        module_qn: str,
+        lang_queries: LanguageQueries | None = None,
+    ) -> bool:
         if not go_utils.is_receiver_method(func_node):
             return False
         receiver_type = go_utils.extract_receiver_type_name(func_node)
@@ -774,14 +801,17 @@ def _register_function(
         module_qn: str,
         language: cs.SupportedLanguage,
         lang_config: LanguageSpec,
+        lang_queries: LanguageQueries,
     ) -> None:
         unique_qn = self.function_registry.register_unique_qn(
             resolution.qualified_name, func_node.start_point[0] + 1
         )
         if unique_qn != resolution.qualified_name:
             resolution = resolution._replace(qualified_name=unique_qn)
 
-        func_props = self._build_function_props(func_node, resolution, module_qn)
+        func_props = self._build_function_props(
+            func_node, resolution, module_qn, lang_queries
+        )
         is_macro = func_node.type == cs.TS_RS_MACRO_DEFINITION
         if is_macro:
             # (H) Rust macros live in a separate namespace from functions;
@@ -863,13 +893,21 @@ def _record_cpp_template_child_location(
                 break
 
     def _build_function_props(
-        self, func_node: Node, resolution: FunctionResolution, module_qn: str
+        self,
+        func_node: Node,
+        resolution: FunctionResolution,
+        module_qn: str,
+        lang_queries: LanguageQueries,
     ) -> PropertyDict:
         file_path = self.module_qn_to_file_path.get(module_qn)
+        modifiers, decorators = extract_modifiers_and_decorators(
+            func_node, lang_queries
+        )
         props: PropertyDict = {
             cs.KEY_QUALIFIED_NAME: resolution.qualified_name,
             cs.KEY_NAME: resolution.name,
-            cs.KEY_DECORATORS: self._extract_decorators(func_node),
+            cs.KEY_MODIFIERS: modifiers,
+            cs.KEY_DECORATORS: decorators,
             cs.KEY_START_LINE: func_node.start_point[0] + 1,
             cs.KEY_END_LINE: func_node.end_point[0] + 1,
             cs.KEY_DOCSTRING: self._get_docstring(func_node),