wip

Author: thetheodor

python_src/program_markers/instrumenter.py

@@ -1,13 +1,20 @@
 from __future__ import annotations
 
+import json
 from dataclasses import replace
 from enum import Enum
 from functools import cache
 from itertools import dropwhile, takewhile
 from pathlib import Path
 
-from diopter.compiler import ClangTool, ClangToolMode, CompilerExe, SourceProgram
-from program_markers.iprogram import InstrumentedProgram
+from diopter.compiler import (
+    ClangTool,
+    ClangToolMode,
+    CompilerExe,
+    SourceProgram,
+    SourceFile,
+)
+from program_markers.iprogram import InstrumentedProgram, InstrumentedFile
 from program_markers.markers import (
     DCEMarker,
     EnableEmitter,
@@ -306,3 +313,59 @@ def get_code_and_markers(mode: str) -> tuple[str, list[Marker]]:
 # (filename argument) plus an include file for the directives. It can use
 # the temp InstrumentedProgram to serialize everything and read it back in.
 # I can also add a debug mode that checks if the parsed is the same as the original.
+
+
+def instrument_file(
+    file: SourceFile,
+    output: Path,
+    ignore_functions_with_macros: bool = False,
+    mode: InstrumenterMode = InstrumenterMode.DCE,
+    instrumenter: ClangTool | None = None,
+    clang: CompilerExe | None = None,
+    timeout: int | None = None,
+) -> InstrumentedFile:
+    # TODO: operate directly on a SourceFile
+    program = SourceProgram(
+        language=file.language,
+        defined_macros=file.defined_macros,
+        include_paths=file.include_paths + (str(file.filename.parent.resolve()),),
+        system_include_paths=file.system_include_paths,
+        flags=file.flags,
+        code=open(file.filename, "r").read(),
+    )
+    iprogram = instrument_program(
+        program, ignore_functions_with_macros, mode, instrumenter, clang, timeout
+    )
+
+    output = output.resolve()
+    inc_file = output.with_suffix(".h").with_stem(output.stem + "_program_markers")
+    with open(output, "w") as f:
+        f.write(program.code)
+
+    with open(inc_file, "w") as f:
+        f.write(iprogram.generate_preprocessor_directives())
+
+    jprogram = iprogram.to_json_dict()
+    del jprogram["code"]
+    del jprogram["language"]
+    del jprogram["include_paths"]
+    del jprogram["system_include_paths"]
+    del jprogram["flags"]
+
+    j_file = output.with_suffix(".json")
+    with open(j_file, "w") as f:
+        json.dump(jprogram, f)
+
+    return InstrumentedFile(
+        marker_strategy=iprogram.marker_strategy,
+        markers=iprogram.markers,
+        directive_emitters=iprogram.directive_emitters,
+        directives_include_file=inc_file,
+        directives_json_file=j_file,
+        filename=output,
+        language=program.language,
+        defined_macros=program.defined_macros,
+        include_paths=program.include_paths,
+        system_include_paths=program.system_include_paths,
+        flags=program.flags + (f"-include {str(inc_file)}",),
+    )

python_src/program_markers/iprogram.py

@@ -1,18 +1,23 @@
 from __future__ import annotations
 
+import json
 import re
+from abc import ABC, abstractmethod
 from collections import defaultdict
 from dataclasses import dataclass, replace
 from typing import Any, Sequence
+from pathlib import Path
 
 from diopter.compiler import (
+    Source,
     ASMCompilationOutput,
     CompilationOutputType,
     CompilationResult,
     CompilationSetting,
     ExeCompilationOutput,
     Language,
     SourceProgram,
+    SourceFile,
 )
 from program_markers.markers import (
     AbortEmitter,
@@ -113,11 +118,15 @@ def collect_markers(
 
 
 @dataclass(frozen=True, kw_only=True)
-class InstrumentedProgram(SourceProgram):
+class InstrumentedSourceMixin(ABC):
     marker_strategy: MarkerDetectionStrategy
     markers: tuple[Marker, ...]
     directive_emitters: dict[Marker, MarkerDirectiveEmitter]
 
+    @abstractmethod
+    def self_as_source(self) -> Source:
+        raise NotImplementedError
+
     def __post_init__(self) -> None:
         # All markers ids are unique
         marker_ids = set()
@@ -162,21 +171,14 @@ def generate_preprocessor_directives(self) -> str:
             for marker, emitter in self.directive_emitters.items()
         )
 
-    def get_modified_code(self) -> str:
-        """Returns the necessary preprocessor directives for markers + self.code.
-
-        Only directives for the enabled, disabled and made unreachable markers
-        are added.
-
-        If any markers have not been enabled, disabled, or made unreachable,
-        then the code not compilable but it can be preprocessed.
+    def process_tracking_reachable_markers_output(
+        self, output: str
+    ) -> tuple[Marker, ...]:
+        return tuple(
+            marker for marker in self.enabled_markers() if marker.name in output
+        )
 
-        Returns:
-            str:
-                the source code including the necessary preprocessor directives
-        """
 
-        return self.generate_preprocessor_directives() + "\n" + self.code
 
     def find_non_eliminated_markers(
         self, compilation_setting: CompilationSetting
@@ -197,7 +199,7 @@ def find_non_eliminated_markers(
                 The non_eliminated markers for the given compilation setting.
         """
         asm = compilation_setting.compile_program(
-            self, ASMCompilationOutput()
+            self.self_as_source(), ASMCompilationOutput()
         ).output.read()
         non_eliminated_markers = find_non_eliminated_markers_impl(
             asm, self.enabled_markers(), self.marker_strategy
@@ -228,6 +230,32 @@ def find_eliminated_markers(
             eliminated_markers = eliminated_markers & set(self.enabled_markers())
         return tuple(eliminated_markers)
 
+
+@dataclass(frozen=True, kw_only=True)
+class InstrumentedProgram(SourceProgram, InstrumentedSourceMixin):
+    marker_strategy: MarkerDetectionStrategy
+    markers: tuple[Marker, ...]
+    directive_emitters: dict[Marker, MarkerDirectiveEmitter]
+
+    def self_as_source(self) -> SourceProgram:
+        return self
+
+    def get_modified_code(self) -> str:
+        """Returns the necessary preprocessor directives for markers + self.code.
+
+        Only directives for the enabled, disabled and made unreachable markers
+        are added.
+
+        If any markers have not been enabled, disabled, or made unreachable,
+        then the code not compilable but it can be preprocessed.
+
+        Returns:
+            str:
+                the source code including the necessary preprocessor directives
+        """
+
+        return self.generate_preprocessor_directives() + "\n" + self.code
+
     def replace_markers(self, new_markers: tuple[Marker, ...]) -> InstrumentedProgram:
         """Replaces the markers in the program with the new ones.
         Each original marker whose id matches one of the new ones is replaced,
@@ -275,13 +303,6 @@ def compile_program_for_tracking(
         tracked_program = replace(self, directive_emitters=new_emitters)
         return setting.compile_program(tracked_program, output, timeout=timeout)
 
-    def process_tracking_reachable_markers_output(
-        self, output: str
-    ) -> tuple[Marker, ...]:
-        return tuple(
-            marker for marker in self.enabled_markers() if marker.name in output
-        )
-
     def track_reachable_markers(
         self,
         args: tuple[str, ...],
@@ -667,3 +688,100 @@ def from_json_dict_impl(
             markers=markers,
             directive_emitters=directive_emitters,
         )
+
+
+@dataclass(frozen=True, kw_only=True)
+class InstrumentedFile(SourceFile, InstrumentedSourceMixin):
+    marker_strategy: MarkerDetectionStrategy
+    markers: tuple[Marker, ...]
+    directive_emitters: dict[Marker, MarkerDirectiveEmitter]
+    directives_include_file: Path  # this should be included
+    directives_json_file: Path
+    debug: bool = True
+
+    def self_as_source(self) -> SourceFile:
+        return self
+
+    def __post_init__(self) -> None:
+        assert self.directives_include_file.is_absolute()
+        assert self.directives_json_file.is_absolute()
+        assert self.directives_include_file.exists()
+        assert self.directives_json_file.exists()
+        assert f"-include {str(self.directives_include_file)}" in self.flags, self.flags
+
+        if self.debug:
+            with open(self.directives_json_file) as f:
+                j = json.load(f)
+            j["strategy"] = self.marker_strategy.to_json_dict()
+            directive_emitters = {
+                Marker.from_json_dict(m): MarkerDirectiveEmitter.from_json_dict(
+                    directive
+                )
+                for m, directive in j["directive_emitters"]
+            }
+            assert directive_emitters == self.directive_emitters, (directive_emitters, self.directive_emitters)
+
+            markers = tuple(Marker.from_json_dict(m) for m in j["markers"])
+            assert markers == self.markers
+            with open(self.directives_include_file) as f:
+                assert f.read() == self.generate_preprocessor_directives()
+
+    def generate_tracking(self) -> InstrumentedFile:
+        new_emitters = self.directive_emitters.copy()
+        tracked_markers = self.enabled_markers()
+        te = TrackingEmitter()
+        for marker in tracked_markers:
+            new_emitters[marker] = te
+
+        new_stem = self.directives_include_file.stem + "_tracking"
+        inc_file = self.directives_include_file.with_stem(new_stem)
+        with open(inc_file, "w") as f:
+            f.write(
+                "\n".join(
+                    emitter.emit_directive(marker)
+                    for marker, emitter in new_emitters.items()
+                )
+            )
+
+        json_file = self.directives_json_file.with_stem(new_stem)
+
+        with open(json_file, "w") as f:
+            json.dump(
+                {
+                    "marker_strategy": self.marker_strategy.to_json_dict(),
+                    "directive_emitters": [
+                        (m.to_json_dict(), e.to_json_dict())
+                        for m, e in new_emitters.items()
+                    ],
+                    "markers": [m.to_json_dict() for m in self.markers],
+                },
+                f,
+            )
+
+        flags = tuple(
+            f
+            for f in self.flags
+            if f != f"-include {str(self.directives_include_file)}"
+        ) + (f"-include {str(inc_file)}",)
+
+        return replace(
+            self,
+            directive_emitters=new_emitters,
+            flags=flags,
+            directives_include_file=inc_file,
+            directives_json_file=json_file,
+        )
+
+    def generate_tracking_for_refinement_version(self) -> InstrumentedFile:
+        pass
+
+    def refine_markers(self, markers: tuple[Marker, ...]) -> InstrumentedFile:
+        pass
+
+    def make_markers_unreachable(self, markers: tuple[Marker, ...]) -> InstrumentedFile:
+        pass
+
+    def disable_remaining_markers(
+        self, do_not_disable: Sequence[Marker] = tuple()
+    ) -> InstrumentedFile:
+        pass

python_src/program_markers/markers.py

@@ -272,6 +272,7 @@ def from_json_dict(j: dict[str, Any]) -> VRMarker:
 MarkerTypes = (DCEMarker, VRMarker)
 
 
+@dataclass
 class MarkerDirectiveEmitter(ABC):
     def emit_directive(self, marker: Marker) -> str:
         raise NotImplementedError
@@ -316,19 +317,20 @@ def to_json_dict(self) -> dict[str, Any]:
         return j
 
 
+@dataclass
 class NoEmitter(MarkerDirectiveEmitter):
     def emit_directive(self, marker: Marker) -> str:
         return ""
 
 
+@dataclass
 class EnableEmitter(MarkerDirectiveEmitter):
-    def __init__(self, strategy: MarkerDetectionStrategy):
-        self.strategy = strategy
+    strategy: MarkerDetectionStrategy
 
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, MarkerDirectiveEmitter):
-            return NotImplemented
-        return isinstance(other, EnableEmitter) and self.strategy == other.strategy
+    # def __eq__(self, other: object) -> bool:
+        # if not isinstance(other, MarkerDirectiveEmitter):
+            # return NotImplemented
+        # return isinstance(other, EnableEmitter) and self.strategy == other.strategy
 
     def emit_directive(self, marker: Marker) -> str:
         return f"""{self.strategy.definitions_and_declarations(marker)}
@@ -339,18 +341,21 @@ def emit_directive(self, marker: Marker) -> str:
                 """
 
 
+@dataclass
 class DisableEmitter(MarkerDirectiveEmitter):
     def emit_directive(self, marker: Marker) -> str:
         return f"#define {marker.macro()} ;"
 
 
+@dataclass
 class UnreachableEmitter(MarkerDirectiveEmitter):
     def emit_directive(self, marker: Marker) -> str:
         return f"""#define {marker.macro()} \
                 {marker.marker_statement_prefix()}__builtin_unreachable();{marker.marker_statement_postfix()}
                 """
 
 
+@dataclass
 class AbortEmitter(MarkerDirectiveEmitter):
     def emit_directive(self, marker: Marker) -> str:
         return f"""#define {marker.macro()} \
@@ -361,6 +366,7 @@ def emit_directive(self, marker: Marker) -> str:
                 """
 
 
+@dataclass
 class TrackingEmitter(MarkerDirectiveEmitter):
     def emit_directive(self, marker: Marker) -> str:
         return f""" int {marker.name}_ENCOUNTERED = 0;
@@ -377,6 +383,7 @@ def emit_directive(self, marker: Marker) -> str:
                 """
 
 
+@dataclass
 class TrackingForRefinementEmitter(MarkerDirectiveEmitter):
     def emit_directive(self, marker: Marker) -> str:
         match marker: