Refactor scheduler and add tests for ComplieSim deployment

src/dvsim/flow/base.py

@@ -422,7 +422,7 @@ def deploy_objects(self) -> Mapping[str, CompletedJobStatus]:
                 json.dumps(
                     # Sort on full name to ensure consistent ordering
                     sorted(
-                        [d.model_dump() for d in deploy],
+                        [d.dump() for d in deploy],
                         key=lambda d: d["full_name"],
                     ),
                     indent=2,
@@ -445,16 +445,19 @@ def _gen_results(self, results: Mapping[str, CompletedJobStatus]) -> str:
         prints the full list of failures for debug / triage to the final
         report, which is in markdown format.
 
-        Results:
-            dictionary mapping deployed item names to job status.
+        Args:
+            results: dictionary mapping deployed item names to completed job status.
+
+        Returns:
+            Results as a formatted string
 
         """
 
     def gen_results(self, results: Mapping[str, CompletedJobStatus]) -> None:
-        """Public facing API for _gen_results().
+        """Generate flow results.
 
         Args:
-            results: dictionary mapping deployed item names to job status.
+            results: dictionary mapping deployed item names to completed job status.
 
         """
         for item in self.cfgs:

src/dvsim/job/deploy.py

@@ -11,6 +11,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, ClassVar
 
+from pydantic import BaseModel
+from pydantic.config import ConfigDict
 from tabulate import tabulate
 
 from dvsim.job.time import JobTime
@@ -26,6 +28,21 @@
 
 if TYPE_CHECKING:
     from dvsim.flow.sim import SimCfg
+    from dvsim.modes import BuildMode
+
+
+class WorkspaceConfig(BaseModel):
+    """Workspace configuration."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    project: str
+    timestamp: str
+
+    project_root: Path
+    scratch_root: Path
+    scratch_path: Path
+
 
 __all__ = (
     "CompileSim",
@@ -101,6 +118,14 @@ def __init__(self, sim_cfg: "SimCfg") -> None:
         # Job's wall clock time (a.k.a CPU time, or runtime).
         self.job_runtime = JobTime()
 
+        self.workspace_cfg = WorkspaceConfig(
+            project=sim_cfg.name,
+            project_root=sim_cfg.proj_root,
+            scratch_root=Path(sim_cfg.scratch_root),
+            scratch_path=Path(sim_cfg.scratch_path),
+            timestamp=sim_cfg.args.timestamp,
+        )
+
     def _define_attrs(self) -> None:
         """Define the attributes this instance needs to have.
 
@@ -293,7 +318,7 @@ def pre_launch(self, launcher: Launcher) -> None:
         This is invoked by launcher::_pre_launch().
         """
 
-    def post_finish(self, status) -> None:
+    def post_finish(self, status: str) -> None:
         """Perform additional post-finish activities (callback).
 
         This is invoked by launcher::_post_finish().
@@ -328,12 +353,9 @@ def extract_info_from_log(self, job_runtime_secs: int, log_text: list) -> None:
             log.warning(f"{self.full_name}: {e} Using dvsim-maintained job_runtime instead.")
             self.job_runtime.set(job_runtime_secs, "s")
 
-    def model_dump(self) -> Mapping:
+    def dump(self) -> Mapping:
         """Dump the deployment object to mapping object.
 
-        This method matches the interface provided by pydantic models to dump a
-        subset of the class attributes
-
         Returns:
             Representation of a deployment object as a dict.
 
@@ -371,6 +393,20 @@ def __init__(self, build_mode, sim_cfg) -> None:
                 self.build_timeout_mins,
             )
 
+    @staticmethod
+    def new(build_mode_obj: "BuildMode", sim_cfg: "SimCfg") -> "CompileSim":
+        """Create a new CompileSim object.
+
+        Args:
+            build_mode_obj: build mode instance
+            sim_cfg: Simulation config object
+
+        Returns:
+            new CompileSim object.
+
+        """
+        return CompileSim(build_mode=build_mode_obj, sim_cfg=sim_cfg)
+
     def _define_attrs(self) -> None:
         """Define attributes."""
         super()._define_attrs()
@@ -711,6 +747,10 @@ def __init__(self, run_items, sim_cfg) -> None:
                 else:
                     self.cov_db_dirs.append(run.cov_db_dir)
 
+        # Sort the cov_db_dir except for the first directory
+        if len(self.cov_db_dirs) > 1:
+            self.cov_db_dirs = [self.cov_db_dirs[0], *sorted(self.cov_db_dirs[1:])]
+
         # Early lookup the cov_merge_db_dir, which is a mandatory misc
         # attribute anyway. We need it to compute additional cov db dirs.
         self.cov_merge_db_dir = subst_wildcards("{cov_merge_db_dir}", sim_cfg.__dict__)

src/dvsim/launcher/base.py

@@ -19,7 +19,7 @@
 from dvsim.utils import clean_odirs, mk_symlink, rm_path
 
 if TYPE_CHECKING:
-    from dvsim.job.deploy import Deploy
+    from dvsim.job.deploy import Deploy, WorkspaceConfig
 
 
 class LauncherError(Exception):
@@ -67,7 +67,7 @@ class Launcher(ABC):
     poll_freq = 1
 
     # Points to the python virtual env area.
-    pyvenv = None
+    pyvenv: Path | None = None
 
     # If a history of previous invocations is to be maintained, then keep no
     # more than this many directories.
@@ -97,18 +97,18 @@ def __init__(self, deploy: "Deploy") -> None:
             deploy: deployment object that will be launched.
 
         """
-        cfg = deploy.sim_cfg
+        workspace_cfg = deploy.workspace_cfg
 
         # One-time preparation of the workspace.
         if not Launcher.workspace_prepared:
-            # TODO: CLI args should be processed far earlier than this
-            self.prepare_workspace(cfg.project, cfg.proj_root, cfg.args)
+            self.prepare_workspace(workspace_cfg)
             Launcher.workspace_prepared = True
 
         # One-time preparation of the workspace, specific to the cfg.
-        if cfg not in Launcher.workspace_prepared_for_cfg:
-            self.prepare_workspace_for_cfg(cfg)
-            Launcher.workspace_prepared_for_cfg.add(cfg)
+        project = workspace_cfg.project
+        if project not in Launcher.workspace_prepared_for_cfg:
+            self.prepare_workspace_for_cfg(workspace_cfg)
+            Launcher.workspace_prepared_for_cfg.add(project)
 
         # Store the deploy object handle.
         self.deploy = deploy
@@ -155,34 +155,40 @@ def set_pyvenv(project: str) -> None:
         # The code below allows each launcher variant to set its own virtualenv
         # because the loading / activating mechanism could be different between
         # them.
-        Launcher.pyvenv = os.environ.get(
-            f"{project.upper()}_PYVENV_{Launcher.variant.upper()}",
-        )
+        common_venv = f"{project.upper()}_PYVENV"
+        variant = Launcher.variant.upper()
+
+        venv_path = os.environ.get(f"{common_venv}_{variant}")
+
+        if not venv_path:
+            venv_path = os.environ.get(common_venv)
 
-        if not Launcher.pyvenv:
-            Launcher.pyvenv = os.environ.get(f"{project.upper()}_PYVENV")
+        if venv_path:
+            Launcher.pyvenv = Path(venv_path)
 
     @staticmethod
     @abstractmethod
-    def prepare_workspace(project: str, repo_top: str, args: Mapping) -> None:
+    def prepare_workspace(cfg: "WorkspaceConfig") -> None:
         """Prepare the workspace based on the chosen launcher's needs.
 
         This is done once for the entire duration for the flow run.
 
         Args:
-            project: the name of the project.
-            repo_top: the path to the repository.
-            args: command line args passed to dvsim.
+            cfg: workspace configuration
 
         """
 
     @staticmethod
     @abstractmethod
-    def prepare_workspace_for_cfg(cfg: Mapping) -> None:
+    def prepare_workspace_for_cfg(cfg: "WorkspaceConfig") -> None:
         """Prepare the workspace for a cfg.
 
         This is invoked once for each cfg.
         'cfg' is the flow configuration object.
+
+        Args:
+            cfg: workspace configuration
+
         """
 
     def __str__(self) -> str:
@@ -250,6 +256,10 @@ def poll(self) -> str | None:
         """Poll the launched job for completion.
 
         Invokes _check_status() and _post_finish() when the job completes.
+
+        Returns:
+            status of the job or None
+
         """
 
     @abstractmethod

src/dvsim/launcher/fake.py

@@ -4,14 +4,14 @@
 
 """Fake Launcher that returns random results."""
 
-from collections.abc import Mapping
 from random import choice, random
 from typing import TYPE_CHECKING
 
 from dvsim.launcher.base import ErrorMessage, Launcher
 
 if TYPE_CHECKING:
-    from dvsim.job.deploy import CovReport, Deploy, RunTest
+    from dvsim.job.deploy import CovReport, Deploy, RunTest, WorkspaceConfig
+
 
 __all__ = ("FakeLauncher",)
 
@@ -75,22 +75,23 @@ def kill(self) -> None:
         )
 
     @staticmethod
-    def prepare_workspace(project: str, repo_top: str, args: Mapping) -> None:
+    def prepare_workspace(cfg: "WorkspaceConfig") -> None:
         """Prepare the workspace based on the chosen launcher's needs.
 
         This is done once for the entire duration for the flow run.
 
         Args:
-            project: the name of the project.
-            repo_top: the path to the repository.
-            args: command line args passed to dvsim.
+            cfg: workspace configuration
 
         """
 
     @staticmethod
-    def prepare_workspace_for_cfg(cfg: Mapping) -> None:
+    def prepare_workspace_for_cfg(cfg: "WorkspaceConfig") -> None:
         """Prepare the workspace for a cfg.
 
         This is invoked once for each cfg.
-        'cfg' is the flow configuration object.
+
+        Args:
+            cfg: workspace configuration
+
         """

src/dvsim/launcher/local.py

@@ -8,14 +8,13 @@
 import os
 import shlex
 import subprocess
-from collections.abc import Mapping
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from dvsim.launcher.base import ErrorMessage, Launcher, LauncherBusyError, LauncherError
 
 if TYPE_CHECKING:
-    from dvsim.job.deploy import Deploy
+    from dvsim.job.deploy import Deploy, WorkspaceConfig
 
 
 class LocalLauncher(Launcher):
@@ -184,22 +183,23 @@ def _close_job_log_file(self) -> None:
             self._log_file.close()
 
     @staticmethod
-    def prepare_workspace(project: str, repo_top: str, args: Mapping) -> None:
+    def prepare_workspace(cfg: "WorkspaceConfig") -> None:
         """Prepare the workspace based on the chosen launcher's needs.
 
         This is done once for the entire duration for the flow run.
 
         Args:
-            project: the name of the project.
-            repo_top: the path to the repository.
-            args: command line args passed to dvsim.
+            cfg: workspace configuration
 
         """
 
     @staticmethod
-    def prepare_workspace_for_cfg(cfg: Mapping) -> None:
+    def prepare_workspace_for_cfg(cfg: "WorkspaceConfig") -> None:
         """Prepare the workspace for a cfg.
 
         This is invoked once for each cfg.
-        'cfg' is the flow configuration object.
+
+        Args:
+            cfg: workspace configuration
+
         """