Release 2.1.1

looper/default_config/divvy_templates/slurm_singularity_template.sub → divvy_templates/bulker_slurm_template.sub

@@ -11,7 +11,10 @@
 echo 'Compute node:' `hostname`
 echo 'Start time:' `date +'%Y-%m-%d %T'`
 
-singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image
-srun singularity exec instance://{JOBNAME}_image {CODE}
+{PRE_COMMAND}
 
-singularity instance.stop {JOBNAME}_image
+eval "$(bulker activate -e {BULKER_CRATE})"
+
+{CODE}
+
+{POST_COMMAND}

divvy_templates/localhost_apptainer_template.sub

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo 'Compute node:' `hostname`
+echo 'Start time:' `date +'%Y-%m-%d %T'`
+
+{PRE_COMMAND}
+
+{
+apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image
+apptainer exec instance://{JOBNAME}_image {CODE}
+} | tee {LOGFILE} --ignore-interrupts
+
+{POST_COMMAND}

divvy_templates/localhost_bulker_template.sub

@@ -3,8 +3,12 @@
 echo 'Compute node:' `hostname`
 echo 'Start time:' `date +'%Y-%m-%d %T'`
 
+{PRE_COMMAND}
+
 eval "$(bulker activate -e {BULKER_CRATE})"
 
 {
   {CODE}
 } | tee {LOGFILE} -i
+
+{POST_COMMAND}

divvy_templates/localhost_docker_template.sub

@@ -3,6 +3,10 @@
 echo 'Compute node:' `hostname`
 echo 'Start time:' `date +'%Y-%m-%d %T'`
 
+{PRE_COMMAND}
+
 {
 docker run --rm -it {DOCKER_ARGS} {DOCKER_IMAGE} {CODE}
 } | tee {LOGFILE} --ignore-interrupts
+
+{POST_COMMAND}

divvy_templates/localhost_template.sub

@@ -3,6 +3,10 @@
 echo 'Compute node:' `hostname`
 echo 'Start time:' `date +'%Y-%m-%d %T'`
 
+{PRE_COMMAND}
+
 {
 {CODE}
 } | tee {LOGFILE}
+
+{POST_COMMAND}

divvy_templates/lsf_template.sub

@@ -1,4 +1,7 @@
 #!/bin/bash
 
-bsub -n{CORES} -W {TIME} -R \"rusage[mem={MEM}]\" -o {LOGFILE} {CODE}
+{PRE_COMMAND}
 
+bsub -n{CORES} -W {TIME} -R "rusage[mem={MEM}]" -o {LOGFILE} {CODE}
+
+{POST_COMMAND}

divvy_templates/sge_template.sub

@@ -1 +1,15 @@
-This has not been implemented, but you could add whatever cluster submission systems here, just use the slurm_template as an example.
+#!/bin/bash
+#$ -N '{JOBNAME}'
+#$ -o '{LOGFILE}'
+#$ -l mem_free={MEM}
+#$ -pe smp {CORES}
+#$ -l h_rt={TIME}
+
+echo 'Compute node:' `hostname`
+echo 'Start time:' `date +'%Y-%m-%d %T'`
+
+{PRE_COMMAND}
+
+{CODE}
+
+{POST_COMMAND}

divvy_templates/slurm_apptainer_template.sub

@@ -0,0 +1,21 @@
+#!/bin/bash
+#SBATCH --job-name='{JOBNAME}'
+#SBATCH --output='{LOGFILE}'
+#SBATCH --mem='{MEM}'
+#SBATCH --cpus-per-task='{CORES}'
+#SBATCH --time='{TIME}'
+#SBATCH --partition='{PARTITION}'
+#SBATCH -m block
+#SBATCH --ntasks=1
+
+echo 'Compute node:' `hostname`
+echo 'Start time:' `date +'%Y-%m-%d %T'`
+
+{PRE_COMMAND}
+
+apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image
+srun apptainer exec instance://{JOBNAME}_image {CODE}
+
+apptainer instance stop {JOBNAME}_image
+
+{POST_COMMAND}

divvy_templates/slurm_template.sub

@@ -11,4 +11,8 @@
 echo 'Compute node:' `hostname`
 echo 'Start time:' `date +'%Y-%m-%d %T'`
 
+{PRE_COMMAND}
+
 {CODE}
+
+{POST_COMMAND}

looper/__init__.py

@@ -9,9 +9,6 @@
 
 from importlib.metadata import version
 
-import logmuse
-
-logmuse.init_logger("looper")
 __version__ = version("looper")
 
 # Lazy imports - only loaded when accessed

looper/__main__.py

@@ -1,7 +1,137 @@
+import argparse
 import sys
 
 from .cli_pydantic import main
 
+
+def divvy_main(argv=None):
+    """Entry point for the divvy CLI.
+
+    Args:
+        argv: List of arguments (defaults to sys.argv[1:]).
+
+    Returns:
+        int: Exit code (0 for success, 1 for failure).
+    """
+    from .const import DEFAULT_CONFIG_FILEPATH
+    from .divvy import ComputingConfiguration, divvy_init, select_divvy_config
+
+    parser = argparse.ArgumentParser(
+        prog="divvy",
+        description="Write and submit compute job scripts.",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    # -- list --
+    sp_list = subparsers.add_parser("list", help="List available compute packages")
+    sp_list.add_argument("--config", help="Path to divvy configuration file")
+
+    # -- init --
+    sp_init = subparsers.add_parser("init", help="Initialize a new divvy config file")
+    sp_init.add_argument("--config", required=True, help="Path for new config file")
+
+    # -- inspect --
+    sp_inspect = subparsers.add_parser(
+        "inspect", help="Display the template for a compute package"
+    )
+    sp_inspect.add_argument(
+        "-p", "--package", default="default", help="Compute package to inspect"
+    )
+    sp_inspect.add_argument("--config", help="Path to divvy configuration file")
+
+    # -- write --
+    sp_write = subparsers.add_parser(
+        "write", help="Write a job submission script from a template"
+    )
+    sp_write.add_argument(
+        "-p", "--package", default="default", help="Compute package to use"
+    )
+    sp_write.add_argument("-s", "--settings", help="YAML file with job settings")
+    sp_write.add_argument(
+        "-c", "--compute", nargs="+", help="Extra key=value variable pairs"
+    )
+    sp_write.add_argument("-o", "--outfile", help="Output file path")
+    sp_write.add_argument("--config", help="Path to divvy configuration file")
+
+    # -- submit --
+    sp_submit = subparsers.add_parser(
+        "submit", help="Write and submit a job submission script"
+    )
+    sp_submit.add_argument(
+        "-p", "--package", default="default", help="Compute package to use"
+    )
+    sp_submit.add_argument("-s", "--settings", help="YAML file with job settings")
+    sp_submit.add_argument(
+        "-c", "--compute", nargs="+", help="Extra key=value variable pairs"
+    )
+    sp_submit.add_argument("-o", "--outfile", help="Output file path")
+    sp_submit.add_argument("--config", help="Path to divvy configuration file")
+
+    args = parser.parse_args(argv)
+
+    if args.command is None:
+        parser.print_help()
+        return 1
+
+    # Load divvy config
+    config_path = getattr(args, "config", None)
+
+    if args.command == "init":
+        divvy_init(config_path, DEFAULT_CONFIG_FILEPATH)
+        return 0
+
+    dcc = ComputingConfiguration.from_yaml_file(
+        filepath=select_divvy_config(config_path)
+    )
+
+    if args.command == "list":
+        print("Available compute packages:\n")
+        for pkg_name in sorted(dcc.list_compute_packages()):
+            print(pkg_name)
+        return 0
+
+    if args.command == "inspect":
+        dcc.activate_package(args.package)
+        print(dcc.template())
+        return 0
+
+    # Parse --compute key=value pairs into a dict
+    extra_vars = None
+    if args.compute:
+        extra_vars = {}
+        for item in args.compute:
+            if "=" in item:
+                k, v = item.split("=", 1)
+                extra_vars[k] = v
+            else:
+                print(
+                    f"Invalid compute spec (expected key=value): {item}",
+                    file=sys.stderr,
+                )
+                return 1
+
+    # Load --settings YAML if provided
+    if args.settings:
+        from yacman import load_yaml
+
+        settings_data = load_yaml(args.settings)
+        if extra_vars:
+            settings_data.update(extra_vars)
+        extra_vars = settings_data
+
+    dcc.activate_package(args.package)
+
+    if args.command == "write":
+        dcc.write_script(args.outfile, extra_vars)
+        return 0
+
+    if args.command == "submit":
+        dcc.submit(args.outfile, extra_vars)
+        return 0
+
+    return 1
+
+
 if __name__ == "__main__":
     try:
         sys.exit(main())

looper/conductor.py

@@ -175,8 +175,8 @@ class SubmissionConductor(object):
 
     def __init__(
         self,
-        pipeline_interface,
-        prj,
+        pipeline_interface: "PipelineInterface",
+        prj: "Project",
         delay: float = 0,
         extra_args: str | None = None,
         extra_args_override: str | None = None,
@@ -196,7 +196,7 @@ def __init__(
         be overseen by this instance, respectively.
 
         Args:
-            pipeline_interface (PipelineInterface): Collection of important
+            pipeline_interface: Collection of important
                 data for one or more pipelines, like resource allocation packages
                 and option/argument specifications.
             prj: Project with which each sample being considered is

looper/default_config/divvy_config.yaml

@@ -10,29 +10,54 @@ adapters:
   MEM: compute.mem
   DOCKER_ARGS: compute.docker_args
   DOCKER_IMAGE: compute.docker_image
-  SINGULARITY_IMAGE: compute.singularity_image
-  SINGULARITY_ARGS: compute.singularity_args
+  APPTAINER_IMAGE: compute.apptainer_image
+  APPTAINER_ARGS: compute.apptainer_args
+  BULKER_CRATE: compute.bulker_crate
+  PRE_COMMAND: compute.pre_command
+  POST_COMMAND: compute.post_command
 compute_packages:
   default:
     submission_template: divvy_templates/localhost_template.sub
     submission_command: .
+    pre_command: ""
+    post_command: ""
   local:
     submission_template: divvy_templates/localhost_template.sub
     submission_command: .
+    pre_command: ""
+    post_command: ""
   slurm:
     submission_template: divvy_templates/slurm_template.sub
     submission_command: sbatch
-  singularity:
-    submission_template: divvy_templates/localhost_singularity_template.sub
+    pre_command: ""
+    post_command: ""
+  sge:
+    submission_template: divvy_templates/sge_template.sub
+    submission_command: qsub
+    pre_command: ""
+    post_command: ""
+  apptainer:
+    submission_template: divvy_templates/localhost_apptainer_template.sub
     submission_command: .
-    singularity_args: ""
-  singularity_slurm:
-    submission_template: divvy_templates/slurm_singularity_template.sub
+    apptainer_args: ""
+    pre_command: ""
+    post_command: ""
+  apptainer_slurm:
+    submission_template: divvy_templates/slurm_apptainer_template.sub
     submission_command: sbatch
-    singularity_args: ""
+    apptainer_args: ""
+    pre_command: ""
+    post_command: ""
   bulker_local:
     submission_template: divvy_templates/localhost_bulker_template.sub
     submission_command: sh
+    pre_command: ""
+    post_command: ""
+  bulker_slurm:
+    submission_template: divvy_templates/bulker_slurm_template.sub
+    submission_command: sbatch
+    pre_command: ""
+    post_command: ""
   docker:
     submission_template: divvy_templates/localhost_docker_template.sub
     submission_command: .
@@ -42,6 +67,7 @@ compute_packages:
       --volume="/etc/group:/etc/group:ro" \
       --volume="/etc/passwd:/etc/passwd:ro" \
       --volume="/etc/shadow:/etc/shadow:ro"  \
-      --volume="/etc/sudoers.d:/etc/sudoers.d:ro" \
       --volume="/tmp/.X11-unix:/tmp/.X11-unix:rw" \
       --workdir="`pwd`" \
+    pre_command: ""
+    post_command: ""

tests/divvytests/data/divcfg-master/templates/slurm_singularity_template.sub → looper/default_config/divvy_templates/bulker_slurm_template.sub

@@ -11,7 +11,10 @@
 echo 'Compute node:' `hostname`
 echo 'Start time:' `date +'%Y-%m-%d %T'`
 
-singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image
-srun singularity exec instance://{JOBNAME}_image {CODE}
+{PRE_COMMAND}
 
-singularity instance.stop {JOBNAME}_image
+eval "$(bulker activate -e {BULKER_CRATE})"
+
+{CODE}
+
+{POST_COMMAND}

looper/default_config/divvy_templates/localhost_apptainer_template.sub

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo 'Compute node:' `hostname`
+echo 'Start time:' `date +'%Y-%m-%d %T'`
+
+{PRE_COMMAND}
+
+{
+apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image
+apptainer exec instance://{JOBNAME}_image {CODE}
+} | tee {LOGFILE} --ignore-interrupts
+
+{POST_COMMAND}