From 44e60c9f01138e72efc7c9ccc56d75b9c57e904c Mon Sep 17 00:00:00 2001 From: nsheff Date: Thu, 5 Mar 2026 21:32:20 -0500 Subject: [PATCH 1/8] remove old logmuse setup --- looper/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/looper/__init__.py b/looper/__init__.py index 015da2c0..772615f6 100644 --- a/looper/__init__.py +++ b/looper/__init__.py @@ -9,9 +9,6 @@ from importlib.metadata import version -import logmuse - -logmuse.init_logger("looper") __version__ = version("looper") # Lazy imports - only loaded when accessed From a8b2b7434847eba76a698102fe065c08485106f4 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Mar 2026 07:50:59 -0400 Subject: [PATCH 2/8] Fix divvy cli --- looper/__main__.py | 131 +++++++++++++++++++++++++++++ tests/divvytests/test_divvy_cli.py | 78 +++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 tests/divvytests/test_divvy_cli.py diff --git a/looper/__main__.py b/looper/__main__.py index ff3d95f8..af632a7f 100644 --- a/looper/__main__.py +++ b/looper/__main__.py @@ -1,7 +1,138 @@ +import argparse import sys from .cli_pydantic import main + +def divvy_main(argv=None): + """Entry point for the divvy CLI. + + Args: + argv: List of arguments (defaults to sys.argv[1:]). + + Returns: + int: Exit code (0 for success, 1 for failure). + """ + from .const import DEFAULT_CONFIG_FILEPATH + from .divvy import ComputingConfiguration, divvy_init, select_divvy_config + + parser = argparse.ArgumentParser( + prog="divvy", + description="Write and submit compute job scripts.", + ) + subparsers = parser.add_subparsers(dest="command") + + # -- list -- + sp_list = subparsers.add_parser("list", help="List available compute packages") + sp_list.add_argument("--config", help="Path to divvy configuration file") + + # -- init -- + sp_init = subparsers.add_parser("init", help="Initialize a new divvy config file") + sp_init.add_argument("--config", required=True, help="Path for new config file") + + # -- inspect -- + sp_inspect = subparsers.add_parser( + "inspect", help="Display the template for a compute package" + ) + sp_inspect.add_argument( + "-p", "--package", default="default", help="Compute package to inspect" + ) + sp_inspect.add_argument("--config", help="Path to divvy configuration file") + + # -- write -- + sp_write = subparsers.add_parser( + "write", help="Write a job submission script from a template" + ) + sp_write.add_argument( + "-p", "--package", default="default", help="Compute package to use" + ) + sp_write.add_argument("-s", "--settings", help="YAML file with job settings") + sp_write.add_argument( + "-c", "--compute", nargs="+", help="Extra key=value variable pairs" + ) + sp_write.add_argument("-o", "--outfile", help="Output file path") + sp_write.add_argument("--config", help="Path to divvy configuration file") + + # -- submit -- + sp_submit = subparsers.add_parser( + "submit", help="Write and submit a job submission script" + ) + sp_submit.add_argument( + "-p", "--package", default="default", help="Compute package to use" + ) + sp_submit.add_argument("-s", "--settings", help="YAML file with job settings") + sp_submit.add_argument( + "-c", "--compute", nargs="+", help="Extra key=value variable pairs" + ) + sp_submit.add_argument("-o", "--outfile", help="Output file path") + sp_submit.add_argument("--config", help="Path to divvy configuration file") + + args = parser.parse_args(argv) + + if args.command is None: + parser.print_help() + return 1 + + # Load divvy config + config_path = getattr(args, "config", None) + + if args.command == "init": + divvy_init(config_path, DEFAULT_CONFIG_FILEPATH) + return 0 + + dcc = ComputingConfiguration.from_yaml_file( + filepath=select_divvy_config(config_path) + ) + + if args.command == "list": + print("Available compute packages:\n") + for pkg_name in sorted(dcc.list_compute_packages()): + print(pkg_name) + return 0 + + if args.command == "inspect": + dcc.activate_package(args.package) + print(dcc.template()) + return 0 + + # Parse --compute key=value pairs into a dict + extra_vars = None + if args.compute: + extra_vars = {} + for item in args.compute: + if "=" in item: + k, v = item.split("=", 1) + extra_vars[k] = v + else: + print( + f"Invalid compute spec (expected key=value): {item}", + file=sys.stderr, + ) + return 1 + + # Load --settings YAML if provided + if args.settings: + from yacman import load_yaml + + settings_data = load_yaml(args.settings) + if extra_vars: + settings_data.update(extra_vars) + extra_vars = settings_data + + dcc.activate_package(args.package) + + if args.command == "write": + outfile = args.outfile or "submit_script.sub" + dcc.write_script(outfile, extra_vars) + return 0 + + if args.command == "submit": + dcc.submit(args.outfile, extra_vars) + return 0 + + return 1 + + if __name__ == "__main__": try: sys.exit(main()) diff --git a/tests/divvytests/test_divvy_cli.py b/tests/divvytests/test_divvy_cli.py new file mode 100644 index 00000000..0925ec06 --- /dev/null +++ b/tests/divvytests/test_divvy_cli.py @@ -0,0 +1,78 @@ +"""Tests for the divvy CLI entry point (divvy_main).""" + +import os + + +class TestDivvyMainImport: + def test_divvy_main_is_importable(self): + from looper.__main__ import divvy_main + + assert callable(divvy_main) + + +class TestDivvyList: + def test_list_exits_zero(self): + from looper.__main__ import divvy_main + + ret = divvy_main(["list"]) + assert ret == 0 + + def test_list_output_contains_packages(self, capsys): + from looper.__main__ import divvy_main + + divvy_main(["list"]) + captured = capsys.readouterr() + assert "default" in captured.out + assert "slurm" in captured.out + + +class TestDivvyInit: + def test_init_creates_config(self, tmp_path): + from looper.__main__ import divvy_main + + config_path = str(tmp_path / "new_config" / "divvy_config.yaml") + ret = divvy_main(["init", "--config", config_path]) + assert ret == 0 + assert os.path.exists(config_path) + + +class TestDivvyInspect: + def test_inspect_default_package(self, capsys): + from looper.__main__ import divvy_main + + ret = divvy_main(["inspect"]) + assert ret == 0 + captured = capsys.readouterr() + assert len(captured.out) > 0 + + def test_inspect_slurm_package(self, capsys): + from looper.__main__ import divvy_main + + ret = divvy_main(["inspect", "--package", "slurm"]) + assert ret == 0 + captured = capsys.readouterr() + assert "SBATCH" in captured.out + + +class TestDivvyWrite: + def test_write_creates_script(self, tmp_path): + from looper.__main__ import divvy_main + + outfile = str(tmp_path / "test_submit.sh") + ret = divvy_main( + [ + "write", + "--package", + "slurm", + "--compute", + "code=echo hello", + "jobname=test", + "--outfile", + outfile, + ] + ) + assert ret == 0 + assert os.path.exists(outfile) + with open(outfile) as f: + contents = f.read() + assert "echo hello" in contents From 4994feee1f825569a915aecf76a562030ea75937 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Mar 2026 08:40:24 -0400 Subject: [PATCH 3/8] update divvy tests --- looper/__main__.py | 3 +- tests/divvytests/test_divvy_cli.py | 18 +++++ tests/divvytests/test_divvy_simple.py | 95 +++++++++++---------------- 3 files changed, 58 insertions(+), 58 deletions(-) diff --git a/looper/__main__.py b/looper/__main__.py index af632a7f..c52789ea 100644 --- a/looper/__main__.py +++ b/looper/__main__.py @@ -122,8 +122,7 @@ def divvy_main(argv=None): dcc.activate_package(args.package) if args.command == "write": - outfile = args.outfile or "submit_script.sub" - dcc.write_script(outfile, extra_vars) + dcc.write_script(args.outfile, extra_vars) return 0 if args.command == "submit": diff --git a/tests/divvytests/test_divvy_cli.py b/tests/divvytests/test_divvy_cli.py index 0925ec06..fc47bad8 100644 --- a/tests/divvytests/test_divvy_cli.py +++ b/tests/divvytests/test_divvy_cli.py @@ -76,3 +76,21 @@ def test_write_creates_script(self, tmp_path): with open(outfile) as f: contents = f.read() assert "echo hello" in contents + + def test_write_no_outfile_prints_to_stdout(self, capsys): + from looper.__main__ import divvy_main + + ret = divvy_main( + [ + "write", + "--package", + "slurm", + "--compute", + "code=echo hello", + "jobname=test", + ] + ) + assert ret == 0 + captured = capsys.readouterr() + assert "echo hello" in captured.out + assert "SBATCH" in captured.out diff --git a/tests/divvytests/test_divvy_simple.py b/tests/divvytests/test_divvy_simple.py index 91271603..1d1586bf 100644 --- a/tests/divvytests/test_divvy_simple.py +++ b/tests/divvytests/test_divvy_simple.py @@ -1,4 +1,5 @@ -import os +import pytest +from yacman import YAMLConfigManager import looper.divvy as divvy from looper.divvy import select_divvy_config @@ -23,71 +24,53 @@ def test_activate_package(self): class TestWriting: - def test_write_script(self): + def test_write_script(self, tmp_path): dcc_filepath = select_divvy_config(None) - dcc = divvy.ComputingConfiguration().from_yaml_file(filepath=dcc_filepath) - dcc + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) dcc.activate_package("singularity_slurm") extra_vars = { "singularity_image": "simg", "jobname": "jbname", "code": "mycode", } - dcc.write_script("test.sub", extra_vars) - with open("test.sub", "r") as f: + outfile = str(tmp_path / "test.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: contents = f.read() assert contents.find("mycode") > 0 assert contents.find("{SINGULARITY_ARGS}") < 0 - os.remove("test.sub") - -# class TestAdapters: -# @pytest.mark.parametrize( -# "compute", -# [ -# dict({"mem": 1000, "test": 0}), -# YAMLConfigManager({"mem": 1000, "test": 0}), -# OrderedDict({"mem": 1000, "test": 0}), -# ], -# ) -# @pytest.mark.parametrize("package", ["singularity_slurm", "slurm"]) -# def test_write_script_adapters(self, compute, package): -# """Test successful adapter sourcing from various Mapping types""" -# dcc = divvy.ComputingConfiguration() -# dcc.activate_package(package) -# extra_vars = {"compute": compute} -# dcc.write_script("test.sub", extra_vars) -# with open("test.sub", "r") as f: -# contents = f.read() -# assert contents.find("1000") > 0 -# os.remove("test.sub") -# -# def test_adapters_overwitten_by_others(self): -# dcc = divvy.ComputingConfiguration() -# dcc.activate_package("singularity_slurm") -# compute = YAMLConfigManager({"mem": 1000}) -# extra_vars = [{"compute": compute}, {"MEM": 333}] -# dcc.write_script("test1.sub", extra_vars) -# with open("test1.sub", "r") as f: -# contents = f.read() -# assert not (contents.find("1000") > 0) -# assert contents.find("333") > 0 -# os.remove("test1.sub") -# -# def test_update(): -# # probably will be removed later -# dcc1 = divvy.ComputingConfiguration() -# dcc1.update_packages("code/divvy/tests/data/pepenv-master/cemm.yaml") -# dcc2 = divvy.ComputingConfiguration() -# y = yacman.load_yaml("code/divvy/tests/data/pepenv-master/cemm.yaml") -# dcc2.update(y) -# dcc1 == dcc2 - -# class ptest(object): -# @property -# def doubleslash(self): -# return '//' +class TestAdapters: + @pytest.mark.parametrize( + "compute", + [ + {"mem": 1000, "test": 0}, + YAMLConfigManager({"mem": 1000, "test": 0}), + ], + ) + @pytest.mark.parametrize("package", ["singularity_slurm", "slurm"]) + def test_write_script_adapters(self, compute, package, tmp_path): + """Test successful adapter sourcing from various Mapping types""" + dcc_filepath = select_divvy_config(None) + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) + dcc.activate_package(package) + extra_vars = {"compute": compute} + outfile = str(tmp_path / "test.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: + contents = f.read() + assert contents.find("1000") > 0 -# p = ptest() -# p.doubleslash + def test_adapters_overwritten_by_others(self, tmp_path): + dcc_filepath = select_divvy_config(None) + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) + dcc.activate_package("singularity_slurm") + compute = YAMLConfigManager({"mem": 1000}) + extra_vars = [{"compute": compute}, {"MEM": 333}] + outfile = str(tmp_path / "test1.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: + contents = f.read() + assert not (contents.find("1000") > 0) + assert contents.find("333") > 0 From 4131afb08820c275aa64c6664e48e12329f4a670 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Mar 2026 09:22:28 -0400 Subject: [PATCH 4/8] improve docstring type hints --- looper/conductor.py | 6 +++--- looper/divvy.py | 2 -- looper/project.py | 4 ---- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/looper/conductor.py b/looper/conductor.py index d3a4760b..83756614 100644 --- a/looper/conductor.py +++ b/looper/conductor.py @@ -175,8 +175,8 @@ class SubmissionConductor(object): def __init__( self, - pipeline_interface, - prj, + pipeline_interface: "PipelineInterface", + prj: "Project", delay: float = 0, extra_args: str | None = None, extra_args_override: str | None = None, @@ -196,7 +196,7 @@ def __init__( be overseen by this instance, respectively. Args: - pipeline_interface (PipelineInterface): Collection of important + pipeline_interface: Collection of important data for one or more pipelines, like resource allocation packages and option/argument specifications. prj: Project with which each sample being considered is diff --git a/looper/divvy.py b/looper/divvy.py index 913f6451..862a7009 100644 --- a/looper/divvy.py +++ b/looper/divvy.py @@ -32,8 +32,6 @@ class ComputingConfiguration(YAMLConfigManager): Args: entries (str | Iterable[(str, object)] | Mapping[str, object]): Config collection of key-value pairs. - filepath (str): YAML file specifying computing package data (the - `DIVCFG` file). """ def __init__( diff --git a/looper/project.py b/looper/project.py index 7b58a4e5..60262fd1 100644 --- a/looper/project.py +++ b/looper/project.py @@ -130,10 +130,6 @@ class Project(peppyProject): amendments (Iterable[str]): Name indicating amendment to use, optional. divcfg_path (str): Path to an environment configuration YAML file specifying compute settings. - permissive (bool): Whether a error should be thrown if a sample input - file(s) do not exist or cannot be open. - compute_env_file (str): Environment configuration YAML file specifying - compute settings. """ def __init__( From c00c1e7eaa3c18ad0a948810ae96dd5831f0d737 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Mar 2026 10:00:04 -0400 Subject: [PATCH 5/8] modernize divvy templates. add hooks, update to apptainer --- .../bulker_slurm_template.sub | 9 +- .../localhost_apptainer_template.sub | 13 +++ divvy_templates/localhost_bulker_template.sub | 4 + divvy_templates/localhost_docker_template.sub | 4 + .../localhost_singularity_template.sub | 9 -- divvy_templates/localhost_template.sub | 4 + divvy_templates/lsf_template.sub | 5 +- divvy_templates/sge_template.sub | 16 ++- divvy_templates/slurm_apptainer_template.sub | 21 ++++ divvy_templates/slurm_template.sub | 4 + looper/default_config/divvy_config.yaml | 44 +++++-- .../divvy_templates/bulker_slurm_template.sub | 9 +- .../localhost_apptainer_template.sub | 13 +++ .../localhost_bulker_template.sub | 4 + .../localhost_docker_template.sub | 4 + .../localhost_singularity_template.sub | 9 -- .../divvy_templates/localhost_template.sub | 6 +- .../divvy_templates/lsf_template.sub | 5 +- .../divvy_templates/sge_template.sub | 16 ++- .../slurm_apptainer_template.sub | 21 ++++ .../divvy_templates/slurm_template.sub | 4 + .../pipeline_interface_schema_generic.yaml | 4 +- .../pipeline_interface_schema_project.yaml | 4 +- .../pipeline_interface_schema_sample.yaml | 4 +- tests/divvytests/data/divcfg-master/README.md | 4 +- tests/divvytests/data/divcfg-master/cemm.yaml | 4 +- .../data/divcfg-master/local_containers.yaml | 8 +- tests/divvytests/data/divcfg-master/puma.yaml | 6 +- .../localhost_apptainer_template.sub | 9 ++ .../localhost_singularity_template.sub | 9 -- .../templates/slurm_apptainer_template.sub | 6 +- .../data/divcfg-master/uva_rivanna.yaml | 16 +-- .../data/divcfg-master/uva_rivanna_old.yaml | 12 +- tests/divvytests/test_divvy_simple.py | 107 +++++++++++++++++- 34 files changed, 331 insertions(+), 86 deletions(-) rename tests/divvytests/data/divcfg-master/templates/slurm_singularity_template.sub => divvy_templates/bulker_slurm_template.sub (62%) create mode 100644 divvy_templates/localhost_apptainer_template.sub delete mode 100755 divvy_templates/localhost_singularity_template.sub create mode 100644 divvy_templates/slurm_apptainer_template.sub rename divvy_templates/slurm_singularity_template.sub => looper/default_config/divvy_templates/bulker_slurm_template.sub (62%) mode change 100755 => 100644 create mode 100644 looper/default_config/divvy_templates/localhost_apptainer_template.sub delete mode 100755 looper/default_config/divvy_templates/localhost_singularity_template.sub create mode 100644 looper/default_config/divvy_templates/slurm_apptainer_template.sub create mode 100644 tests/divvytests/data/divcfg-master/templates/localhost_apptainer_template.sub delete mode 100755 tests/divvytests/data/divcfg-master/templates/localhost_singularity_template.sub rename looper/default_config/divvy_templates/slurm_singularity_template.sub => tests/divvytests/data/divcfg-master/templates/slurm_apptainer_template.sub (62%) mode change 100755 => 100644 diff --git a/tests/divvytests/data/divcfg-master/templates/slurm_singularity_template.sub b/divvy_templates/bulker_slurm_template.sub similarity index 62% rename from tests/divvytests/data/divcfg-master/templates/slurm_singularity_template.sub rename to divvy_templates/bulker_slurm_template.sub index eb8359be..b684d9be 100644 --- a/tests/divvytests/data/divcfg-master/templates/slurm_singularity_template.sub +++ b/divvy_templates/bulker_slurm_template.sub @@ -11,7 +11,10 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -srun singularity exec instance://{JOBNAME}_image {CODE} +{PRE_COMMAND} -singularity instance.stop {JOBNAME}_image +eval "$(bulker activate -e {BULKER_CRATE})" + +{CODE} + +{POST_COMMAND} diff --git a/divvy_templates/localhost_apptainer_template.sub b/divvy_templates/localhost_apptainer_template.sub new file mode 100644 index 00000000..99c5c3cd --- /dev/null +++ b/divvy_templates/localhost_apptainer_template.sub @@ -0,0 +1,13 @@ +#!/bin/bash + +echo 'Compute node:' `hostname` +echo 'Start time:' `date +'%Y-%m-%d %T'` + +{PRE_COMMAND} + +{ +apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image +apptainer exec instance://{JOBNAME}_image {CODE} +} | tee {LOGFILE} --ignore-interrupts + +{POST_COMMAND} diff --git a/divvy_templates/localhost_bulker_template.sub b/divvy_templates/localhost_bulker_template.sub index 8ef7e7e3..01b21336 100755 --- a/divvy_templates/localhost_bulker_template.sub +++ b/divvy_templates/localhost_bulker_template.sub @@ -3,8 +3,12 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + eval "$(bulker activate -e {BULKER_CRATE})" { {CODE} } | tee {LOGFILE} -i + +{POST_COMMAND} diff --git a/divvy_templates/localhost_docker_template.sub b/divvy_templates/localhost_docker_template.sub index a00ad037..3d2d301c 100755 --- a/divvy_templates/localhost_docker_template.sub +++ b/divvy_templates/localhost_docker_template.sub @@ -3,6 +3,10 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + { docker run --rm -it {DOCKER_ARGS} {DOCKER_IMAGE} {CODE} } | tee {LOGFILE} --ignore-interrupts + +{POST_COMMAND} diff --git a/divvy_templates/localhost_singularity_template.sub b/divvy_templates/localhost_singularity_template.sub deleted file mode 100755 index fdb1e486..00000000 --- a/divvy_templates/localhost_singularity_template.sub +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -{ -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -singularity exec instance://{JOBNAME}_image {CODE} -} | tee {LOGFILE} --ignore-interrupts diff --git a/divvy_templates/localhost_template.sub b/divvy_templates/localhost_template.sub index 9b64b754..1b859646 100755 --- a/divvy_templates/localhost_template.sub +++ b/divvy_templates/localhost_template.sub @@ -3,6 +3,10 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + { {CODE} } | tee {LOGFILE} + +{POST_COMMAND} diff --git a/divvy_templates/lsf_template.sub b/divvy_templates/lsf_template.sub index ac2bfc1e..642d4b42 100755 --- a/divvy_templates/lsf_template.sub +++ b/divvy_templates/lsf_template.sub @@ -1,4 +1,7 @@ #!/bin/bash -bsub -n{CORES} -W {TIME} -R \"rusage[mem={MEM}]\" -o {LOGFILE} {CODE} +{PRE_COMMAND} +bsub -n{CORES} -W {TIME} -R "rusage[mem={MEM}]" -o {LOGFILE} {CODE} + +{POST_COMMAND} diff --git a/divvy_templates/sge_template.sub b/divvy_templates/sge_template.sub index 0964f4df..0855c383 100755 --- a/divvy_templates/sge_template.sub +++ b/divvy_templates/sge_template.sub @@ -1 +1,15 @@ -This has not been implemented, but you could add whatever cluster submission systems here, just use the slurm_template as an example. +#!/bin/bash +#$ -N '{JOBNAME}' +#$ -o '{LOGFILE}' +#$ -l mem_free={MEM} +#$ -pe smp {CORES} +#$ -l h_rt={TIME} + +echo 'Compute node:' `hostname` +echo 'Start time:' `date +'%Y-%m-%d %T'` + +{PRE_COMMAND} + +{CODE} + +{POST_COMMAND} diff --git a/divvy_templates/slurm_apptainer_template.sub b/divvy_templates/slurm_apptainer_template.sub new file mode 100644 index 00000000..be968485 --- /dev/null +++ b/divvy_templates/slurm_apptainer_template.sub @@ -0,0 +1,21 @@ +#!/bin/bash +#SBATCH --job-name='{JOBNAME}' +#SBATCH --output='{LOGFILE}' +#SBATCH --mem='{MEM}' +#SBATCH --cpus-per-task='{CORES}' +#SBATCH --time='{TIME}' +#SBATCH --partition='{PARTITION}' +#SBATCH -m block +#SBATCH --ntasks=1 + +echo 'Compute node:' `hostname` +echo 'Start time:' `date +'%Y-%m-%d %T'` + +{PRE_COMMAND} + +apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image +srun apptainer exec instance://{JOBNAME}_image {CODE} + +apptainer instance stop {JOBNAME}_image + +{POST_COMMAND} diff --git a/divvy_templates/slurm_template.sub b/divvy_templates/slurm_template.sub index f9d74564..fcbf7307 100755 --- a/divvy_templates/slurm_template.sub +++ b/divvy_templates/slurm_template.sub @@ -11,4 +11,8 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + {CODE} + +{POST_COMMAND} diff --git a/looper/default_config/divvy_config.yaml b/looper/default_config/divvy_config.yaml index 81db360f..d31f82f9 100644 --- a/looper/default_config/divvy_config.yaml +++ b/looper/default_config/divvy_config.yaml @@ -10,29 +10,54 @@ adapters: MEM: compute.mem DOCKER_ARGS: compute.docker_args DOCKER_IMAGE: compute.docker_image - SINGULARITY_IMAGE: compute.singularity_image - SINGULARITY_ARGS: compute.singularity_args + APPTAINER_IMAGE: compute.apptainer_image + APPTAINER_ARGS: compute.apptainer_args + BULKER_CRATE: compute.bulker_crate + PRE_COMMAND: compute.pre_command + POST_COMMAND: compute.post_command compute_packages: default: submission_template: divvy_templates/localhost_template.sub submission_command: . + pre_command: "" + post_command: "" local: submission_template: divvy_templates/localhost_template.sub submission_command: . + pre_command: "" + post_command: "" slurm: submission_template: divvy_templates/slurm_template.sub submission_command: sbatch - singularity: - submission_template: divvy_templates/localhost_singularity_template.sub + pre_command: "" + post_command: "" + sge: + submission_template: divvy_templates/sge_template.sub + submission_command: qsub + pre_command: "" + post_command: "" + apptainer: + submission_template: divvy_templates/localhost_apptainer_template.sub submission_command: . - singularity_args: "" - singularity_slurm: - submission_template: divvy_templates/slurm_singularity_template.sub + apptainer_args: "" + pre_command: "" + post_command: "" + apptainer_slurm: + submission_template: divvy_templates/slurm_apptainer_template.sub submission_command: sbatch - singularity_args: "" + apptainer_args: "" + pre_command: "" + post_command: "" bulker_local: submission_template: divvy_templates/localhost_bulker_template.sub submission_command: sh + pre_command: "" + post_command: "" + bulker_slurm: + submission_template: divvy_templates/bulker_slurm_template.sub + submission_command: sbatch + pre_command: "" + post_command: "" docker: submission_template: divvy_templates/localhost_docker_template.sub submission_command: . @@ -42,6 +67,7 @@ compute_packages: --volume="/etc/group:/etc/group:ro" \ --volume="/etc/passwd:/etc/passwd:ro" \ --volume="/etc/shadow:/etc/shadow:ro" \ - --volume="/etc/sudoers.d:/etc/sudoers.d:ro" \ --volume="/tmp/.X11-unix:/tmp/.X11-unix:rw" \ --workdir="`pwd`" \ + pre_command: "" + post_command: "" diff --git a/divvy_templates/slurm_singularity_template.sub b/looper/default_config/divvy_templates/bulker_slurm_template.sub old mode 100755 new mode 100644 similarity index 62% rename from divvy_templates/slurm_singularity_template.sub rename to looper/default_config/divvy_templates/bulker_slurm_template.sub index eb8359be..b684d9be --- a/divvy_templates/slurm_singularity_template.sub +++ b/looper/default_config/divvy_templates/bulker_slurm_template.sub @@ -11,7 +11,10 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -srun singularity exec instance://{JOBNAME}_image {CODE} +{PRE_COMMAND} -singularity instance.stop {JOBNAME}_image +eval "$(bulker activate -e {BULKER_CRATE})" + +{CODE} + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/localhost_apptainer_template.sub b/looper/default_config/divvy_templates/localhost_apptainer_template.sub new file mode 100644 index 00000000..99c5c3cd --- /dev/null +++ b/looper/default_config/divvy_templates/localhost_apptainer_template.sub @@ -0,0 +1,13 @@ +#!/bin/bash + +echo 'Compute node:' `hostname` +echo 'Start time:' `date +'%Y-%m-%d %T'` + +{PRE_COMMAND} + +{ +apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image +apptainer exec instance://{JOBNAME}_image {CODE} +} | tee {LOGFILE} --ignore-interrupts + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/localhost_bulker_template.sub b/looper/default_config/divvy_templates/localhost_bulker_template.sub index 8ef7e7e3..01b21336 100755 --- a/looper/default_config/divvy_templates/localhost_bulker_template.sub +++ b/looper/default_config/divvy_templates/localhost_bulker_template.sub @@ -3,8 +3,12 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + eval "$(bulker activate -e {BULKER_CRATE})" { {CODE} } | tee {LOGFILE} -i + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/localhost_docker_template.sub b/looper/default_config/divvy_templates/localhost_docker_template.sub index a00ad037..3d2d301c 100755 --- a/looper/default_config/divvy_templates/localhost_docker_template.sub +++ b/looper/default_config/divvy_templates/localhost_docker_template.sub @@ -3,6 +3,10 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + { docker run --rm -it {DOCKER_ARGS} {DOCKER_IMAGE} {CODE} } | tee {LOGFILE} --ignore-interrupts + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/localhost_singularity_template.sub b/looper/default_config/divvy_templates/localhost_singularity_template.sub deleted file mode 100755 index fdb1e486..00000000 --- a/looper/default_config/divvy_templates/localhost_singularity_template.sub +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -{ -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -singularity exec instance://{JOBNAME}_image {CODE} -} | tee {LOGFILE} --ignore-interrupts diff --git a/looper/default_config/divvy_templates/localhost_template.sub b/looper/default_config/divvy_templates/localhost_template.sub index 5c8e2af4..1b859646 100644 --- a/looper/default_config/divvy_templates/localhost_template.sub +++ b/looper/default_config/divvy_templates/localhost_template.sub @@ -3,6 +3,10 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + { {CODE} -} | tee {LOGFILE} \ No newline at end of file +} | tee {LOGFILE} + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/lsf_template.sub b/looper/default_config/divvy_templates/lsf_template.sub index ac2bfc1e..642d4b42 100755 --- a/looper/default_config/divvy_templates/lsf_template.sub +++ b/looper/default_config/divvy_templates/lsf_template.sub @@ -1,4 +1,7 @@ #!/bin/bash -bsub -n{CORES} -W {TIME} -R \"rusage[mem={MEM}]\" -o {LOGFILE} {CODE} +{PRE_COMMAND} +bsub -n{CORES} -W {TIME} -R "rusage[mem={MEM}]" -o {LOGFILE} {CODE} + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/sge_template.sub b/looper/default_config/divvy_templates/sge_template.sub index 0964f4df..0855c383 100755 --- a/looper/default_config/divvy_templates/sge_template.sub +++ b/looper/default_config/divvy_templates/sge_template.sub @@ -1 +1,15 @@ -This has not been implemented, but you could add whatever cluster submission systems here, just use the slurm_template as an example. +#!/bin/bash +#$ -N '{JOBNAME}' +#$ -o '{LOGFILE}' +#$ -l mem_free={MEM} +#$ -pe smp {CORES} +#$ -l h_rt={TIME} + +echo 'Compute node:' `hostname` +echo 'Start time:' `date +'%Y-%m-%d %T'` + +{PRE_COMMAND} + +{CODE} + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/slurm_apptainer_template.sub b/looper/default_config/divvy_templates/slurm_apptainer_template.sub new file mode 100644 index 00000000..be968485 --- /dev/null +++ b/looper/default_config/divvy_templates/slurm_apptainer_template.sub @@ -0,0 +1,21 @@ +#!/bin/bash +#SBATCH --job-name='{JOBNAME}' +#SBATCH --output='{LOGFILE}' +#SBATCH --mem='{MEM}' +#SBATCH --cpus-per-task='{CORES}' +#SBATCH --time='{TIME}' +#SBATCH --partition='{PARTITION}' +#SBATCH -m block +#SBATCH --ntasks=1 + +echo 'Compute node:' `hostname` +echo 'Start time:' `date +'%Y-%m-%d %T'` + +{PRE_COMMAND} + +apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image +srun apptainer exec instance://{JOBNAME}_image {CODE} + +apptainer instance stop {JOBNAME}_image + +{POST_COMMAND} diff --git a/looper/default_config/divvy_templates/slurm_template.sub b/looper/default_config/divvy_templates/slurm_template.sub index f9d74564..fcbf7307 100755 --- a/looper/default_config/divvy_templates/slurm_template.sub +++ b/looper/default_config/divvy_templates/slurm_template.sub @@ -11,4 +11,8 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` +{PRE_COMMAND} + {CODE} + +{POST_COMMAND} diff --git a/looper/schemas/pipeline_interface_schema_generic.yaml b/looper/schemas/pipeline_interface_schema_generic.yaml index 8528b613..68bc2b8f 100644 --- a/looper/schemas/pipeline_interface_schema_generic.yaml +++ b/looper/schemas/pipeline_interface_schema_generic.yaml @@ -39,9 +39,9 @@ properties: docker_image: type: string description: "Docker image identifier" - singularity_image: + apptainer_image: type: string - description: "Singularity image identifier" + description: "Apptainer image identifier" if: properties: pipeline_type: diff --git a/looper/schemas/pipeline_interface_schema_project.yaml b/looper/schemas/pipeline_interface_schema_project.yaml index 7f0a2aaa..c474086b 100644 --- a/looper/schemas/pipeline_interface_schema_project.yaml +++ b/looper/schemas/pipeline_interface_schema_project.yaml @@ -42,9 +42,9 @@ properties: docker_image: type: string description: "Docker image identifier" - singularity_image: + apptainer_image: type: string - description: "Singularity image identifier" + description: "Apptainer image identifier" inject_env_vars: type: object description: "Environment variables to inject into submission scripts. Keys are variable names, values are Jinja2 templates." diff --git a/looper/schemas/pipeline_interface_schema_sample.yaml b/looper/schemas/pipeline_interface_schema_sample.yaml index 63b9d8b0..a377edd1 100644 --- a/looper/schemas/pipeline_interface_schema_sample.yaml +++ b/looper/schemas/pipeline_interface_schema_sample.yaml @@ -42,9 +42,9 @@ properties: docker_image: type: string description: "Docker image identifier" - singularity_image: + apptainer_image: type: string - description: "Singularity image identifier" + description: "Apptainer image identifier" inject_env_vars: type: object description: "Environment variables to inject into submission scripts. Keys are variable names, values are Jinja2 templates." diff --git a/tests/divvytests/data/divcfg-master/README.md b/tests/divvytests/data/divcfg-master/README.md index 4a57e599..2f6bd910 100644 --- a/tests/divvytests/data/divcfg-master/README.md +++ b/tests/divvytests/data/divcfg-master/README.md @@ -13,7 +13,7 @@ If you're at one of the following places, set-up is very simple. Here's a list o * `nih_biowulf2.yaml`: [Biowulf2](https://hpc.nih.gov/docs/userguide.html) cluster at the NIH * `stanford_sherlock.yaml`: [Sherlock](http://sherlock.stanford.edu/mediawiki/index.php/Current_policies) cluster at Stanford * `ski-cer_lilac.yaml`: *lilac* cluster at Memorial Sloan Kettering - * `local_containers.yaml`: A generic local desktop or server (with no cluster management system) that will use docker or singularity containers. + * `local_containers.yaml`: A generic local desktop or server (with no cluster management system) that will use docker or apptainer containers. To configure `divvy` to use one of these, all you have to do is: @@ -52,4 +52,4 @@ The `divvy` documentation includes detailed instructions for [how to write your # What is divvy? -`Divvy` enables any tool to seamlessly switch between cluster resource managers (SGE, SLURM, *etc.*), linux containers (`docker`, `singularity`, *etc.*), or other computing environments. +`Divvy` enables any tool to seamlessly switch between cluster resource managers (SGE, SLURM, *etc.*), linux containers (`docker`, `apptainer`, *etc.*), or other computing environments. diff --git a/tests/divvytests/data/divcfg-master/cemm.yaml b/tests/divvytests/data/divcfg-master/cemm.yaml index 0549c4ff..ef150757 100644 --- a/tests/divvytests/data/divcfg-master/cemm.yaml +++ b/tests/divvytests/data/divcfg-master/cemm.yaml @@ -10,8 +10,8 @@ adapters: BULKER_CRATE: compute.bulker_crate DOCKER_ARGS: compute.docker_args DOCKER_IMAGE: compute.docker_image - SINGULARITY_IMAGE: compute.singularity_image - SINGULARITY_ARGS: compute.singularity_args + APPTAINER_IMAGE: compute.apptainer_image + APPTAINER_ARGS: compute.apptainer_args compute_packages: default: submission_template: templates/slurm_template.sub diff --git a/tests/divvytests/data/divcfg-master/local_containers.yaml b/tests/divvytests/data/divcfg-master/local_containers.yaml index 2ff15bb3..b128eea3 100644 --- a/tests/divvytests/data/divcfg-master/local_containers.yaml +++ b/tests/divvytests/data/divcfg-master/local_containers.yaml @@ -1,15 +1,15 @@ # Environment configuration file for looper # This version describes the compute environment for a local computer that uses -# docker or singularity containers +# docker or apptainer containers compute_packages: default: submission_template: templates/localhost_template.sub submission_command: sh - singularity: - submission_template: templates/localhost_singularity_template.sub + apptainer: + submission_template: templates/localhost_apptainer_template.sub submission_command: sh - singularity_args: -B /ext:/ext + apptainer_args: -B /ext:/ext docker: submission_template: templates/localhost_docker_template.sub submission_command: sh diff --git a/tests/divvytests/data/divcfg-master/puma.yaml b/tests/divvytests/data/divcfg-master/puma.yaml index 7aa31b07..94b33cf1 100644 --- a/tests/divvytests/data/divcfg-master/puma.yaml +++ b/tests/divvytests/data/divcfg-master/puma.yaml @@ -11,10 +11,10 @@ compute_packages: default: submission_template: templates/localhost_template.sub submission_command: sh - singularity: - submission_template: templates/localhost_singularity_template.sub + apptainer: + submission_template: templates/localhost_apptainer_template.sub submission_command: sh - singularity_args: -B /ext:/ext + apptainer_args: -B /ext:/ext docker: submission_template: templates/localhost_docker_template.sub submission_command: sh diff --git a/tests/divvytests/data/divcfg-master/templates/localhost_apptainer_template.sub b/tests/divvytests/data/divcfg-master/templates/localhost_apptainer_template.sub new file mode 100644 index 00000000..e27c62b7 --- /dev/null +++ b/tests/divvytests/data/divcfg-master/templates/localhost_apptainer_template.sub @@ -0,0 +1,9 @@ +#!/bin/bash + +echo 'Compute node:' `hostname` +echo 'Start time:' `date +'%Y-%m-%d %T'` + +{ +apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image +apptainer exec instance://{JOBNAME}_image {CODE} +} | tee -i {LOGFILE} diff --git a/tests/divvytests/data/divcfg-master/templates/localhost_singularity_template.sub b/tests/divvytests/data/divcfg-master/templates/localhost_singularity_template.sub deleted file mode 100755 index a8fa2e3c..00000000 --- a/tests/divvytests/data/divcfg-master/templates/localhost_singularity_template.sub +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -echo 'Compute node:' `hostname` -echo 'Start time:' `date +'%Y-%m-%d %T'` - -{ -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -singularity exec instance://{JOBNAME}_image {CODE} -} | tee -i {LOGFILE} diff --git a/looper/default_config/divvy_templates/slurm_singularity_template.sub b/tests/divvytests/data/divcfg-master/templates/slurm_apptainer_template.sub old mode 100755 new mode 100644 similarity index 62% rename from looper/default_config/divvy_templates/slurm_singularity_template.sub rename to tests/divvytests/data/divcfg-master/templates/slurm_apptainer_template.sub index eb8359be..956f28fa --- a/looper/default_config/divvy_templates/slurm_singularity_template.sub +++ b/tests/divvytests/data/divcfg-master/templates/slurm_apptainer_template.sub @@ -11,7 +11,7 @@ echo 'Compute node:' `hostname` echo 'Start time:' `date +'%Y-%m-%d %T'` -singularity instance.start {SINGULARITY_ARGS} {SINGULARITY_IMAGE} {JOBNAME}_image -srun singularity exec instance://{JOBNAME}_image {CODE} +apptainer instance start {APPTAINER_ARGS} {APPTAINER_IMAGE} {JOBNAME}_image +srun apptainer exec instance://{JOBNAME}_image {CODE} -singularity instance.stop {JOBNAME}_image +apptainer instance stop {JOBNAME}_image diff --git a/tests/divvytests/data/divcfg-master/uva_rivanna.yaml b/tests/divvytests/data/divcfg-master/uva_rivanna.yaml index 1020d257..23bca64f 100644 --- a/tests/divvytests/data/divcfg-master/uva_rivanna.yaml +++ b/tests/divvytests/data/divcfg-master/uva_rivanna.yaml @@ -10,8 +10,8 @@ adapters: BULKER_CRATE: compute.bulker_crate DOCKER_ARGS: compute.docker_args DOCKER_IMAGE: compute.docker_image - SINGULARITY_IMAGE: compute.singularity_image - SINGULARITY_ARGS: compute.singularity_args + APPTAINER_IMAGE: compute.apptainer_image + APPTAINER_ARGS: compute.apptainer_args compute_packages: default: submission_template: templates/slurm_template.sub @@ -32,14 +32,14 @@ compute_packages: local: submission_template: templates/localhost_template.sub submission_command: . - singularity_slurm: - submission_template: templates/slurm_singularity_template.sub + apptainer_slurm: + submission_template: templates/slurm_apptainer_template.sub submission_command: sbatch - singularity_args: -B /sfs/lustre:/sfs/lustre,/nm/t1:/nm/t1 - singularity_local: - submission_template: templates/localhost_singularity_template.sub + apptainer_args: -B /sfs/lustre:/sfs/lustre,/nm/t1:/nm/t1 + apptainer_local: + submission_template: templates/localhost_apptainer_template.sub submission_command: . - singularity_args: -B /apps:/apps + apptainer_args: -B /apps:/apps bulker_slurm: submission_template: templates/slurm_bulker_template.sub submission_command: sbatch diff --git a/tests/divvytests/data/divcfg-master/uva_rivanna_old.yaml b/tests/divvytests/data/divcfg-master/uva_rivanna_old.yaml index 421620ab..6805b0e0 100644 --- a/tests/divvytests/data/divcfg-master/uva_rivanna_old.yaml +++ b/tests/divvytests/data/divcfg-master/uva_rivanna_old.yaml @@ -21,14 +21,14 @@ compute_packages: local: submission_template: templates/localhost_template.sub submission_command: sh - singularity_slurm: - submission_template: templates/slurm_singularity_template.sub + apptainer_slurm: + submission_template: templates/slurm_apptainer_template.sub submission_command: sbatch - singularity_args: -B /sfs/lustre:/sfs/lustre,/nm/t1:/nm/t1 - singularity_local: - submission_template: templates/localhost_singularity_template.sub + apptainer_args: -B /sfs/lustre:/sfs/lustre,/nm/t1:/nm/t1 + apptainer_local: + submission_template: templates/localhost_apptainer_template.sub submission_command: sh - singularity_args: -B /apps:/apps + apptainer_args: -B /apps:/apps bulker_slurm: submission_template: templates/slurm_bulker_template.sub submission_command: sbatch diff --git a/tests/divvytests/test_divvy_simple.py b/tests/divvytests/test_divvy_simple.py index 1d1586bf..58c9a213 100644 --- a/tests/divvytests/test_divvy_simple.py +++ b/tests/divvytests/test_divvy_simple.py @@ -1,3 +1,5 @@ +import os + import pytest from yacman import YAMLConfigManager @@ -9,6 +11,12 @@ # logmuse.init_logger("divvy", "DEBUG") +@pytest.fixture(autouse=True) +def _unset_divcfg(monkeypatch): + """Ensure tests use the built-in default config, not a user's DIVCFG.""" + monkeypatch.delenv("DIVCFG", raising=False) + + class TestPackageAtivation: def test_activate_package(self): dcc_filepath = select_divvy_config(None) @@ -27,9 +35,9 @@ class TestWriting: def test_write_script(self, tmp_path): dcc_filepath = select_divvy_config(None) dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) - dcc.activate_package("singularity_slurm") + dcc.activate_package("apptainer_slurm") extra_vars = { - "singularity_image": "simg", + "apptainer_image": "simg", "jobname": "jbname", "code": "mycode", } @@ -38,7 +46,7 @@ def test_write_script(self, tmp_path): with open(outfile, "r") as f: contents = f.read() assert contents.find("mycode") > 0 - assert contents.find("{SINGULARITY_ARGS}") < 0 + assert contents.find("{APPTAINER_ARGS}") < 0 class TestAdapters: @@ -49,7 +57,7 @@ class TestAdapters: YAMLConfigManager({"mem": 1000, "test": 0}), ], ) - @pytest.mark.parametrize("package", ["singularity_slurm", "slurm"]) + @pytest.mark.parametrize("package", ["apptainer_slurm", "slurm"]) def test_write_script_adapters(self, compute, package, tmp_path): """Test successful adapter sourcing from various Mapping types""" dcc_filepath = select_divvy_config(None) @@ -65,7 +73,7 @@ def test_write_script_adapters(self, compute, package, tmp_path): def test_adapters_overwritten_by_others(self, tmp_path): dcc_filepath = select_divvy_config(None) dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) - dcc.activate_package("singularity_slurm") + dcc.activate_package("apptainer_slurm") compute = YAMLConfigManager({"mem": 1000}) extra_vars = [{"compute": compute}, {"MEM": 333}] outfile = str(tmp_path / "test1.sub") @@ -74,3 +82,92 @@ def test_adapters_overwritten_by_others(self, tmp_path): contents = f.read() assert not (contents.find("1000") > 0) assert contents.find("333") > 0 + + +class TestPrePostCommand: + def test_pre_post_command_appear_in_script(self, tmp_path): + """Test PRE_COMMAND/POST_COMMAND appear in rendered script""" + dcc_filepath = select_divvy_config(None) + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) + dcc.activate_package("default") + extra_vars = { + "code": "echo hello", + "pre_command": "module load python", + "post_command": "echo done", + } + outfile = str(tmp_path / "test.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: + contents = f.read() + assert "module load python" in contents + assert "echo done" in contents + + def test_pre_post_command_default_to_empty(self, tmp_path): + """Test PRE_COMMAND/POST_COMMAND default to empty (no unreplaced placeholders)""" + dcc_filepath = select_divvy_config(None) + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) + dcc.activate_package("default") + extra_vars = {"code": "echo hello"} + outfile = str(tmp_path / "test.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: + contents = f.read() + assert "{PRE_COMMAND}" not in contents + assert "{POST_COMMAND}" not in contents + + +class TestBulkerSlurm: + def test_bulker_slurm_package(self, tmp_path): + """Test bulker_slurm package has sbatch and bulker activate""" + dcc_filepath = select_divvy_config(None) + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) + dcc.activate_package("bulker_slurm") + assert dcc.compute["submission_command"] == "sbatch" + extra_vars = { + "code": "echo hello", + "bulker_crate": "mycrate", + } + outfile = str(tmp_path / "test.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: + contents = f.read() + assert "SBATCH" in contents + assert "bulker activate" in contents + + +class TestApptainer: + def test_apptainer_package(self, tmp_path): + """Test apptainer package uses apptainer commands, not singularity""" + dcc_filepath = select_divvy_config(None) + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) + dcc.activate_package("apptainer") + extra_vars = { + "code": "echo hello", + "apptainer_image": "myimage.sif", + "jobname": "testjob", + } + outfile = str(tmp_path / "test.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: + contents = f.read() + assert "apptainer" in contents + assert "singularity" not in contents + + def test_apptainer_slurm_package(self, tmp_path): + """Test apptainer_slurm has SLURM headers and apptainer commands""" + dcc_filepath = select_divvy_config(None) + dcc = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath) + dcc.activate_package("apptainer_slurm") + assert dcc.compute["submission_command"] == "sbatch" + extra_vars = { + "code": "echo hello", + "apptainer_image": "myimage.sif", + "jobname": "testjob", + } + outfile = str(tmp_path / "test.sub") + dcc.write_script(outfile, extra_vars) + with open(outfile, "r") as f: + contents = f.read() + assert "SBATCH" in contents + assert "apptainer" in contents + assert "singularity" not in contents From 64e32dfe7f60799eeba684e256b64349805d89e7 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Mar 2026 10:01:57 -0400 Subject: [PATCH 6/8] format --- tests/divvytests/test_divvy_simple.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/divvytests/test_divvy_simple.py b/tests/divvytests/test_divvy_simple.py index 58c9a213..66a6bfe8 100644 --- a/tests/divvytests/test_divvy_simple.py +++ b/tests/divvytests/test_divvy_simple.py @@ -1,4 +1,3 @@ -import os import pytest from yacman import YAMLConfigManager From 45dfc6684d1e32a23598bfd4f4f23d344b792ee9 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Mar 2026 14:44:15 -0400 Subject: [PATCH 7/8] fix lint --- .github/workflows/{black.yml => lint.yml} | 0 pyproject.toml | 1 + tests/divvytests/test_divvy_simple.py | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{black.yml => lint.yml} (100%) diff --git a/.github/workflows/black.yml b/.github/workflows/lint.yml similarity index 100% rename from .github/workflows/black.yml rename to .github/workflows/lint.yml diff --git a/pyproject.toml b/pyproject.toml index 346903d3..ac7b66d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ ignore = ["F403", "F405", "E501"] [tool.ruff.lint.per-file-ignores] "looper/__init__.py" = ["E402", "F401"] "looper/looper.py" = ["E402"] +"looper/conductor.py" = ["F821"] "looper/processed_project.py" = ["E402", "F821"] "tests/**" = ["F841", "E712", "E722"] diff --git a/tests/divvytests/test_divvy_simple.py b/tests/divvytests/test_divvy_simple.py index 66a6bfe8..af69e8df 100644 --- a/tests/divvytests/test_divvy_simple.py +++ b/tests/divvytests/test_divvy_simple.py @@ -1,4 +1,3 @@ - import pytest from yacman import YAMLConfigManager From d0e3887bfc314c170627574381d3d81e76bee76e Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Mar 2026 15:17:32 -0400 Subject: [PATCH 8/8] version bump --- changelog.md | 482 ------------------------------------------------- pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 483 deletions(-) delete mode 100644 changelog.md diff --git a/changelog.md b/changelog.md deleted file mode 100644 index c77934a9..00000000 --- a/changelog.md +++ /dev/null @@ -1,482 +0,0 @@ -# Changelog - -This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. - -## [2.1.0] -- 2026-02-25 -### Changed -- Migrated to new yacman API (`YAMLConfigManager.from_yaml_file()`, `write_lock`/`read_lock` context managers); requires yacman >=0.9.5 -- Migrated CLI configuration to pydantic-settings -- Improved CLI startup time by deferring heavy imports to module level -- Made signal handling thread-safe -- Replaced wildcard imports with explicit imports -- Converted docstrings to Google style -- Better shell inference for submission commands (#282) -- Updated pipestat constructor usage to classmethod format -- Separated fast unit tests from slow CLI integration tests - -### Added -- HTTP API server (experimental/alpha) with FastAPI: `looper serve` -- Stricter looper-pipestat interface validation - -### Removed -- Old documentation files (moved to separate docs site) - -## [2.0.3] -- 2025-09-23 -### Fixed -- Fixed [#543](https://github.com/pepkit/looper/issues/543) -- Fixed [#547](https://github.com/pepkit/looper/issues/547) -- Fixed [#548](https://github.com/pepkit/looper/issues/548) - - -## [2.0.2] -- 2025-09-22 -### Changed -- Remove veracitools dependency from the requirements. - -## [2.0.1] -- 2025-03-05 - -### Changed -- update ubiquerg>=0.8.1 - -### Fixed -- [#541](https://github.com/pepkit/looper/issues/541) - -## [2.0.0] -- 2025-01-16 - -This release breaks backwards compatibility for Looper versions < 2.0.0 - -### Fixed -- divvy init [#520](https://github.com/pepkit/looper/issues/520) -- replaced deprecated PEPHubClient function, `_load_raw_pep` with `.load_raw_pep` -- looper cli parameters now take priority as originally intended [#518](https://github.com/pepkit/looper/issues/518) -- fix divvy inspect -- remove printed dictionary at looper finish [#511](https://github.com/pepkit/looper/issues/511) -- fix [#536](https://github.com/pepkit/looper/issues/536) -- fix [#522](https://github.com/pepkit/looper/issues/522) -- fix [#537](https://github.com/pepkit/looper/issues/537) -- fix [#534](https://github.com/pepkit/looper/issues/534) - -### Changed -- `--looper-config` is now `--config`, `-c`. [#455](https://github.com/pepkit/looper/issues/455) -- A pipeline interface now consolidates a `sample_interface` and a `project_interface` [#493](https://github.com/pepkit/looper/issues/493) -- Updated documentation for Looper 2.0.0, removing previous versions [pepspec PR #34](https://github.com/pepkit/pepspec/pull/34) -- remove position based argument for divvy config, must use --config or run as default config - -### Added -- `looper init` tutorial [#466](https://github.com/pepkit/looper/issues/466) -- looper config allows for `pephub_path` in pipestat config section of `.looper.yaml` [#519](https://github.com/pepkit/looper/issues/519) -- improve error messaging for bad/malformed looper configurations [#515](https://github.com/pepkit/looper/issues/515) -- add shortform argument for --package (alias is now -p) - - -## [1.9.1] -- 2024-07-18 - -### Changed -- ensure peppy requirement peppy>=0.40.0,<=0.40.2 - -## [1.9.0] -- 2024-06-26 - -### Added -- user can now add cli modifiers to looper config instead of PEP project [#270](https://github.com/pepkit/looper/issues/270) -- pipeline interfaces no longer must be nested under sample and project keys within looper config file [#465](https://github.com/pepkit/looper/issues/465) -- var_templates can now be hierarchical [#334](https://github.com/pepkit/looper/issues/334) -- looper can now gracefully halt spawned subprocesses when the user sends a keyboard interrupt [#37](https://github.com/pepkit/looper/issues/37) - -## [1.8.1] -- 2024-06-06 - -### Fixed -- added `-v` and `--version` to the CLI -- fixed running project level with `--project` argument - -## [1.8.0] -- 2024-06-04 - -### Added -- looper destroy now destroys individual results when pipestat is configured: https://github.com/pepkit/looper/issues/469 -- comprehensive smoketests: https://github.com/pepkit/looper/issues/464 -- allow rerun to work on both failed or waiting flags: https://github.com/pepkit/looper/issues/463 - -### Changed -- Migrated `argparse` CLI definition to a pydantic basis for all commands. See: https://github.com/pepkit/looper/issues/438 -- during project load, check if PEP file path is a file first, then check if it is a registry path: https://github.com/pepkit/looper/issues/456 -- Looper now uses FutureYamlConfigManager due to the yacman refactor v0.9.3: https://github.com/pepkit/looper/issues/452 - -### Fixed -- inferring project name when loading PEP from csv: https://github.com/pepkit/looper/issues/484 -- fix inconsistency resolving pipeline interface paths if multiple paths are supplied: https://github.com/pepkit/looper/issues/474 -- fix bug with checking for completed flags: https://github.com/pepkit/looper/issues/470 -- fix looper destroy not properly destroying all related files: https://github.com/pepkit/looper/issues/468 -- looper rerun now only runs failed jobs as intended: https://github.com/pepkit/looper/issues/467 -- looper inspect now inspects the looper config: https://github.com/pepkit/looper/issues/462 -- Load PEP from CSV: https://github.com/pepkit/looper/issues/456 -- looper now works with sample_table_index https://github.com/pepkit/looper/issues/458 - -## [1.7.1] -- 2024-05-28 - -### Fixed -- pin pipestat version to be between pipestat>=0.8.0,<0.9.0 https://github.com/pepkit/looper/issues/494 - -## [1.7.0] -- 2024-01-26 - -### Added -- `--portable` flag to `looper report` to create a portable version of the html report -- `--lump-j` allows grouping samples into a defined number of jobs - -### Changed -- `--lumpn` is now `--lump-n` -- `--lump` is now `--lump-s` - -## [1.6.0] -- 2023-12-22 - -### Added -- `looper link` creates symlinks for results grouped by record_identifier. It requires pipestat to be configured. [#72](https://github.com/pepkit/looper/issues/72) -- basic tab completion. - -### Changed -- looper now works with pipestat v0.6.0 and greater. -- `looper table`, `check` now use pipestat and therefore require pipestat configuration. [#390](https://github.com/pepkit/looper/issues/390) -- changed how looper configures pipestat [#411](https://github.com/pepkit/looper/issues/411) -- initializing pipeline interface also writes an example `output_schema.yaml` and `count_lines.sh` pipeline - -### Fixed -- filtering via attributes that are integers. - -## [1.5.1] -- 2023-08-14 - -### Fixed -- fix `looper table` failing without `sample.protocol` - -### Changed -- correct `--looper_conifg` to `--looper-config` - -## [1.5.0] -- 2023-08-09 - -### Added - -- ability to use PEPs from PEPhub without downloading project [#341](https://github.com/pepkit/looper/issues/341) -- ability to specify pipeline interfaces inside looper config [Looper Config](https://looper.databio.org/en/dev/how_to_define_looper_config/) -- divvy re-integrated in looper -- divvy inspect -p package -- Looper will now check that the command path provided in the pipeline interface is callable before submitting. - - -### Changed -- initialization of generic pipeline interface available using subcommand `init-piface` -- `looper report` will now use pipestat to generate browsable HTML reports if pipestat is configured. -- looper now works with pipestat v0.5.0. -- Removed --toggle-key functionality. -- Allow for user to input single integer value for --sel-incl or --sel-excl - -## [1.4.3] -- 2023-08-01 - -### Fixed -- Fix regression for var_templates expansion. - -## [1.4.2] -- 2023-07-31 - -### Fixed -- Fix for expanding paths properly. - -## [1.4.1] -- 2023-06-22 - - -## [1.4.0] -- 2023-04-24 - -### Added - -- preliminary support for [pipestat](http://pipestat.databio.org). -- ability to skip samples using `-k` or `--skip` [#367](https://github.com/pepkit/looper/pull/367) -- ability to input a range into `limit` and `skip`[#367](https://github.com/pepkit/looper/pull/367) -- `limit` and `skip` are now both usable with Destroy and Run. [#367](https://github.com/pepkit/looper/pull/367) -- ability to generate generic pipeline interface using `init -p` or `init --piface` [#368](https://github.com/pepkit/looper/pull/368) -- Fixed ability to use custom sample index -- Added `write_custom_template`, a built-in pre-submit plugin for writing templates - -### Changed -- looper now returns nonzero if any samples fail submission -- various other developer changes - -### Deprecated -- `path` variable will be deprecated in favor of `var_templates` [#322](https://github.com/pepkit/looper/issues/322) - -## [1.3.2] -- 2022-02-09 - -### Changed -- Fixed bug with use_2to3 for setuptools compatibility. - -## [1.3.1] -- 2021-06-18 - -### Changed -- If remote schemas are not accessible, the job submission doesn't fail anymore -- Fixed a bug where looper stated "No failed flag found" when a failed flag was found - -### Deprecated -- Fixed and deprecated `looper inspect`. Use `eido inspect` from now on. - - -## [1.3.0] -- 2020-10-07 - -### Added -- New plugin system for pre-submission hooks -- Included plugin functions: `write_sample_yaml`, `write_sample_yaml_prj`, `write_sample_yaml_cwl` and `write_submission_yaml` -- New `var_templates` section for defining variables in the pipeline interface - -### Changed -- Pipeline interface specification was updated to accommodate new `var_templates` section and pre-submission hooks - -### Deprecated -- pipeline interface sections: - - `dynamic_variables_command_template`, which can now be more simply accomplished with a pre-submission hook - - `path`, which is replaced by a more generic `var_templates` section - -## [1.2.1] - 2020-08-26 - -### Added -- Environment variables expansion in custom sample YAML paths; [Issue 273](https://github.com/pepkit/looper/issues/273) -- `dynamic_variables_script_path` key in the pipeline interface. Path, absolute or relative to the pipeline interface file; [Issue 276](https://github.com/pepkit/looper/issues/276) -### Changed -- Resolve project pipeline interface path by making it relative to the config not current directory; [Issue 268](https://github.com/pepkit/looper/issues/268) -### Fixed -- Unclear error when `output_dir` was not provided in a config `looper` section; [Issue 286](https://github.com/pepkit/looper/issues/286) - -## [1.2.0] - 2020-05-26 - -**This version introduced backwards-incompatible changes.** - -### Added -- Commands: - - `init`; initializes `.looper.yaml` file - - `inspect`; inspects `Project` or `Sample` objects - - `table`; writes summary stats table - - `runp`; runs project level pipelines -- Input schemas and output schemas -- `--settings` argument to specify compute resources as a YAML file -- Option to preset CLI options in a dotfile -- `--command-extra` and `--command-extra-override` arguments that append specified string to pipeline commands. These functions supercede the previous `pipeline_config` and `pipeline_args` sections, which are now deprecated. The new method is more universal, and can accomplish the same functionality but more simply, using the built-in PEP machinery to selectively apply commands to samples. -- Option to specify destination of sample YAML in pipeline interface -- `--pipeline_interfaces` argument that allows pipeline interface specification via CLI - -### Changed -- `looper summarize` to `looper report` -- Pipeline interface format changed drastically -- The PyPi name changed from 'loopercli' to 'looper' -- resources section in pipeline interface replaced with `size_dependent_attributes` or `dynamic_variables_command_template`. -- `--compute` can be used to specify arguments other than resources -- `all_input_files` and `required_input_files` keys in pipeline interface moved to the input schema and renamed to `files` and `required_files` -- pipeline interface specification - -## [0.12.6] -- 2020-02-21 - -### Added -- possibility to execute library module as a script: `python -m looper ...` - -### Changed -- in the summary page account for missing values when plotting; the value is disregarded in such a case and plot is still created -- show 50 rows in the summary table -- make links to the summary page relative -- long entries in the sample stats table are truncated with an option to see original value in a popover - -### Fixed -- inactive jQuery dependent components in the status page -- project objects layout in the summary index page -- inactivation of popovers after Bootstrap Table events -- non-homogeneous status flags appearance - -## [0.12.5] -- 2019-12-13 -### Changed -- reduce verbosity of missing options; [Issue 174](https://github.com/pepkit/looper/issues/174) -- switch to [Bootstrap Table](https://bootstrap-table.com/) in the summary index page table and sample status tables - -## [0.12.4] -- 2019-07-18 -### Added -- Ability to declare `required_executables` in a `PipelineInterface`, to trigger a naive "runnability" check for a sample submission -- A possibility to opt out of status page inclusion in the navbar - -### Changed -- The status tables now use DataTables jQuery plugin to make them interactive - -### Fixed -- Navbar links creation - -## [0.12.3] -- 2019-06-20 -### Fixed -- Bug in `Sample` YAML naming, whereby a base `Sample` was being suffixed as a subtype would be, leading to a pipeline argument based on `yaml_file` that did not exist on disk. - -## [0.12.2] -- 2019-06-06 - -### Fixed -- Fixed various bugs related to populating derived attributes, including using attributes like `sample_name` as keys. -- Fixed a bug related to singularity attributes not being passed from a pipeline interface file. -- Fixed several bugs with incorrect version requirements. - -## [0.12.1] -- 2019-05-20 - -### Added -- Made `looper.Sample` include more specific functionality from `peppy` - -### Changed -- Status table creation is possible outside of `looper`. -- In the summary index page the plottable columns list is now scrollable -- Status page relies on the `profile.tsv` file rather than `*.log`; [Issue 159](https://github.com/pepkit/looper/issues/159) - -### Fixed -- In HTML reporting module, do not ignore objects which are neither HTMLs nor images in the summary, e.g. CSVs -- Restore parsing and application of pipeline-level computing resource specification from a pipeline interface file; [Issue 184](https://github.com/pepkit/looper/issues/184) -- Allow `ignore_flags` to properly modulate submission messaging; [Issue 179](https://github.com/pepkit/looper/issues/179) -- Do not display time-like summary columns as the plottable ones; [Issue 182](https://github.com/pepkit/looper/issues/182) - -## [0.12.0] -- 2019-05-03 - -### Added -- First implementation of pipeline interface 'outputs', so pipeline authors can specify items of interest produced by the pipeline. -- Functions and attributes on `Project` to support "outputs" (`interfaces`, `get_interfaces`, `get_outputs`) - -### Changed -- Start "compute" --> "compute_packages" transition -- `get_logger` moved to `peppy` - -### Fixed -- Prevent CLI option duplication in pipeline commands generated -- Make functional CLI spec of particular attribute on which to base selection of a subset of a project's samples ([`peppy` 298](https://github.com/pepkit/peppy/issues/298)) - -## [0.11.1] -- 2019-04-17 - -### Changed -- Improved documentation -- Improved interaction with `peppy` and `divvy` dependencies - -## [0.11] -- 2019-04-17 - -### Added -- Implemented `looper rerun` command. -- Support use of custom `resources` in pipeline's `compute` section -- Listen for itemized compute resource specification on command-line with `--resources` -- Support pointing to `Project` config file with folder path rather than full filepath -- Add `selector-attribute` parameter for more generic sample selection. - -### Changed -- Switched to a Jinja-style templating system for summary output -- Made various UI changes to adapt to `caravel` use. -- Using `attmap` for "attribute-style key-vale store" implementation -- Removed Python 3.4 support. -- UI: change parameter names `in/exclude-samples` to `selector-in/exclude`. - -## [0.10.0] -- 2018-12-20 - -### Changed -- `PipelineInterface` now derives from `peppy.AttributeDict`. -- On `PipelineInterface`, iteration over pipelines now is with `iterpipes`. -- Rename `parse_arguments` to `build_parser`, which returns `argparse.ArgumentParser` object -- Integers in HTML reports are made more human-readable by including commas. -- Column headers in HTML reports are now stricly for sorting; there's a separate list for plottable columns. -- More informative error messages -- HTML samples list is fully populated. -- Existence of an object lacking an anchor image is no longer problematic for `summarize`. -- Basic package test in Python 3 now succeeds: `python3 setup.py test`. - -## [v0.9.2] -- 2018-11-12 - -### Changed -- Fixed bugs with `looper summarize` when no summarizers were present -- Added CLI flag to force `looper destroy` for programmatic access -- Fixed a bug for samples with duplicate names -- Added new display features (graphs, table display) for HTML summary output. - - -## [0.9.1] -- 2018-06-30 - -### Changed -- Fixed several bugs with `looper summarize` that caused failure on edge cases. - -## [0.9.0] -- 2018-06-25 - -### Added -- Support for custom summarizers -- Add `allow-duplicate-names` command-line options -- Allow any variables in environment config files or other `compute` sections to be used in submission templates. This allows looper to be used with containers. -- Add nice universal project-level HTML reporting - -## [0.8.1] -- 2018-04-02 - -### Changed -- Minor documentation and packaging updates for first Pypi release. -- Fix a bug that incorrectly mapped protocols due to case sensitive issues -- Fix a bug with `report_figure` that made it output pandas code - - -## [0.8.0] -- 2018-01-19 - -### Changed -- Use independent `peppy` package, replacing `models` module for core data types. -- Integrate `ProtocolInterface` functionality into `PipelineInterface`. - -## [0.7.2] -- 2017-11-16 -### Changed -- Correctly count successful command submissions when not using `--dry-run`. - -## [0.7.1] -- 2017-11-15 - -### Changed -- No longer falsely display that there's a submission failure. -- Allow non-string values to be unquoted in the `pipeline_args` section. - -## [0.7] -- 2017-11-15 -### Added -- Add `--lump` and `--lumpn` options -- Catch submission errors from cluster resource managers -- Implied columns can now be derived -- Now protocols can be specified on the command-line `--include-protocols` -- Add rudimentary figure summaries -- Simplifies command-line help display -- Allow wildcard protocol_mapping for catch-all pipeline assignment -- Improve user messages -- New sample_subtypes section in pipeline_interface - -### Changed -- Sample child classes are now defined explicitly in the pipeline interface. Previously, they were guessed based on presence of a class extending Sample in a pipeline script. -- Changed 'library' key sample attribute to 'protocol' - -## [0.6] -- 2017-07-21 -### Added - - Add support for implied_column section of the project config file - - Add support for Python 3 - - Merges pipeline interface and protocol mappings. This means we now allow direct pointers to `pipeline_interface.yaml` files, increasing flexibility, so this relaxes the specified folder structure that was previously used for `pipelines_dir` (with `config` subfolder). - - Allow URLs as paths to sample sheets. - - Allow tsv format for sample sheets. - - Checks that the path to a pipeline actually exists before writing the submission script. - -### Changed -- Changed LOOPERENV environment variable to PEPENV, generalizing it to generic models -- Changed name of `pipelines_dir` to `pipeline_interfaces` (but maintained backwards compatibility for now). -- Changed name of `run` column to `toggle`, since `run` can also refer to a sequencing run. -- Relaxes many constraints (like resources sections, pipelines_dir columns), making project configuration files useful outside looper. This moves us closer to dividing models from looper, and improves flexibility. -- Various small bug fixes and dev improvements. -- Require `setuptools` for installation, and `pandas 0.20.2`. If `numexpr` is installed, version `2.6.2` is required. -- Allows tilde in `pipeline_interfaces` - -## [0.5] -- 2017-03-01 -### Added -- Add new looper version tracking, with `--version` and `-V` options and printing version at runtime -- Add support for asterisks in file paths -- Add support for multiple pipeline directories in priority order -- Revamp of messages make more intuitive output -- Colorize output -- Complete rehaul of logging and test infrastructure, using logging and pytest packages - -### Changed -- Removes pipelines_dir requirement for models, making it useful outside looper -- Small bug fixes related to `all_input_files` and `required_input_files` attributes -- More robust installation and more explicit requirement of Python 2.7 - - -## [0.4] -- 2017-01-12 -### Added -- New command-line interface (CLI) based on sub-commands -- New subcommand (`looper summarize`) replacing the `summarizePipelineStats.R` script -- New subcommand (`looper check`) replacing the `flagCheck.sh` script -- New command (`looper destroy`) to remove all output of a project -- New command (`looper clean`) to remove intermediate files of a project flagged for deletion -- Support for portable and pipeline-independent allocation of computing resources with Looperenv. - -### Changed -- Removed requirement to have `pipelines` repository installed in order to extend base Sample objects -- Maintenance of sample attributes as provided by user by means of reading them in as strings (to be improved further) -- Improved serialization of Sample objects diff --git a/pyproject.toml b/pyproject.toml index ac7b66d8..fbd9b755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "looper" -version = "2.1.0" +version = "2.1.1" description = "A pipeline submission engine that parses sample inputs and submits pipelines for each sample." readme = "README.md" license = "BSD-2-Clause"