Add durations to pytest in CI.

.github/workflows/code_test_and_deploy.yml

@@ -7,11 +7,10 @@ on:
     tags:
       - 'v*'
   pull_request:
-
   schedule:
-    # Cron runs on the 1st and 15th of every month.
+    # Weekly cron job at 12:00 AM UTC on Mondays.
     # This will only run on main by default.
-    - cron: "0 0 1,15 * *"
+    - cron: '0 0 * * 1'
 
 jobs:
   linting:
@@ -20,7 +19,6 @@ jobs:
       - uses: neuroinformatics-unit/actions/lint@v2
 
   test:
-    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
     needs: [linting]
     name: ${{ matrix.os }} py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
@@ -30,18 +28,13 @@ jobs:
         shell: bash -l {0}
 
     strategy:
-      fail-fast: true
       matrix:
         # macos-14 is M1, macos-13 is intel. Run on earliest and
         # latest python versions. All python versions are tested in
         # the weekly cron job.
-         # Test all Python versions for cron job, and only first/last for other triggers
-        os: ${{ fromJson(github.event_name == 'schedule'
-              && '["ubuntu-latest","windows-latest","macos-14","macos-13"]'
-              || '["ubuntu-latest","windows-latest","macos-latest"]') }}
-        python-version: ${{ fromJson(github.event_name == 'schedule'
-              && '["3.9","3.10","3.11","3.12","3.13"]'
-              || '["3.9","3.13"]') }}
+        os: [ ubuntu-latest]  # , windows-latest, macos-14, macos-13
+        # Test all Python versions for cron job, and only first/last for other triggers
+        python-version: ${{ fromJson(github.event_name == 'schedule' && '["3.9", "3.10", "3.11", "3.12", "3.13"]' || '["3.9", "3.13"]') }}
 
     steps:
       - uses: actions/checkout@v5
@@ -73,7 +66,7 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           sudo service mysql stop  # free up port 3306 for ssh tests
-          pytest tests/tests_transfers/ssh
+          pytest --durations=0 tests/tests_transfers/ssh
 
       - name: Test Google Drive
         env:
@@ -82,7 +75,7 @@ jobs:
           GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }}
           GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }}
         run: |
-          pytest tests/tests_transfers/gdrive
+          pytest --durations=0 tests/tests_transfers/gdrive
 
       - name: Test AWS
         env:
@@ -91,11 +84,11 @@ jobs:
           AWS_REGION: ${{ secrets.AWS_REGION }}
           AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
         run: |
-          pytest tests/tests_transfers/aws
+          pytest --durations=0 tests/tests_transfers/aws
 
       - name: All Other Tests
         run: |
-          pytest --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws
+          pytest --durations=0 --ignore=tests/tests_transfers/ssh --ignore=tests/tests_transfers/gdrive --ignore=tests/tests_transfers/aws
 
 
   build_sdist_wheels:
@@ -113,7 +106,7 @@ jobs:
     if: github.event_name == 'push' && github.ref_type == 'tag'
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/download-artifact@v6
+    - uses: actions/download-artifact@v5
       with:
         name: artifact
         path: dist

.pre-commit-config.yaml

@@ -31,12 +31,12 @@ repos:
         - id: rst-directive-colons
         - id: rst-inline-touching-normal
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.14.3
+      rev: v0.12.11
       hooks:
         - id: ruff
         - id: ruff-format
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v1.18.2
+      rev: v1.17.1
       hooks:
           - id: mypy
             additional_dependencies:

datashuttle/__init__.py

@@ -1,7 +1,7 @@
 from importlib.metadata import PackageNotFoundError, version
 
 from datashuttle.datashuttle_class import DataShuttle
-from datashuttle.datashuttle_functions import validate_project_from_path
+from datashuttle.datashuttle_functions import quick_validate_project
 
 
 try:

datashuttle/configs/canonical_configs.py

@@ -14,6 +14,7 @@
     TYPE_CHECKING,
     Dict,
     List,
+    Literal,
     Optional,
     Union,
     get_args,
@@ -28,20 +29,21 @@
 from datashuttle.configs.aws_regions import AwsRegion
 from datashuttle.utils import folders, utils
 from datashuttle.utils.custom_exceptions import ConfigError
-from datashuttle.utils.custom_types import ConnectionMethods
 
+connection_methods = Literal["ssh", "local_filesystem", "gdrive", "aws"]
 
-def get_connection_methods_list() -> List:
+
+def get_connection_methods_list() -> List[str]:
     """Return the canonical connection methods."""
-    return list(get_args(ConnectionMethods))
+    return list(get_args(connection_methods))
 
 
 def get_canonical_configs() -> dict:
     """Return the only permitted types for DataShuttle config values."""
     canonical_configs = {
         "local_path": Union[str, Path],
         "central_path": Optional[Union[str, Path]],
-        "connection_method": Optional[ConnectionMethods],
+        "connection_method": Optional[connection_methods],
         "central_host_id": Optional[str],
         "central_host_username": Optional[str],
         "gdrive_client_id": Optional[str],
@@ -112,80 +114,73 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None:
 
     check_config_types(config_dict)
 
-    if config_dict["connection_method"] == "local_only":
-        if config_dict["central_path"] is not None:
-            utils.log_and_raise_error(
-                "Cannot set `central_path` when `connection_method` is 'local_only'.",
-                ConfigError,
-            )
-
-    elif config_dict["connection_method"] == "aws":
-        if config_dict["central_path"] is None:
-            utils.log_and_raise_error(
-                "`central_path` cannot be `None` when `connection_method` is 'aws'. "
-                "`central_path` must include the s3 bucket name.",
-                ConfigError,
-            )
+    if config_dict["connection_method"] != "gdrive":
         if (
-            not config_dict["aws_access_key_id"]
-            or not config_dict["aws_region"]
+            config_dict["connection_method"] == "aws"
+            and config_dict["central_path"] is None
         ):
             utils.log_and_raise_error(
-                "Both aws_access_key_id and aws_region must be present for AWS connection.",
+                "`central_path` cannot be `None` when `connection_method` is 'aws'. "
+                "`central_path` must include the s3 bucket name.",
                 ConfigError,
             )
+        else:
+            raise_on_bad_local_only_project_configs(config_dict)
 
-    elif config_dict["connection_method"] == "ssh":
-        if (
-            not config_dict["central_host_id"]
-            or not config_dict["central_host_username"]
-        ):
-            utils.log_and_raise_error(
-                "'central_host_id' and 'central_host_username' are "
-                "required if 'connection_method' is 'ssh'.",
-                ConfigError,
-            )
+    if list(config_dict.keys()) != list(canonical_dict.keys()):
+        utils.log_and_raise_error(
+            f"New config keys are in the wrong order. The"
+            f" order should be: {canonical_dict.keys()}.",
+            ConfigError,
+        )
+
+    raise_on_bad_path_syntax(
+        config_dict["local_path"].as_posix(), "local_path"
+    )
+
+    if config_dict["central_path"] is not None:
+        raise_on_bad_path_syntax(
+            config_dict["central_path"].as_posix(), "central_path"
+        )
+
+    # Check SSH settings
+    if config_dict["connection_method"] == "ssh" and (
+        not config_dict["central_host_id"]
+        or not config_dict["central_host_username"]
+    ):
+        utils.log_and_raise_error(
+            "'central_host_id' and 'central_host_username' are "
+            "required if 'connection_method' is 'ssh'.",
+            ConfigError,
+        )
 
+    # Check gdrive settings
     elif config_dict["connection_method"] == "gdrive":
         if not config_dict["gdrive_root_folder_id"]:
             utils.log_and_raise_error(
                 "'gdrive_root_folder_id' is required if 'connection_method' "
                 "is 'gdrive'.",
                 ConfigError,
             )
+
         if not config_dict["gdrive_client_id"]:
             utils.log_and_message(
                 "`gdrive_client_id` not found in config. default rlcone client will be used (slower)."
             )
 
-    if config_dict["connection_method"] in ["ssh", "local_filesystem"]:
-        if config_dict["central_path"] is None:
-            utils.log_and_raise_error(
-                f"`central_path` must be set if `connection_method` is {config_dict['connection_method']}",
-                ConfigError,
-            )
-
-    if list(config_dict.keys()) != list(canonical_dict.keys()):
+    # Check AWS settings
+    elif config_dict["connection_method"] == "aws" and (
+        not config_dict["aws_access_key_id"] or not config_dict["aws_region"]
+    ):
         utils.log_and_raise_error(
-            f"New config keys are in the wrong order. The"
-            f" order should be: {canonical_dict.keys()}.",
+            "Both aws_access_key_id and aws_region must be present for AWS connection.",
             ConfigError,
         )
 
-    raise_on_bad_path_syntax(
-        config_dict["local_path"].as_posix(), "local_path"
-    )
-
-    if config_dict["central_path"] is not None:
-        raise_on_bad_path_syntax(
-            config_dict["central_path"].as_posix(), "central_path"
-        )
-
     # Initialise the local project folder
     utils.print_message_to_user(
         f"Making project folder at: {config_dict['local_path']}"
     )
-
     try:
         folders.create_folders(config_dict["local_path"])
     except OSError:
@@ -196,6 +191,26 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None:
         )
 
 
+def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None:
+    """Check that both or neither of `central_path` and `connection_method` are set.
+
+    There is no circumstance where one is set and not the other. Either both are set
+    ('full' project) or both are `None` ('local only' project).
+    """
+    params_are_none = [
+        config_dict[key] is None
+        for key in ["central_path", "connection_method"]
+    ]
+
+    if any(params_are_none):
+        if not all(params_are_none):
+            utils.log_and_raise_error(
+                "Either both `central_path` and `connection_method` must be set, "
+                "or must both be `None` (for local-project mode).",
+                ConfigError,
+            )
+
+
 def raise_on_bad_path_syntax(
     path_name: str,
     path_type: str,
@@ -264,7 +279,6 @@ def get_tui_config_defaults() -> Dict:
             "overwrite_existing_files": "never",
             "dry_run": False,
             "suggest_next_sub_ses_central": False,
-            "allow_letters_in_sub_ses_values": False,
         }
     }
 
@@ -292,9 +306,9 @@ def get_tui_config_defaults() -> Dict:
     return settings
 
 
-def get_validation_templates_defaults() -> Dict:
-    """Return the default values for validation_templates."""
-    return {"validation_templates": {"on": False, "sub": None, "ses": None}}
+def get_name_templates_defaults() -> Dict:
+    """Return the default values for name_templates."""
+    return {"name_templates": {"on": False, "sub": None, "ses": None}}
 
 
 def get_persistent_settings_defaults() -> Dict:
@@ -306,7 +320,7 @@ def get_persistent_settings_defaults() -> Dict:
     """
     settings = {}
     settings.update(get_tui_config_defaults())
-    settings.update(get_validation_templates_defaults())
+    settings.update(get_name_templates_defaults())
 
     return settings
 

datashuttle/configs/config_class.py

@@ -62,6 +62,7 @@ def __init__(
 
         self.logging_path: Path
         self.hostkeys_path: Path
+        self.ssh_key_path: Path
         self.project_metadata_path: Path
 
     def setup_after_load(self) -> None:
@@ -161,9 +162,6 @@ def update_config_for_backward_compatability_if_required(
             for key in canonical_config_keys_to_add:
                 config_dict[key] = None
 
-        if config_dict["connection_method"] is None:
-            config_dict["connection_method"] = "local_only"
-
     # -------------------------------------------------------------------------
     # Utils
     # -------------------------------------------------------------------------
@@ -259,10 +257,6 @@ def get_rclone_config_name(
         if connection_method is None:
             connection_method = self["connection_method"]
 
-        assert connection_method != "local_only", (
-            "This state assumes a central connection."
-        )
-
         return f"central_{self.project_name}_{connection_method}"
 
     def make_rclone_transfer_options(
@@ -299,6 +293,8 @@ def init_paths(self) -> None:
             self.project_name
         )
 
+        self.ssh_key_path = datashuttle_path / f"{self.project_name}_ssh_key"
+
         self.hostkeys_path = datashuttle_path / "hostkeys"
 
         self.logging_path = self.make_and_get_logging_path()
@@ -344,4 +340,4 @@ def is_local_project(self):
         A project is 'local-only' if it has no `central_path` and `connection_method`.
         It can be used to make folders and validate, but not for transfer.
         """
-        return self["connection_method"] == "local_only"
+        return self["connection_method"] is None