Refactor notebooks and badges checks into pre-commit hooks; check-badges fix Colab header

.github/workflows/checks.yml

@@ -20,8 +20,8 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v5
+    - run: pip install .
     - run: pip install pylint
-    - run: pip install -r requirements.txt
     - run: pylint --unsafe-load-any-extension=y --disable=fixme $(git ls-files '*.py')
 
   precommit:
@@ -30,6 +30,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
       - run: |
+          pip install .
           pip install pre-commit
           pre-commit clean
           pre-commit autoupdate
@@ -47,8 +48,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install -r requirements.txt
+      - run: pip install .
       - env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: pytest -vv -rP -We .
-

.pre-commit-config.yaml

@@ -1,20 +1,13 @@
-files: '.py'
-exclude: '.git'
 default_stages: [pre-commit]
 
 repos:
-  - repo: https://github.com/psf/black
-    rev: 25.1.0
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 25.9.0
     hooks:
       - id: black
 
-  - repo: https://github.com/timothycrosley/isort
-    rev: 6.0.1
-    hooks:
-      - id: isort
-
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -23,4 +16,25 @@ repos:
   - repo: https://github.com/christopher-hacker/enforce-notebook-run-order
     rev: 2.0.1
     hooks:
-      - id: enforce-notebook-run-order
+      - id: enforce-notebook-run-order
+
+  - repo: local
+    hooks:
+      - id: check-notebooks
+        name: check notebooks
+        entry: hooks/check_notebooks.py
+        additional_dependencies:
+          - nbformat
+        language: python
+        types: [jupyter]
+
+      - id: check-badges
+        name: check badges
+        entry: hooks/check_badges.py
+        additional_dependencies:
+          - "."
+          - nbformat
+          - pytest
+          - gitpython
+        language: python
+        types: [jupyter]

.pre-commit-hooks.yaml

@@ -0,0 +1,15 @@
+- id: check-notebooks
+  name: check notebooks
+  description: check Jupyter Notebook contents
+  entry: check_notebooks
+  language: python
+  stages: [pre-commit]
+  types: [jupyter]
+
+- id: check-badges
+  name: check badges
+  description: check badges in Jupyter Notebook
+  entry: check_badges
+  language: python
+  stages: [pre-commit]
+  types: [jupyter]

devops_tests/test_notebooks.py

@@ -0,0 +1,47 @@
+"""test execution of notebooks in the repository"""
+
+import gc
+import os
+import sys
+import warnings
+
+import nbformat
+import pytest
+
+from .utils import find_files
+
+if sys.platform == "win32" and sys.version_info[:2] >= (3, 7):
+    import asyncio
+
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore")
+    from nbconvert.preprocessors import ExecutePreprocessor
+
+
+@pytest.fixture(
+    params=find_files(file_extension=".ipynb"),
+    name="notebook_filename",
+)
+def _notebook_filename(request):
+    return request.param
+
+
+def test_run_notebooks(notebook_filename, tmp_path):
+    """executes a given notebook"""
+    os.environ["JUPYTER_PLATFORM_DIRS"] = "1"
+
+    executor = ExecutePreprocessor(timeout=15 * 60, kernel_name="python3")
+
+    with open(notebook_filename, encoding="utf8") as notebook_file:
+        # https://github.com/pytest-dev/pytest-asyncio/issues/212
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", message="There is no current event loop")
+            executor.preprocess(
+                nbformat.read(notebook_file, as_version=4),
+                {"metadata": {"path": tmp_path}},
+            )
+
+    # so that nbconvert perplexities are reported here, and not at some dtor test later on
+    gc.collect()

devops_tests/utils.py

@@ -0,0 +1,45 @@
+"""
+Utils functions to reuse in different parts of the codebase
+"""
+
+import os
+import pathlib
+
+from git import Git
+
+
+def find_files(path_to_folder_from_project_root=".", file_extension=None):
+    """
+    Returns all files in a current git repo.
+    The list of returned files may be filtered with `file_extension` param.
+    """
+    all_files = [
+        path
+        for path in Git(
+            Git(path_to_folder_from_project_root).rev_parse("--show-toplevel")
+        )
+        .ls_files()
+        .split("\n")
+        if os.path.isfile(path)
+    ]
+    if file_extension is not None:
+        return list(filter(lambda path: path.endswith(file_extension), all_files))
+
+    return all_files
+
+
+def relative_path(absolute_path):
+    """returns a path relative to the repo base (converting backslashes to slashes on Windows)"""
+    relpath = os.path.relpath(absolute_path, repo_path().absolute())
+    posixpath_to_make_it_usable_in_urls_even_on_windows = pathlib.Path(
+        relpath
+    ).as_posix()
+    return posixpath_to_make_it_usable_in_urls_even_on_windows
+
+
+def repo_path():
+    """returns absolute path to the repo base (ignoring .git location if in a submodule)"""
+    path = pathlib.Path(__file__)
+    while not (path.is_dir() and Git(path).rev_parse("--git-dir") == ".git"):
+        path = path.parent
+    return path

hooks/check_badges.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""
+Checks whether notebooks contain badges."""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from collections.abc import Sequence
+
+import nbformat
+
+repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.insert(0, repo_root)
+from devops_tests.utils import (  # pylint: disable=wrong-import-position
+    relative_path,
+    repo_path,
+)
+
+HEADER = f"""import os, sys
+os.environ['NUMBA_THREADING_LAYER'] = 'omp'  # PySDM and PyMPDATA are incompatible with TBB threads
+if 'google.colab' in sys.modules:
+    !pip --quiet install open-atmos-jupyter-utils
+    from open_atmos_jupyter_utils import pip_install_on_colab
+    pip_install_on_colab('{repo_path().name}-examples')"""
+
+HEADER_KEY_PATTERNS = [
+    "install open-atmos-jupyter-utils",
+    "google.colab",
+    "pip_install_on_colab",
+]
+
+
+def is_colab_header(cell_source: str) -> bool:
+    """Return True if the cell looks like a Colab header."""
+    return all(pat in cell_source for pat in HEADER_KEY_PATTERNS)
+
+
+def fix_colab_header(notebook_path):
+    """Check Colab-magic cell and fix if is misspelled, in wrong position or not exists"""
+    nb = nbformat.read(notebook_path, as_version=nbformat.NO_CONVERT)
+
+    header_index = None
+    for idx, cell in enumerate(nb.cells):
+        if cell.cell_type == "code" and is_colab_header(cell.source):
+            header_index = idx
+            break
+
+    modified = False
+    if header_index is not None:
+        if nb.cells[header_index].source != HEADER:
+            nb.cells[header_index].source = HEADER
+            modified = True
+        if header_index != 2:
+            nb.cells.insert(2, nb.cells.pop(header_index))
+            modified = True
+    else:
+        new_cell = nbformat.v4.new_code_cell(HEADER)
+        nb.cells.insert(2, new_cell)
+        modified = True
+    if modified:
+        nbformat.write(nb, notebook_path)
+    return modified
+
+
+def print_hook_summary(reformatted_files, unchanged_files):
+    """Print a Black-style summary."""
+    for f in reformatted_files:
+        print(f"\nreformatted {f}")
+
+    total_ref = len(reformatted_files)
+    total_unchanged = len(unchanged_files)
+    if total_ref > 0:
+        print("\nAll done! ✨ 🍰 ✨")
+        print(
+            f"{total_ref} file{'s' if total_ref != 1 else ''} reformatted, "
+            f"{total_unchanged} file{'s' if total_unchanged != 1 else ''} left unchanged."
+        )
+
+
+def _preview_badge_markdown(absolute_path):
+    svg_badge_url = (
+        "https://img.shields.io/static/v1?"
+        + "label=render%20on&logo=github&color=87ce3e&message=GitHub"
+    )
+    link = (
+        f"https://github.com/open-atmos/{repo_path().name}/blob/main/"
+        + f"{relative_path(absolute_path)}"
+    )
+    return f"[![preview notebook]({svg_badge_url})]({link})"
+
+
+def _mybinder_badge_markdown(absolute_path):
+    svg_badge_url = "https://mybinder.org/badge_logo.svg"
+    link = (
+        f"https://mybinder.org/v2/gh/open-atmos/{repo_path().name}.git/main?urlpath=lab/tree/"
+        + f"{relative_path(absolute_path)}"
+    )
+    return f"[![launch on mybinder.org]({svg_badge_url})]({link})"
+
+
+def _colab_badge_markdown(absolute_path):
+    svg_badge_url = "https://colab.research.google.com/assets/colab-badge.svg"
+    link = (
+        f"https://colab.research.google.com/github/open-atmos/{repo_path().name}/blob/main/"
+        + f"{relative_path(absolute_path)}"
+    )
+    return f"[![launch on Colab]({svg_badge_url})]({link})"
+
+
+def test_notebook_has_at_least_three_cells(notebook_filename):
+    """checks if all notebooks have at least three cells"""
+    with open(notebook_filename, encoding="utf8") as fp:
+        nb = nbformat.read(fp, nbformat.NO_CONVERT)
+        if len(nb.cells) < 3:
+            raise ValueError("Notebook should have at least 4 cells")
+
+
+def test_first_cell_contains_three_badges(notebook_filename):
+    """checks if all notebooks feature three badges in the first cell"""
+    with open(notebook_filename, encoding="utf8") as fp:
+        nb = nbformat.read(fp, nbformat.NO_CONVERT)
+    if nb.cells[0].cell_type != "markdown":
+        raise ValueError("First cell is not a markdown cell")
+    lines = nb.cells[0].source.split("\n")
+    if len(lines) != 3:
+        raise ValueError("First cell does not contain exactly 3 lines (badges)")
+    if lines[0] != _preview_badge_markdown(notebook_filename):
+        raise ValueError("First badge does not match Github preview badge")
+    if lines[1] != _mybinder_badge_markdown(notebook_filename):
+        raise ValueError("Second badge does not match MyBinder badge")
+    if lines[2] != _colab_badge_markdown(notebook_filename):
+        raise ValueError("Third badge does not match Colab badge")
+
+
+def test_second_cell_is_a_markdown_cell(notebook_filename):
+    """checks if all notebooks have their second cell with some markdown
+    (hopefully clarifying what the example is about)"""
+    with open(notebook_filename, encoding="utf8") as fp:
+        nb = nbformat.read(fp, nbformat.NO_CONVERT)
+    if nb.cells[1].cell_type != "markdown":
+        raise ValueError("Second cell is not a markdown cell")
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """collect failed notebook checks"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filenames", nargs="*", help="Filenames to check.")
+    args = parser.parse_args(argv)
+
+    failed_files = False
+    reformatted_files = []
+    unchanged_files = []
+    for filename in args.filenames:
+        try:
+            modified = fix_colab_header(filename)
+            if modified:
+                reformatted_files.append(str(filename))
+            else:
+                unchanged_files.append(str(filename))
+        except ValueError as exc:
+            print(f"[ERROR] {filename}: {exc}")
+            failed_files = True
+        try:
+            test_notebook_has_at_least_three_cells(filename)
+            test_first_cell_contains_three_badges(filename)
+            test_second_cell_is_a_markdown_cell(filename)
+
+        except ValueError as exc:
+            print(f"[ERROR] {filename}: {exc}")
+            failed_files = True
+
+    print_hook_summary(reformatted_files, unchanged_files)
+    return 1 if (reformatted_files or failed_files) else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())