From a433847e99265f7523abc2928243c8c790a80555 Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 16 Jul 2026 16:21:48 +0300 Subject: [PATCH] Add a framework to allow certain errors to silently fail the notebooks without causing the CI to fail --- tests/conftest.py | 18 ++++++ tests/utils_for_error_silencing.py | 92 ++++++++++++++++++++++++++++++ tests/utils_for_testbook.py | 22 ++++++- 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 tests/conftest.py create mode 100644 tests/utils_for_error_silencing.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..599c1d72b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,18 @@ +import os + +from tests.utils_for_error_silencing import COLLECTED_SILENCED_ERRORS + + +def pytest_sessionfinish(session, exitstatus): + summary_file = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_file: + return + + if not COLLECTED_SILENCED_ERRORS: + return + + with open(summary_file, "a") as f: + f.write("### Pytest Warnings :warning:\n\n```text\n") + for warning in COLLECTED_SILENCED_ERRORS: + f.write(f"{warning}\n") + f.write("```\n") diff --git a/tests/utils_for_error_silencing.py b/tests/utils_for_error_silencing.py new file mode 100644 index 000000000..1aaaba096 --- /dev/null +++ b/tests/utils_for_error_silencing.py @@ -0,0 +1,92 @@ +import logging +import re +from typing import Any + +from nbclient.exceptions import CellExecutionError # as type hint +from nbformat import NotebookNode # as type hint + +logger = logging.getLogger(__name__) + + +# a list of [error-name, regex-pattern-of-error-message] +# for example, to catch `ValueError("abcd")` add +# ("ValueError", "abcd") +# or +# ("ValueError", "....") +ALLOWED_ERRORS: list[tuple[str, str]] = [ + ("ClassiqAPIError", ".*\\b429\\b.*"), +] + +COLLECTED_SILENCED_ERRORS: list[str] = [] + + +def create_hooks(notebook_name): + # we have a function that defines 3 functions in order + # to make those functions share a 'global' variable + # but we want this variable to be global-per-notebook + # thus, each @wrap_testbook creates it's own triplet + + did_the_notebook_raise_an_error_we_wish_to_ignore = False + index = -1 + + def my_on_cell_error( + cell: NotebookNode, cell_index: int, execute_reply: dict[str, Any] + ): + # the caller is + # site-packages/nbclient/client.py:1062:async_execute_cell + # the caller verifies that + # - `execute_reply["content"]` is indeed a dict + # - `execute_reply["content"]["status"]` is indeed "error" + ename = execute_reply["content"].get("ename") + evalue = execute_reply["content"].get("evalue") + + for error_name, error_pattern in ALLOWED_ERRORS: + if ename == error_name and re.match(error_pattern, evalue): + nonlocal did_the_notebook_raise_an_error_we_wish_to_ignore + did_the_notebook_raise_an_error_we_wish_to_ignore = True + nonlocal index + index = cell_index + + logger.warning( + f"Silencing error: '{notebook_name}': '{error_name}'('{error_pattern}')" + ) + logger.info(f"error message (cell {cell_index}): '{evalue}'") + COLLECTED_SILENCED_ERRORS.append( + f"'{notebook_name}': '{error_name}'('{error_pattern}')" + ) + + break + else: + raise CellExecutionError.from_cell_and_msg(cell, execute_reply["content"]) + + def my_on_cell_start(cell: NotebookNode, cell_index: int): + if did_the_notebook_raise_an_error_we_wish_to_ignore: + # just some verification + print(cell_index > index) + + if cell.cell_type == "code": + cell.source = "" + + def maybe_skip_the_entire_test_if_an_expected_error_we_want_to_silence_was_raised( + func, + ): + # this will be a decorator for the test. + # We cannot have it be: `if ...: return lambda no-op ; else return func` + # since at the time in which the decorator decorates the test function, + # the notebook hasn't been run, thus we'll always get `did_..._raise = False` + # thus, we create a function that will be evaluated "in order of decorators" + # which promises that it will run after the `testbook` decorator + # thus making sure that the notebook finished running, and the error-silencing is done. + def evaluate_only_when_called(*args, **kwargs): + if did_the_notebook_raise_an_error_we_wish_to_ignore: + return None + else: + return func(*args, **kwargs) + + return evaluate_only_when_called + + return ( + my_on_cell_error, + my_on_cell_start, + maybe_skip_the_entire_test_if_an_expected_error_we_want_to_silence_was_raised, + ) diff --git a/tests/utils_for_testbook.py b/tests/utils_for_testbook.py index 99cffc4d1..f302830a2 100644 --- a/tests/utils_for_testbook.py +++ b/tests/utils_for_testbook.py @@ -18,6 +18,7 @@ resolve_notebook_path, should_skip_notebook, ) +from tests.utils_for_error_silencing import create_hooks, ALLOWED_ERRORS from classiq.interface.generator.quantum_program import QuantumProgram @@ -45,6 +46,11 @@ since before the decorator, the function takes 0 arguments and after the decorator, it takes 1 - `tb`. +5 - if the notebook raised an error we wish to ignore then skip the test +more documentation in the decorator implementation +note that this decorator can be placed pretty much everywhere post `testbook` +as it only requires it's inner function to be run after the notebook was executed + Other - replacements We allow running "regex replace" on the ipynb file, in order to ease the load on the tests. """ @@ -61,12 +67,26 @@ def inner_decorator(func: Callable) -> Any: notebook_path = resolve_notebook_path(notebook_name) + ( + my_on_cell_error, + my_on_cell_start, + maybe_skip_the_entire_test_if_an_expected_error_we_want_to_silence_was_raised, + ) = create_hooks(notebook_name) + with NotebookEdit( notebook_path, replacements_regex, replacements_variables ) as nr: for decorator in [ + maybe_skip_the_entire_test_if_an_expected_error_we_want_to_silence_was_raised, _build_patch_testbook_client_decorator(notebook_name), - testbook(notebook_path, execute=True, timeout=timeout_seconds), + testbook( + notebook_path, + execute=True, + timeout=timeout_seconds, + allow_error_names=[i[0] for i in ALLOWED_ERRORS], + on_cell_error=my_on_cell_error, + on_cell_start=my_on_cell_start, + ), _build_cd_decorator(notebook_path), _build_skip_decorator(notebook_path), ]: