diff --git a/docs/changelog/+release-at-exit.bugfix.rst b/docs/changelog/+release-at-exit.bugfix.rst new file mode 100644 index 00000000..a52fdacd --- /dev/null +++ b/docs/changelog/+release-at-exit.bugfix.rst @@ -0,0 +1,2 @@ +A lock still referenced when the interpreter exits is released from an ``atexit`` handler before module teardown, +except holds of daemon threads that are still running, and a ``thread_local=False`` lock while any other thread runs. diff --git a/docs/concepts.rst b/docs/concepts.rst index 14e5af27..3c68e1d0 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -619,6 +619,14 @@ writes ``filelock: fork child exits: ...`` to stderr and leaves through ``os._ex half-finished descriptor state, so no ``atexit`` handler or ``finally`` block runs in it. An operation that never finishes delays every later fork by the same 30 seconds. +On normal interpreter exit, an ``atexit`` handler registered when filelock is imported releases every lock the process +still references, before module teardown can break the release; handlers registered earlier run after it. Daemon +threads still run while ``atexit`` handlers do, so a hold of one that is alive stays in place, and a lock built with +``thread_local=False``, which cannot tell whose hold it has, stays held whenever any other thread is alive. +``os._exit``, a signal that kills the process and a crash skip the handler: native locks still go with the descriptor, +soft markers stay for contenders to reclaim, and a strict claim stays until +:meth:`force_break `. + Task cancellation ================= @@ -1106,7 +1114,8 @@ each thread gets its own context via ``threading.local``. This means: place it is ever set and ``lock.mode = ...`` raises ``AttributeError``. - The instance still keeps track of every thread's hold. ``release(force=True)`` also releases holds left by threads that exited, and garbage collecting the - lock releases every thread's hold. + lock releases every thread's hold. Interpreter exit releases a lock still + referenced, except holds of daemon threads that are still running. When ``thread_local=False``, all threads share the same context, including configuration values. This is useful for objects passed between threads or diff --git a/src/filelock/_api.py b/src/filelock/_api.py index f7c732b3..e9714215 100644 --- a/src/filelock/_api.py +++ b/src/filelock/_api.py @@ -39,9 +39,9 @@ from functools import cache, partial from itertools import count, starmap from numbers import Real -from threading import TIMEOUT_MAX, Condition, Event, RLock, Thread, current_thread, get_ident, local +from threading import TIMEOUT_MAX, Condition, RLock, Thread, active_count, current_thread, get_ident, local from typing import TYPE_CHECKING, Final, Generic, Literal, NoReturn, TypedDict, TypeVar, cast -from weakref import WeakKeyDictionary, WeakSet, WeakValueDictionary, ref +from weakref import WeakKeyDictionary, WeakValueDictionary, ref from ._error import ( SoftFileLockLifetimeWarning, @@ -666,8 +666,9 @@ def _supported_init_params( @contextmanager def fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Generator[None]: - # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. - if not _HAS_REGISTER_AT_FORK or sys.is_finalizing(): + # A daemon thread stopped inside this bookkeeping holds its lock through exit. Only the exit release and + # finalization skip it: daemon threads still run, and fork, while atexit handlers do. + if not _HAS_REGISTER_AT_FORK or interpreter_exiting(): yield # pragma: lacks fork return # pragma: lacks fork ensure_current_process() # pragma: needs fork @@ -697,6 +698,10 @@ def fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Gen _FORK_STATE.gate.notify_all() +def interpreter_exiting() -> bool: + return _EXIT_RELEASE.running or sys.is_finalizing() + + @dataclass(frozen=True, slots=True, kw_only=True) class _InitParameterModel: accepted_params: frozenset[str] @@ -916,7 +921,7 @@ def __init__( # ruff:ignore[too-many-arguments] # public constructor: one para def _adopt_settings(self, settings: LockSettings) -> None: self._creator_pid = os.getpid() - _LIVE_LOCKS.add(self) + _LIVE_LOCKS[id(self)] = self #: Holder id in the deadlock registry; a stand-in that finishes the finalizer has another id(self). self._holder_id = id(self) self._transition_lock = RLock() @@ -1405,7 +1410,9 @@ def release(self, force: bool = False) -> None: # ruff:ignore[boolean-type-hint :param force: If true, the lock counter is ignored and the lock is released in every case. On a thread-local lock it also releases the holds of threads that exited without releasing; a hold of a thread still running - stays with that thread. Dropping the last reference to the lock releases every thread's hold. + stays with that thread. Dropping the last reference to the lock releases every thread's hold. A lock + still referenced when the interpreter exits is released too, except holds of daemon threads still + running, and a ``thread_local=False`` lock while any other thread runs. """ counter_before: int | None = None @@ -1513,7 +1520,7 @@ def __del__( else: # Teardown may also have cleared the globals the release path reads, here or in a subclass's module. with _suppress(Exception): - self._release_abandoned_holds() + self._release_abandoned_holds(exited_only=False) def _release_once_unreachable(self) -> None: # The callback runs once the lock is gone, so it reads the state through a holder. The state alone keeps the @@ -1521,15 +1528,15 @@ def _release_once_unreachable(self) -> None: self._finalizer_state = _FinalizerState(state=self.__dict__) _UNREACHABLE_LOCKS.add(ref(self, partial(_finalize_unreachable, ref(self._finalizer_state), type(self)))) - def _release_abandoned_holds(self) -> None: + def _release_abandoned_holds(self, *, exited_only: bool) -> None: # A finalizer must not raise. A release error during garbage collection would otherwise surface as an # unraisable-exception warning, attributed to whichever code triggered collection. The dropped lock still gets # best-effort cleanup; an explicit release() reports the same error to a caller who can act on it. with contextlib.suppress(Exception): self.release(force=True) - # Nothing references the lock any more, so no thread can still be inside it through this instance. + # A dropped lock has no thread left inside it; at exit, a live daemon thread still is, so its hold stays. with contextlib.suppress(Exception): - self._release_thread_holds(exited_only=False) + self._release_thread_holds(exited_only=exited_only) @property def is_singleton(self) -> bool: @@ -1905,8 +1912,8 @@ def is_system_call_failure(error: BaseException) -> bool: def unregister_owned_descriptor(token: int) -> None: # pragma: needs fork - # No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever. - if sys.is_finalizing(): # pragma: no cover # only exit reaches it, after coverage stops measuring + # The exit release skips this bookkeeping for the same reason fork_transition does. + if interpreter_exiting(): # pragma: no cover # only exit reaches it, after coverage stops measuring return with fork_transition(), _FORK_STATE.registry_lock: _OWNED_DESCRIPTORS.pop(token, None) @@ -1977,23 +1984,37 @@ def acquire_thread_lock( #: Every lock this process built, so exit can release the ones still alive before teardown empties module globals. -_LIVE_LOCKS: Final[WeakSet[BaseFileLock]] = WeakSet() +#: Keyed by identity: a subclass can define __eq__ without __hash__. +_LIVE_LOCKS: Final[WeakValueDictionary[int, BaseFileLock]] = WeakValueDictionary() -#: Set once exit starts releasing live locks, after which no other thread's event loop runs a scheduled release. -_EXITING: Final[Event] = Event() +class _ExitRelease(local): + # Per thread, so a daemon thread still running at exit, and a child it forks, keep their fork bookkeeping. + running = False -def interpreter_exiting() -> bool: - return _EXITING.is_set() or sys.is_finalizing() +#: Marks the thread releasing live locks at exit, where no other thread's event loop runs a scheduled release. +_EXIT_RELEASE: Final[_ExitRelease] = _ExitRelease() def _release_live_locks() -> None: # A lock still referenced at exit is finalized only after module teardown, when the release path can no longer run. - _EXITING.set() pid = os.getpid() - for lock in [lock for lock in _LIVE_LOCKS if lock._creator_pid == pid]: # ruff:ignore[private-member-access] # ditto - lock._release_abandoned_holds() # ruff:ignore[private-member-access] # the module owns the locks it tracks + # Exit handlers run while daemon threads still do, so a hold one of them owns is kept. A shared lock cannot tell + # whose hold it has, so it is kept whenever another thread is alive. + sole_thread = active_count() == 1 + _EXIT_RELEASE.running = True + try: + for lock in [ + lock + for lock in _LIVE_LOCKS.values() + if lock._creator_pid == pid # ruff:ignore[private-member-access] # the module owns the locks it tracks + and (sole_thread or lock.is_thread_local()) + ]: + lock._release_abandoned_holds(exited_only=True) # ruff:ignore[private-member-access] # ditto + finally: + # A later exit handler may still fork, and its child is not exiting. + _EXIT_RELEASE.running = False atexit.register(_release_live_locks) diff --git a/src/filelock/_strict.py b/src/filelock/_strict.py index 77c1467b..f134b37c 100644 --- a/src/filelock/_strict.py +++ b/src/filelock/_strict.py @@ -384,13 +384,13 @@ def force_break(self, claim_name: str) -> None: if cleanup_error is not None: # pragma: needs dir-fd raise cleanup_error - def _release_abandoned_holds(self) -> None: + def _release_abandoned_holds(self, *, exited_only: bool) -> None: self._finalizing = True - super()._release_abandoned_holds() + super()._release_abandoned_holds(exited_only=exited_only) # release() skips a doorway whose cleanup failed, as it is no hold, so a dropped instance would strand claims. with contextlib.suppress(Exception): # GraalPy defers finalizers to the host collector, so its tests cannot drive this cleanup. - self._discard_pending_doorways(exited_only=False) # pragma: needs collected-finalization + self._discard_pending_doorways(exited_only=exited_only) # pragma: needs collected-finalization def _reconcile_failed_acquire(self, canonical: str) -> None: # The acquisition is over, so the next one resolves the working directory again rather than reuse this one's. diff --git a/src/filelock/asyncio.py b/src/filelock/asyncio.py index 53dd9329..a4019d7d 100644 --- a/src/filelock/asyncio.py +++ b/src/filelock/asyncio.py @@ -836,20 +836,21 @@ def loop(self) -> asyncio.AbstractEventLoop | None: """The event loop.""" return self._async_context.loop - def _release_abandoned_holds(self) -> None: + def _release_abandoned_holds(self, *, exited_only: bool) -> None: # Runs from garbage collection, where no event loop may be running. with contextlib.suppress(Exception): # A thread-local lock never uses an executor, so an in-place release matches the loop's and also reaches # threads whose loops it cannot schedule on. if self._settings.thread_local and not iscoroutinefunction(self._release): - self._release_without_loop() + self._release_without_loop(exited_only=exited_only) return try: loop = asyncio.get_running_loop() except RuntimeError: - # At interpreter exit daemon threads no longer run, so a loop running in one never runs the release. + # Once finalizing, daemon threads no longer run, so a loop running in one never runs the release. The + # exit release releases in place too: it skips a shared lock while any other thread is alive. if (loop := self._async_context.loop) is None or loop.is_closed() or _interpreter_exiting(): - self._release_without_loop() + self._release_without_loop(exited_only=exited_only) return if not loop.is_running(): loop.run_until_complete(self.release(force=True)) @@ -865,7 +866,7 @@ def _release_abandoned_holds(self) -> None: _FINALIZER_RELEASES.add(pending) pending.add_done_callback(self._settle_finalizer_release) - def _release_without_loop(self) -> None: + def _release_without_loop(self, *, exited_only: bool) -> None: # No loop can run a release here, so call a sync backend directly as BaseFileLock.__del__ does; a coroutine # backend cannot run and keeps its lock. if iscoroutinefunction(self._release): @@ -876,7 +877,7 @@ def _release_without_loop(self) -> None: finally: self._commit_release_if_released() # A thread-local lock also holds for the threads that ran its loops, and nothing else releases those now. - self._release_thread_holds(exited_only=False) + self._release_thread_holds(exited_only=exited_only) def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future[None]) -> None: _FINALIZER_RELEASES.discard(pending) @@ -885,7 +886,7 @@ def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future if pending.exception() is None: return with contextlib.suppress(Exception): - self._release_without_loop() + self._release_without_loop(exited_only=False) class AsyncAcquireReturnProxy: diff --git a/tests/test_filelock.py b/tests/test_filelock.py index d731fd18..99564380 100644 --- a/tests/test_filelock.py +++ b/tests/test_filelock.py @@ -601,6 +601,32 @@ def test_lock_held_at_exit_is_released_silently(lock_type: type[BaseFileLock], t pass +@pytest.mark.parametrize("lock_type", [FileLock, SoftFileLock]) +@pytest.mark.parametrize("thread_local", [pytest.param(True, id="thread-local"), pytest.param(False, id="shared")]) +def test_lock_a_daemon_thread_holds_at_exit_stays_held( + lock_type: type[BaseFileLock], tmp_path: Path, *, thread_local: bool +) -> None: + # Exit handlers run while daemon threads still do, so releasing their hold would let another process in mid-section. + lock_path = str(tmp_path / "a.lock") + contender = f"import filelock; filelock.{lock_type.__name__}({lock_path!r}).acquire(timeout=0)" + script = f""" +import atexit, subprocess, sys, threading +atexit.register(lambda: print(subprocess.run([sys.executable, "-c", {contender!r}], capture_output=True).returncode)) +import filelock +lock = filelock.{lock_type.__name__}({lock_path!r}, thread_local={thread_local}) +entered = threading.Event() +def hold(): + with lock: + entered.set() + threading.Event().wait() +threading.Thread(target=hold, daemon=True).start() +entered.wait() +""" + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=True) + + assert (result.stdout, result.stderr) == ("1\n", "") + + @pytest.mark.parametrize("lock_type", [FileLock, SoftFileLock]) @pytest.mark.parametrize( "drop", diff --git a/tests/test_fork_coordination.py b/tests/test_fork_coordination.py index 2e402f59..86996191 100644 --- a/tests/test_fork_coordination.py +++ b/tests/test_fork_coordination.py @@ -341,6 +341,40 @@ def stop_inside_fork_bookkeeping(): assert (result.returncode, result.stderr) == (0, "") +@NEEDS_REGISTER_AT_FORK # pragma: needs register-at-fork +def test_child_forked_by_a_daemon_thread_at_exit_keeps_its_fork_bookkeeping(tmp_path: Path) -> None: + # Daemon threads still run, and fork, while atexit handlers do; their children are not exiting. + script: Final = """ +import atexit, os, sys, threading +exiting, forked = threading.Event(), threading.Event() +atexit.register(lambda: (exiting.set(), forked.wait(10))) +from filelock import FileLock +from filelock._api import _OWNED_DESCRIPTORS + +def fork_during_exit(): + exiting.wait() + if (pid := os.fork()) == 0: + for _ in range(20): + with FileLock(sys.argv[1]): + pass + print(len(_OWNED_DESCRIPTORS), flush=True) + os._exit(0) + os.waitpid(pid, 0) + forked.set() + +threading.Thread(target=fork_during_exit, daemon=True).start() +""" + result: Final = subprocess.run( + [sys.executable, "-W", "ignore::DeprecationWarning", "-c", script, str(tmp_path / "a")], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + + assert (result.stdout, result.stderr) == ("0\n", "") + + @NEEDS_FORK # pragma: needs fork @NEEDS_AUDIT_EVENTS def test_import_adds_audit_hook_only_where_settrace_survives_it() -> None: