Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/changelog/+release-at-exit.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
A lock still referenced when the interpreter exits is released from an ``atexit`` handler before module teardown,
except holds of daemon threads that are still running, and a ``thread_local=False`` lock while any other thread runs.
11 changes: 10 additions & 1 deletion docs/concepts.rst
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,14 @@ writes ``filelock: fork child exits: ...`` to stderr and leaves through ``os._ex
half-finished descriptor state, so no ``atexit`` handler or ``finally`` block runs in it. An operation that never
finishes delays every later fork by the same 30 seconds.

On normal interpreter exit, an ``atexit`` handler registered when filelock is imported releases every lock the process
still references, before module teardown can break the release; handlers registered earlier run after it. Daemon
threads still run while ``atexit`` handlers do, so a hold of one that is alive stays in place, and a lock built with
``thread_local=False``, which cannot tell whose hold it has, stays held whenever any other thread is alive.
``os._exit``, a signal that kills the process and a crash skip the handler: native locks still go with the descriptor,
soft markers stay for contenders to reclaim, and a strict claim stays until
:meth:`force_break <filelock.StrictSoftFileLock.force_break>`.

Task cancellation
=================

Expand Down Expand Up @@ -1106,7 +1114,8 @@ each thread gets its own context via ``threading.local``. This means:
place it is ever set and ``lock.mode = ...`` raises ``AttributeError``.
- The instance still keeps track of every thread's hold. ``release(force=True)``
also releases holds left by threads that exited, and garbage collecting the
lock releases every thread's hold.
lock releases every thread's hold. Interpreter exit releases a lock still
referenced, except holds of daemon threads that are still running.

When ``thread_local=False``, all threads share the same context, including
configuration values. This is useful for objects passed between threads or
Expand Down
61 changes: 41 additions & 20 deletions src/filelock/_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@
from functools import cache, partial
from itertools import count, starmap
from numbers import Real
from threading import TIMEOUT_MAX, Condition, Event, RLock, Thread, current_thread, get_ident, local
from threading import TIMEOUT_MAX, Condition, RLock, Thread, active_count, current_thread, get_ident, local
from typing import TYPE_CHECKING, Final, Generic, Literal, NoReturn, TypedDict, TypeVar, cast
from weakref import WeakKeyDictionary, WeakSet, WeakValueDictionary, ref
from weakref import WeakKeyDictionary, WeakValueDictionary, ref

from ._error import (
SoftFileLockLifetimeWarning,
Expand Down Expand Up @@ -666,8 +666,9 @@ def _supported_init_params(

@contextmanager
def fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Generator[None]:
# No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever.
if not _HAS_REGISTER_AT_FORK or sys.is_finalizing():
# A daemon thread stopped inside this bookkeeping holds its lock through exit. Only the exit release and
# finalization skip it: daemon threads still run, and fork, while atexit handlers do.
if not _HAS_REGISTER_AT_FORK or interpreter_exiting():
yield # pragma: lacks fork
return # pragma: lacks fork
ensure_current_process() # pragma: needs fork
Expand Down Expand Up @@ -697,6 +698,10 @@ def fork_transition(descriptor_owner: _ForkDescriptorOwner | None = None) -> Gen
_FORK_STATE.gate.notify_all()


def interpreter_exiting() -> bool:
return _EXIT_RELEASE.running or sys.is_finalizing()


@dataclass(frozen=True, slots=True, kw_only=True)
class _InitParameterModel:
accepted_params: frozenset[str]
Expand Down Expand Up @@ -916,7 +921,7 @@ def __init__( # ruff:ignore[too-many-arguments] # public constructor: one para

def _adopt_settings(self, settings: LockSettings) -> None:
self._creator_pid = os.getpid()
_LIVE_LOCKS.add(self)
_LIVE_LOCKS[id(self)] = self
#: Holder id in the deadlock registry; a stand-in that finishes the finalizer has another id(self).
self._holder_id = id(self)
self._transition_lock = RLock()
Expand Down Expand Up @@ -1405,7 +1410,9 @@ def release(self, force: bool = False) -> None: # ruff:ignore[boolean-type-hint

:param force: If true, the lock counter is ignored and the lock is released in every case. On a thread-local
lock it also releases the holds of threads that exited without releasing; a hold of a thread still running
stays with that thread. Dropping the last reference to the lock releases every thread's hold.
stays with that thread. Dropping the last reference to the lock releases every thread's hold. A lock
still referenced when the interpreter exits is released too, except holds of daemon threads still
running, and a ``thread_local=False`` lock while any other thread runs.

"""
counter_before: int | None = None
Expand Down Expand Up @@ -1513,23 +1520,23 @@ def __del__(
else:
# Teardown may also have cleared the globals the release path reads, here or in a subclass's module.
with _suppress(Exception):
self._release_abandoned_holds()
self._release_abandoned_holds(exited_only=False)

def _release_once_unreachable(self) -> None:
# The callback runs once the lock is gone, so it reads the state through a holder. The state alone keeps the
# holder alive: a reference from the callback would keep the state, and through it the lock, alive.
self._finalizer_state = _FinalizerState(state=self.__dict__)
_UNREACHABLE_LOCKS.add(ref(self, partial(_finalize_unreachable, ref(self._finalizer_state), type(self))))

def _release_abandoned_holds(self) -> None:
def _release_abandoned_holds(self, *, exited_only: bool) -> None:
# A finalizer must not raise. A release error during garbage collection would otherwise surface as an
# unraisable-exception warning, attributed to whichever code triggered collection. The dropped lock still gets
# best-effort cleanup; an explicit release() reports the same error to a caller who can act on it.
with contextlib.suppress(Exception):
self.release(force=True)
# Nothing references the lock any more, so no thread can still be inside it through this instance.
# A dropped lock has no thread left inside it; at exit, a live daemon thread still is, so its hold stays.
with contextlib.suppress(Exception):
self._release_thread_holds(exited_only=False)
self._release_thread_holds(exited_only=exited_only)

@property
def is_singleton(self) -> bool:
Expand Down Expand Up @@ -1905,8 +1912,8 @@ def is_system_call_failure(error: BaseException) -> bool:


def unregister_owned_descriptor(token: int) -> None: # pragma: needs fork
# No fork follows exit, and a daemon thread stopped inside this bookkeeping holds its lock forever.
if sys.is_finalizing(): # pragma: no cover # only exit reaches it, after coverage stops measuring
# The exit release skips this bookkeeping for the same reason fork_transition does.
if interpreter_exiting(): # pragma: no cover # only exit reaches it, after coverage stops measuring
return
with fork_transition(), _FORK_STATE.registry_lock:
_OWNED_DESCRIPTORS.pop(token, None)
Expand Down Expand Up @@ -1977,23 +1984,37 @@ def acquire_thread_lock(


#: Every lock this process built, so exit can release the ones still alive before teardown empties module globals.
_LIVE_LOCKS: Final[WeakSet[BaseFileLock]] = WeakSet()
#: Keyed by identity: a subclass can define __eq__ without __hash__.
_LIVE_LOCKS: Final[WeakValueDictionary[int, BaseFileLock]] = WeakValueDictionary()


#: Set once exit starts releasing live locks, after which no other thread's event loop runs a scheduled release.
_EXITING: Final[Event] = Event()
class _ExitRelease(local):
# Per thread, so a daemon thread still running at exit, and a child it forks, keep their fork bookkeeping.
running = False


def interpreter_exiting() -> bool:
return _EXITING.is_set() or sys.is_finalizing()
#: Marks the thread releasing live locks at exit, where no other thread's event loop runs a scheduled release.
_EXIT_RELEASE: Final[_ExitRelease] = _ExitRelease()


def _release_live_locks() -> None:
# A lock still referenced at exit is finalized only after module teardown, when the release path can no longer run.
_EXITING.set()
pid = os.getpid()
for lock in [lock for lock in _LIVE_LOCKS if lock._creator_pid == pid]: # ruff:ignore[private-member-access] # ditto
lock._release_abandoned_holds() # ruff:ignore[private-member-access] # the module owns the locks it tracks
# Exit handlers run while daemon threads still do, so a hold one of them owns is kept. A shared lock cannot tell
# whose hold it has, so it is kept whenever another thread is alive.
sole_thread = active_count() == 1
_EXIT_RELEASE.running = True
try:
for lock in [
lock
for lock in _LIVE_LOCKS.values()
if lock._creator_pid == pid # ruff:ignore[private-member-access] # the module owns the locks it tracks
and (sole_thread or lock.is_thread_local())
]:
lock._release_abandoned_holds(exited_only=True) # ruff:ignore[private-member-access] # ditto
finally:
# A later exit handler may still fork, and its child is not exiting.
_EXIT_RELEASE.running = False


atexit.register(_release_live_locks)
Expand Down
6 changes: 3 additions & 3 deletions src/filelock/_strict.py
Original file line number Diff line number Diff line change
Expand Up @@ -384,13 +384,13 @@ def force_break(self, claim_name: str) -> None:
if cleanup_error is not None: # pragma: needs dir-fd
raise cleanup_error

def _release_abandoned_holds(self) -> None:
def _release_abandoned_holds(self, *, exited_only: bool) -> None:
self._finalizing = True
super()._release_abandoned_holds()
super()._release_abandoned_holds(exited_only=exited_only)
# release() skips a doorway whose cleanup failed, as it is no hold, so a dropped instance would strand claims.
with contextlib.suppress(Exception):
# GraalPy defers finalizers to the host collector, so its tests cannot drive this cleanup.
self._discard_pending_doorways(exited_only=False) # pragma: needs collected-finalization
self._discard_pending_doorways(exited_only=exited_only) # pragma: needs collected-finalization

def _reconcile_failed_acquire(self, canonical: str) -> None:
# The acquisition is over, so the next one resolves the working directory again rather than reuse this one's.
Expand Down
15 changes: 8 additions & 7 deletions src/filelock/asyncio.py
Original file line number Diff line number Diff line change
Expand Up @@ -836,20 +836,21 @@ def loop(self) -> asyncio.AbstractEventLoop | None:
"""The event loop."""
return self._async_context.loop

def _release_abandoned_holds(self) -> None:
def _release_abandoned_holds(self, *, exited_only: bool) -> None:
# Runs from garbage collection, where no event loop may be running.
with contextlib.suppress(Exception):
# A thread-local lock never uses an executor, so an in-place release matches the loop's and also reaches
# threads whose loops it cannot schedule on.
if self._settings.thread_local and not iscoroutinefunction(self._release):
self._release_without_loop()
self._release_without_loop(exited_only=exited_only)
return
try:
loop = asyncio.get_running_loop()
except RuntimeError:
# At interpreter exit daemon threads no longer run, so a loop running in one never runs the release.
# Once finalizing, daemon threads no longer run, so a loop running in one never runs the release. The
# exit release releases in place too: it skips a shared lock while any other thread is alive.
if (loop := self._async_context.loop) is None or loop.is_closed() or _interpreter_exiting():
self._release_without_loop()
self._release_without_loop(exited_only=exited_only)
return
if not loop.is_running():
loop.run_until_complete(self.release(force=True))
Expand All @@ -865,7 +866,7 @@ def _release_abandoned_holds(self) -> None:
_FINALIZER_RELEASES.add(pending)
pending.add_done_callback(self._settle_finalizer_release)

def _release_without_loop(self) -> None:
def _release_without_loop(self, *, exited_only: bool) -> None:
# No loop can run a release here, so call a sync backend directly as BaseFileLock.__del__ does; a coroutine
# backend cannot run and keeps its lock.
if iscoroutinefunction(self._release):
Expand All @@ -876,7 +877,7 @@ def _release_without_loop(self) -> None:
finally:
self._commit_release_if_released()
# A thread-local lock also holds for the threads that ran its loops, and nothing else releases those now.
self._release_thread_holds(exited_only=False)
self._release_thread_holds(exited_only=exited_only)

def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future[None]) -> None:
_FINALIZER_RELEASES.discard(pending)
Expand All @@ -885,7 +886,7 @@ def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future
if pending.exception() is None:
return
with contextlib.suppress(Exception):
self._release_without_loop()
self._release_without_loop(exited_only=False)


class AsyncAcquireReturnProxy:
Expand Down
26 changes: 26 additions & 0 deletions tests/test_filelock.py
Original file line number Diff line number Diff line change
Expand Up @@ -601,6 +601,32 @@ def test_lock_held_at_exit_is_released_silently(lock_type: type[BaseFileLock], t
pass


@pytest.mark.parametrize("lock_type", [FileLock, SoftFileLock])
@pytest.mark.parametrize("thread_local", [pytest.param(True, id="thread-local"), pytest.param(False, id="shared")])
def test_lock_a_daemon_thread_holds_at_exit_stays_held(
lock_type: type[BaseFileLock], tmp_path: Path, *, thread_local: bool
) -> None:
# Exit handlers run while daemon threads still do, so releasing their hold would let another process in mid-section.
lock_path = str(tmp_path / "a.lock")
contender = f"import filelock; filelock.{lock_type.__name__}({lock_path!r}).acquire(timeout=0)"
script = f"""
import atexit, subprocess, sys, threading
atexit.register(lambda: print(subprocess.run([sys.executable, "-c", {contender!r}], capture_output=True).returncode))
import filelock
lock = filelock.{lock_type.__name__}({lock_path!r}, thread_local={thread_local})
entered = threading.Event()
def hold():
with lock:
entered.set()
threading.Event().wait()
threading.Thread(target=hold, daemon=True).start()
entered.wait()
"""
result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=True)

assert (result.stdout, result.stderr) == ("1\n", "")


@pytest.mark.parametrize("lock_type", [FileLock, SoftFileLock])
@pytest.mark.parametrize(
"drop",
Expand Down
34 changes: 34 additions & 0 deletions tests/test_fork_coordination.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,40 @@ def stop_inside_fork_bookkeeping():
assert (result.returncode, result.stderr) == (0, "")


@NEEDS_REGISTER_AT_FORK # pragma: needs register-at-fork
def test_child_forked_by_a_daemon_thread_at_exit_keeps_its_fork_bookkeeping(tmp_path: Path) -> None:
# Daemon threads still run, and fork, while atexit handlers do; their children are not exiting.
script: Final = """
import atexit, os, sys, threading
exiting, forked = threading.Event(), threading.Event()
atexit.register(lambda: (exiting.set(), forked.wait(10)))
from filelock import FileLock
from filelock._api import _OWNED_DESCRIPTORS

def fork_during_exit():
exiting.wait()
if (pid := os.fork()) == 0:
for _ in range(20):
with FileLock(sys.argv[1]):
pass
print(len(_OWNED_DESCRIPTORS), flush=True)
os._exit(0)
os.waitpid(pid, 0)
forked.set()

threading.Thread(target=fork_during_exit, daemon=True).start()
"""
result: Final = subprocess.run(
[sys.executable, "-W", "ignore::DeprecationWarning", "-c", script, str(tmp_path / "a")],
check=True,
capture_output=True,
text=True,
timeout=30,
)

assert (result.stdout, result.stderr) == ("0\n", "")


@NEEDS_FORK # pragma: needs fork
@NEEDS_AUDIT_EVENTS
def test_import_adds_audit_hook_only_where_settrace_survives_it() -> None:
Expand Down
Loading