Initial commit

This commit is contained in:
Alex committed 2026-11-02 22:54:14 +08:00
commit 57fc3525eb
7311 files changed
+1291398

No files matched your search

+174
View File
@@ -0,0 +1,174 @@
"""
A platform independent file lock that supports the with-statement.
.. autodata:: filelock.__version__
:no-value:
.. autodata:: filelock.OwnerMode
.. autodata:: filelock.CompromiseReason
"""
from __future__ import annotations
__lazy_modules__ = ["filelock._descriptor", "filelock._error", "importlib"]
import sys
import warnings
from importlib import import_module
from typing import TYPE_CHECKING, Final
from filelock._api import AcquireReturnProxy, BaseFileLock, CloseErrorPolicy, ContextErrorPolicy, LockOptions
from filelock._descriptor import lock_descriptor, unlock_descriptor
from filelock._error import LeaseSettingsMismatch, SoftFileLockLifetimeWarning, SoftFileLockProtocolError, Timeout
from filelock._unix import UnixFileLock
from filelock._unix import has_fcntl as _has_fcntl
from filelock._windows import WindowsFileLock
# Keeps ``filelock.version`` the string 4.0.12 exposed rather than the submodule.
from filelock.version import version
if TYPE_CHECKING:
from filelock import asyncio
from filelock._async_read_write import AsyncAcquireReadWriteReturnProxy, AsyncReadWriteLock
from filelock._lease import CompromiseReason, LeaseCompromise, SoftFileLease
from filelock._marker import MarkerSoftFileLock, OwnerMode, OwnerRecord
from filelock._read_write import ReadWriteLock
from filelock._soft import SoftFileLock
from filelock._soft_rw import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock, SoftReadWriteLock
from filelock._strict import StrictSoftFileClaim, StrictSoftFileClaimState, StrictSoftFileLock
from filelock.asyncio import (
AsyncAcquireReturnProxy,
AsyncSoftFileLease,
AsyncSoftFileLock,
AsyncStrictSoftFileLock,
AsyncUnixFileLock,
AsyncWindowsFileLock,
BaseAsyncFileLock,
)
#: version of the project as a string
__version__: Final[str] = version
#: ``True`` when :func:`fcntl.flock` is available. Off Windows, :data:`FileLock` then resolves to
#: :class:`UnixFileLock`, otherwise to :class:`SoftFileLock`. Always ``False`` on Windows.
#:
#: :meta hide-value:
has_fcntl: Final[bool] = _has_fcntl
if sys.platform == "win32": # pragma: win32 cover
_FileLock: type[BaseFileLock] = WindowsFileLock
else: # pragma: win32 no cover # ruff:ignore[collapsible-else-if] # the else carries the win32 no-cover pragma
if has_fcntl:
_FileLock: type[BaseFileLock] = UnixFileLock
else:
from filelock._soft import SoftFileLock
_FileLock = SoftFileLock
warnings.warn("only soft file lock is available", stacklevel=2)
if TYPE_CHECKING:
FileLock = SoftFileLock
AsyncFileLock = AsyncSoftFileLock
else:
#: Alias for the lock, which should be used for the current platform.
FileLock = _FileLock
# Importing asyncio and sqlite3 costs more than the rest of the package, so names that need them load on first access.
_LAZY_ATTRIBUTES: Final[dict[str, str]] = {
"AsyncAcquireReadWriteReturnProxy": "._async_read_write",
"AsyncAcquireReturnProxy": ".asyncio",
"AsyncAcquireSoftReadWriteReturnProxy": "._soft_rw",
"AsyncFileLock": ".asyncio",
"AsyncReadWriteLock": "._async_read_write",
"AsyncSoftFileLease": ".asyncio",
"AsyncSoftFileLock": ".asyncio",
"AsyncSoftReadWriteLock": "._soft_rw",
"AsyncStrictSoftFileLock": ".asyncio",
"AsyncUnixFileLock": ".asyncio",
"AsyncWindowsFileLock": ".asyncio",
"BaseAsyncFileLock": ".asyncio",
"CompromiseReason": "._lease",
"LeaseCompromise": "._lease",
"MarkerSoftFileLock": "._marker",
"OwnerMode": "._marker",
"OwnerRecord": "._marker",
"ReadWriteLock": "._read_write",
"SoftFileLease": "._lease",
"SoftFileLock": "._soft",
"SoftReadWriteLock": "._soft_rw",
"StrictSoftFileClaim": "._strict",
"StrictSoftFileClaimState": "._strict",
"StrictSoftFileLock": "._strict",
"asyncio": ".asyncio",
}
def __getattr__(name: str) -> object:
if (module_name := _LAZY_ATTRIBUTES.get(name)) is None:
msg = f"module {__name__!r} has no attribute {name!r}"
raise AttributeError(msg)
try:
module = import_module(module_name, __name__)
except ImportError: # pragma: lacks sqlite3
if module_name not in {"._read_write", "._async_read_write"}:
raise
value = None
else:
if name == "asyncio":
value = module
else:
value = getattr(module, f"Async{_FileLock.__name__}" if name == "AsyncFileLock" else name)
globals()[name] = value
return value
def __dir__() -> list[str]:
return sorted({*globals(), *_LAZY_ATTRIBUTES})
__all__ = [
"AcquireReturnProxy",
"AsyncAcquireReadWriteReturnProxy",
"AsyncAcquireReturnProxy",
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncFileLock",
"AsyncReadWriteLock",
"AsyncSoftFileLease",
"AsyncSoftFileLock",
"AsyncSoftReadWriteLock",
"AsyncStrictSoftFileLock",
"AsyncUnixFileLock",
"AsyncWindowsFileLock",
"BaseAsyncFileLock",
"BaseFileLock",
"CloseErrorPolicy",
"CompromiseReason",
"ContextErrorPolicy",
"FileLock",
"LeaseCompromise",
"LeaseSettingsMismatch",
"LockOptions",
"MarkerSoftFileLock",
"OwnerMode",
"OwnerRecord",
"ReadWriteLock",
"SoftFileLease",
"SoftFileLock",
"SoftFileLockLifetimeWarning",
"SoftFileLockProtocolError",
"SoftReadWriteLock",
"StrictSoftFileClaim",
"StrictSoftFileClaimState",
"StrictSoftFileLock",
"Timeout",
"UnixFileLock",
"WindowsFileLock",
"__version__",
"asyncio",
"has_fcntl",
"lock_descriptor",
"unlock_descriptor",
]
File diff suppressed because it is too large. Load diff
+507
View File
@@ -0,0 +1,507 @@
"""Separate caller cancellation from backend task and executor-future results."""
from __future__ import annotations
__lazy_modules__ = ["asyncio", "collections", "concurrent.futures", "filelock._error"]
import asyncio
import contextlib
import time
from collections import deque
from concurrent.futures import Future as ConcurrentFuture
from dataclasses import dataclass
from threading import Lock
from typing import TYPE_CHECKING, Final, Generic, Literal, NoReturn, TypeVar, cast
from weakref import WeakKeyDictionary
from filelock._api import (
_append_exception_context,
_fork_transition,
_raise_chained_errors,
_resolve_read_write_timeout,
)
from filelock._error import Timeout
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Awaitable, Callable
from concurrent.futures import Executor
from filelock._read_write import ReadWriteLock
from filelock._soft_rw import SoftReadWriteLock
_T = TypeVar("_T")
#: The holder an async wrapper records its backend level under. No thread has this identifier, so sync code on the
#: executor thread that took the level cannot re-enter or release it.
_TASKS_THREAD_ID: Final[int] = 0
class _AsyncTransitionUnavailableError(Exception):
pass
@dataclass(frozen=True)
class _BackendOutcome(Generic[_T]):
value: _T | None = None
error: BaseException | None = None
class _AsyncTransitionGate:
"""
First-in, first-out admission for the calls that transition one lock context.
Each call queues a ticket, and the head of the queue holds the gate. A call that gives up removes its own ticket,
so abandoned calls do not pile up behind a long holder.
"""
def __init__(self) -> None:
self._queue_lock: Final[Lock] = Lock()
self._queue: Final[deque[ConcurrentFuture[None]]] = deque()
@contextlib.asynccontextmanager
async def hold(self) -> AsyncGenerator[None]:
ticket: ConcurrentFuture[None] = ConcurrentFuture()
try:
self._join(ticket)
if not ticket.done():
await self._wait_for_turn(ticket, blocking=True, cancel_check=None, deadline=None, poll_interval=0.05)
yield
finally:
self._leave(ticket)
@contextlib.asynccontextmanager
async def hold_for_acquire(
self,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
deadline: float | None,
poll_interval: float,
) -> AsyncGenerator[None]:
ticket: ConcurrentFuture[None] = ConcurrentFuture()
try:
self._join(ticket)
if not ticket.done():
await self._wait_for_turn(
ticket,
blocking=blocking,
cancel_check=cancel_check,
deadline=deadline,
poll_interval=poll_interval,
)
yield
finally:
self._leave(ticket)
async def _wait_for_turn(
self,
ticket: ConcurrentFuture[None],
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
deadline: float | None,
poll_interval: float,
) -> None:
if not blocking:
raise _AsyncTransitionUnavailableError
waiter = asyncio.wrap_future(ticket)
while not ticket.done():
if cancel_check is not None and cancel_check():
raise _AsyncTransitionUnavailableError
wait_interval = poll_interval
if deadline is not None:
if (remaining := deadline - time.perf_counter()) <= 0:
raise _AsyncTransitionUnavailableError
wait_interval = min(wait_interval, remaining)
await asyncio.wait((waiter,), timeout=wait_interval)
# A hand-off that a signal handler's exception cut short leaves the head undone, so the head settles itself.
self._settle_head()
def _join(self, ticket: ConcurrentFuture[None]) -> None:
with self._queue_lock:
self._queue.append(ticket)
if len(self._queue) == 1:
ticket.set_result(None)
def _leave(self, ticket: ConcurrentFuture[None]) -> None:
with self._queue_lock:
try:
self._queue.remove(ticket)
except ValueError:
# The join never completed, or this ticket already left.
return
self._settle_head_locked()
def _settle_head(self) -> None:
with self._queue_lock:
self._settle_head_locked()
def _settle_head_locked(self) -> None:
if self._queue and not (head := self._queue[0]).done():
head.set_result(None)
@dataclass
class _TaskHold:
"""
One task's hold, as the nesting count per wrapper it entered through.
A task's hold ends when its last wrapper level is released or closed. Closing one wrapper ends only the levels
taken through it.
"""
depths: dict[object, int]
class _TaskOwners:
"""
Per-task ownership in front of a sync reader/writer lock that tracks threads.
An async wrapper runs each call on an executor thread, and that thread does not identify the calling task. Waiting
writers block new readers so a stream of readers cannot starve them. A thread lock guards the table because tasks on
different event loops share it. The table stands for one backend level, taken once for all tasks. Sync code can
end that level directly, with ``close()`` or a forced release on a shared singleton. The next admission then
forgets the holds instead of letting more tasks join them.
"""
def __init__(self) -> None:
self._lock: Final[Lock] = Lock()
self._mode: Literal["read", "write"] | None = None
self._holds: dict[asyncio.Task[object], _TaskHold] = {}
self._transitioning = False
self._waiting_writers = 0
#: One ticket per waiting call, settled on the next state change. A call that stops waiting takes its own out,
#: so one that times out or is canceled leaves nothing behind.
self._waiters: dict[ConcurrentFuture[None], None] = {}
async def acquire( # ruff:ignore[too-many-arguments] # one keyword per piece of wrapper state the table consults
self,
mode: Literal["read", "write"],
*,
timeout: float,
blocking: bool,
lock_file: str,
via: object,
enter: Callable[[float], Awaitable[None]],
joinable: Callable[[], bool],
) -> None:
task = _current_task()
timeout = _resolve_read_write_timeout(timeout, blocking=blocking)
deadline = None if not blocking or timeout < 0 else time.perf_counter() + timeout
waiting_writer = False
try:
while isinstance(
admission := self._admit(
task, mode, lock_file, via=via, joinable=joinable, waiting_writer=waiting_writer
),
ConcurrentFuture,
):
try:
if not blocking or (deadline is not None and time.perf_counter() >= deadline):
raise Timeout(lock_file)
if mode == "write" and not waiting_writer:
waiting_writer = True
with self._lock:
self._waiting_writers += 1
wait = None if deadline is None else deadline - time.perf_counter()
await asyncio.wait((asyncio.wrap_future(admission),), timeout=wait)
finally:
self._forget(admission)
finally:
if waiting_writer:
with self._lock:
self._waiting_writers -= 1
self._notify()
if admission == "held":
return
try:
await enter(-1 if deadline is None else max(0.0, deadline - time.perf_counter()))
except BaseException:
self._finish_transition(mode=None)
raise
self._finish_transition(mode=mode, holds={task: _TaskHold(depths={via: 1})})
def _admit( # ruff:ignore[too-many-arguments] # the acquire's state, passed through under the table lock
self,
task: asyncio.Task[object],
mode: Literal["read", "write"],
lock_file: str,
*,
via: object,
joinable: Callable[[], bool],
waiting_writer: bool,
) -> Literal["held", "enter"] | ConcurrentFuture[None]:
with self._lock:
# A "held" answer trusts the backend level. Once the backend lost it, the holds are gone and the next task
# has to take the lock again.
if self._mode is not None and not self._transitioning and not joinable():
self._holds.clear()
self._mode = None
if (hold := self._holds.get(task)) is not None:
if self._mode != mode:
msg = (
f"Cannot acquire {mode} lock on {lock_file}: already holding a "
f"{'write' if mode == 'read' else 'read'} lock "
f"({'downgrade' if mode == 'read' else 'upgrade'} not allowed)"
)
raise RuntimeError(msg)
hold.depths[via] = hold.depths.get(via, 0) + 1
return "held"
writers_ahead = self._waiting_writers - waiting_writer
if not self._transitioning:
if mode == "read" and self._mode == "read" and not writers_ahead:
self._holds[task] = _TaskHold(depths={via: 1})
return "held"
if not self._holds and (mode == "write" or not writers_ahead):
self._transitioning, self._mode = True, mode
return "enter"
return self._ticket()
async def release(
self,
*,
force: bool,
lock_file: str,
via: object,
leave: Callable[[], asyncio.Future[_BackendOutcome[None]]],
still_held: Callable[[], bool],
) -> None:
task = _current_task()
with self._lock:
if (hold := self._holds.get(task)) is None or via not in hold.depths:
if force:
return
msg = f"Cannot release a lock on {lock_file} that is not held by this task through this wrapper"
raise RuntimeError(msg)
if not force and hold.depths[via] > 1:
hold.depths[via] -= 1
return
removed = hold.depths.pop(via)
if hold.depths:
return
del self._holds[task]
if self._holds:
return
self._transitioning = True
mode = self._mode
if not still_held():
self._finish_transition(mode=None)
msg = f"Cannot release a lock on {lock_file} that is not held by this task"
raise RuntimeError(msg)
try:
cancellation = await _drain_cancellation(leave())
except BaseException:
hold.depths[via] = removed
# Keep the hold for a retried release only while the sync lock still has the level. Restored after the sync
# lock let go, it would stand for a lock nobody holds.
self._finish_transition(mode=mode if still_held() else None, holds={task: hold})
raise
self._finish_transition(mode=None)
if cancellation is not None:
# The release succeeded, so a restored hold would be a phantom no backend call can clear.
raise cancellation
async def retire(
self,
via: object | None,
*,
leave: Callable[[], asyncio.Future[_BackendOutcome[None]]],
still_held: Callable[[], bool],
) -> None:
"""
End the holds taken through *via*, or every hold when it is ``None``, as one transition.
Leaves the backend through *leave* when that ends the last hold, and always when *via* is ``None``. Arrivals
wait while it runs, so none joins a hold the backend is letting go of. A transition already under way is waited
out first: a task entering through *via* must not get a hold after the wrapper is gone. A cancellation is raised
once the holds are settled.
"""
cancellation: asyncio.CancelledError | None = None
while isinstance(settled := self._drop_holds_via(via), ConcurrentFuture):
try:
await _wait_until_done(asyncio.wrap_future(settled))
except asyncio.CancelledError as error:
cancellation = error
finally:
self._forget(settled)
mode, dropped = settled
if via is None or (mode is not None and still_held()):
try:
cancellation = await _drain_cancellation(leave()) or cancellation
except BaseException:
# A close that failed with the backend still holding stays retryable, as a failed release does.
self._finish_transition(mode=mode if still_held() else None, holds=dropped)
raise
self._finish_transition(mode=None)
elif mode is not None:
# Sync code already ended the level taken for tasks, so there is nothing to give back.
self._finish_transition(mode=None)
if cancellation is not None:
raise cancellation
def _drop_holds_via(
self, via: object | None
) -> tuple[Literal["read", "write"] | None, dict[asyncio.Task[object], _TaskHold]] | ConcurrentFuture[None]:
# Returns the future a running transition settles, or else the mode to leave (None when other holds keep the
# backend level) and the dropped holds. The table stays transitioning while that mode is left.
with self._lock:
if self._transitioning:
return self._ticket()
if via is None:
dropped = dict(self._holds)
self._holds.clear()
else:
dropped = {}
for task, hold in list(self._holds.items()):
if (count := hold.depths.pop(via, None)) is not None:
dropped[task] = _TaskHold(depths={via: count})
if not hold.depths:
del self._holds[task]
if self._holds and via is not None:
return None, dropped
mode = self._mode
self._transitioning = mode is not None or via is None
return mode, dropped
def _finish_transition(
self, *, mode: Literal["read", "write"] | None, holds: dict[asyncio.Task[object], _TaskHold] | None = None
) -> None:
with self._lock:
self._transitioning, self._mode = False, mode
if mode is not None and holds:
for task, hold in holds.items():
self._holds.setdefault(task, _TaskHold(depths={})).depths.update(hold.depths)
self._notify()
def _ticket(self) -> ConcurrentFuture[None]:
# Called under the table lock, in the section that decided to wait, so no state change slips in between.
ticket: ConcurrentFuture[None] = ConcurrentFuture()
self._waiters[ticket] = None
return ticket
def _forget(self, ticket: ConcurrentFuture[None]) -> None:
with self._lock:
self._waiters.pop(ticket, None)
def _notify(self) -> None:
waiters, self._waiters = self._waiters, {}
for ticket in waiters:
ticket.set_result(None)
_TASK_OWNERS: Final[WeakKeyDictionary[ReadWriteLock | SoftReadWriteLock, _TaskOwners]] = WeakKeyDictionary()
_TASK_OWNERS_LOCK: Final[Lock] = Lock()
def _task_owners_for(sync_lock: ReadWriteLock | SoftReadWriteLock) -> _TaskOwners:
"""Singleton wrappers share one sync lock, so they must share its task holds too."""
with _fork_transition(), _TASK_OWNERS_LOCK:
if (owners := _TASK_OWNERS.get(sync_lock)) is None:
owners = _TASK_OWNERS[sync_lock] = _TaskOwners()
return owners
def _current_task() -> asyncio.Task[object]:
if (task := asyncio.current_task()) is None:
msg = "an async reader/writer lock must be used from inside an asyncio task"
raise RuntimeError(msg)
return task
async def _drain_cancellation(future: asyncio.Future[_BackendOutcome[None]]) -> asyncio.CancelledError | None:
# Returned, not raised, so the caller can tell a call that completed under cancellation from one that failed.
try:
await _wait_until_done(future)
except asyncio.CancelledError as cancellation:
try:
await _drain_future(future)
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
_raise_cancelled_error(cancellation, error)
return cancellation
_future_result(future)
return None
async def _drain_future(future: asyncio.Future[_BackendOutcome[_T]]) -> _T:
while not future.done():
with contextlib.suppress(asyncio.CancelledError):
await _wait_until_done(future)
return _future_result(future)
async def _wait_until_done(future: asyncio.Future[_T]) -> None:
if not future.done():
await asyncio.wait((future,))
def _future_result(future: asyncio.Future[_BackendOutcome[_T]]) -> _T:
outcome = future.result()
if (error := outcome.error) is None:
return cast("_T", outcome.value)
context = error.__context__
try:
raise error # ruff:ignore[raise-within-try] # the handler restores context changed across the async boundary
except BaseException:
error.__context__ = context
raise
def _run_in_executor(
loop: asyncio.AbstractEventLoop, executor: Executor | None, func: Callable[[], _T]
) -> asyncio.Future[_BackendOutcome[_T]]:
outcome: asyncio.Future[_BackendOutcome[_T]] = loop.create_future()
def settle(call: asyncio.Future[_BackendOutcome[_T]]) -> None:
try:
result = call.result()
except asyncio.CancelledError:
# shutdown(cancel_futures=True) dropped the call unrun: a CancelledError would cancel a task nobody canceled
# and hide that the lock state never changed.
msg = "the executor shut down before running the lock backend call"
result = _BackendOutcome(error=RuntimeError(msg))
outcome.set_result(result)
loop.run_in_executor(executor, _capture_call, func).add_done_callback(settle)
return outcome
def _capture_call(func: Callable[[], _T]) -> _BackendOutcome[_T]:
try:
return _BackendOutcome(value=func())
except BaseException as error: # ruff:ignore[blind-except] # backend control-flow exceptions are operation results
return _BackendOutcome(error=error)
def _raise_cancelled_error(cancellation: asyncio.CancelledError, error: BaseException) -> NoReturn:
# A reconciliation step failed while unwinding a cancellation, so keep both exception chains. Splice the error's
# existing context onto the cancellation, then make the cancellation the error's context, so both the failure and
# the cancellation that triggered it survive. Shared by the async wrappers so cancellations report the same way.
if (context := error.__context__) is not None and context is not cancellation:
if (cancellation_context := cancellation.__context__) is not None:
_append_exception_context(context, cancellation_context)
cancellation.__context__ = context
error.__context__ = cancellation
_raise_chained_errors(error)
async def _capture_awaitable(awaitable: Awaitable[_T]) -> _BackendOutcome[_T]:
try:
return _BackendOutcome(value=await awaitable)
except BaseException as error: # ruff:ignore[blind-except] # backend cancellation must remain distinct from caller cancellation
return _BackendOutcome(error=error)
__all__ = [
"_TASKS_THREAD_ID",
"_AsyncTransitionGate",
"_AsyncTransitionUnavailableError",
"_BackendOutcome",
"_capture_awaitable",
"_capture_call",
"_drain_future",
"_future_result",
"_raise_cancelled_error",
"_run_in_executor",
"_task_owners_for",
"_wait_until_done",
]
@@ -0,0 +1,413 @@
"""Async wrapper around :class:`ReadWriteLock` for use with ``asyncio``."""
from __future__ import annotations
__lazy_modules__ = ["asyncio", "concurrent.futures", "filelock._async", "filelock._error", "functools", "threading"]
import asyncio
import functools
import os
import sqlite3
import threading
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import thread as executor_threads
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar, cast
from filelock._api import (
_append_exception_context,
_ensure_current_process,
_fork_transition,
_register_fork_object,
)
from filelock._async import (
_TASKS_THREAD_ID,
_BackendOutcome,
_capture_call,
_drain_future,
_future_result,
_raise_cancelled_error,
_task_owners_for,
_wait_until_done,
)
from filelock._error import Timeout
from filelock._read_write import ReadWriteLock
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, MutableMapping
from concurrent import futures
from types import TracebackType
from filelock._api import AcquireReturnProxy
_P = ParamSpec("_P")
_R = TypeVar("_R")
class AsyncReadWriteLock:
"""
Async wrapper around :class:`ReadWriteLock` for use in ``asyncio`` applications.
This wrapper dispatches every blocking SQLite operation to a thread pool via ``loop.run_in_executor()`` because
Python's :mod:`sqlite3` module has no async API. Each ``asyncio`` task owns its own hold. Tasks share a read lock,
and a task asking for the write lock waits until the other holders release, with waiting writers ahead of new
readers. As with :class:`ReadWriteLock`, upgrading or downgrading a held lock raises :class:`RuntimeError`.
Singleton wrappers over one path share their task holds, and closing one ends only the holds taken through it.
:param lock_file: path to the SQLite database file used as the lock
:param timeout: maximum wait time in seconds; ``-1`` waits up to SQLite's busy-timeout cap of about 23 days
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:param is_singleton: if ``True``, reuse existing :class:`ReadWriteLock` instances for the same resolved path
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
:param executor: executor for ``run_in_executor``. When ``None`` this lock creates and owns a dedicated
single-thread executor so every operation runs on the same thread (SQLite affinity requires this) and shuts it
down in :meth:`close`. This lock uses a caller-supplied executor as-is and never shuts it down, so after passing
no executor call :meth:`close` to release the owned one.
.. versionadded:: 3.21.0
"""
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
loop: asyncio.AbstractEventLoop | None = None,
executor: futures.Executor | None = None,
) -> None:
creator_pid = os.getpid()
self._creator_pid = creator_pid
self._fork_invalidated = False
self._closed = False
self._owns_backend = not is_singleton
#: Marks the task holds taken through this wrapper, which its close() ends.
self._via: Final = object()
_register_fork_object(self)
with _fork_transition():
self._lock = ReadWriteLock(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
self._owners = _task_owners_for(self._lock)
self._loop = loop
self._owns_executor = executor is None
self._executor = executor or ThreadPoolExecutor(max_workers=1)
if os.getpid() != creator_pid: # pragma: forked child
msg = "AsyncReadWriteLock construction cannot continue after fork"
raise RuntimeError(msg)
@property
def lock_file(self) -> str:
"""The path to the lock file."""
return self._lock.lock_file
@property
def timeout(self) -> float:
"""The default timeout."""
return self._lock.timeout
@property
def blocking(self) -> bool:
"""Whether blocking is enabled by default."""
return self._lock.blocking
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
"""The event loop (or ``None`` for the running loop)."""
return self._loop
@property
def executor(self) -> futures.Executor:
"""The executor used for ``run_in_executor`` (a dedicated single-thread one if none was supplied)."""
return self._executor
@asynccontextmanager
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
await self.acquire_read(timeout, blocking=blocking)
body_error: BaseException | None = None
try:
yield
except BaseException as error:
body_error = error
raise
finally:
await self._release_in_context(body_error)
@asynccontextmanager
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
"""
await self.acquire_write(timeout, blocking=blocking)
body_error: BaseException | None = None
try:
yield
except BaseException as error:
body_error = error
raise
finally:
await self._release_in_context(body_error)
async def _release_in_context(self, body_error: BaseException | None) -> None:
try:
await self.release()
except BaseException as release_error:
if body_error is not None:
_append_exception_context(release_error, body_error)
raise
async def acquire_read(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireReadWriteReturnProxy:
"""
Acquire a shared read lock.
See :meth:`ReadWriteLock.acquire_read` for full semantics.
:param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's
busy-timeout cap of about 23 days
:param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention;
``None`` uses the instance setting
:returns: a proxy that can be used as an async context manager to release the lock
:raises RuntimeError: if the calling task already holds the write lock
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
self._raise_if_unusable()
await self._acquire("read", timeout, blocking=blocking)
return AsyncAcquireReadWriteReturnProxy(lock=self)
async def acquire_write(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireReadWriteReturnProxy:
"""
Acquire an exclusive write lock.
See :meth:`ReadWriteLock.acquire_write` for full semantics.
:param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's
busy-timeout cap of about 23 days
:param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention;
``None`` uses the instance setting
:returns: a proxy that can be used as an async context manager to release the lock
:raises RuntimeError: if the calling task already holds the read lock
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
self._raise_if_unusable()
await self._acquire("write", timeout, blocking=blocking)
return AsyncAcquireReadWriteReturnProxy(lock=self)
async def _acquire(self, mode: Literal["read", "write"], timeout: float | None, *, blocking: bool | None) -> None:
timeout = self.timeout if timeout is None else timeout
blocking = self.blocking if blocking is None else blocking
async def enter(remaining: float) -> None:
# A close() may have landed while this task waited for the table.
self._raise_if_unusable()
await self._run_acquire(
functools.partial(
self._lock._acquire_level_of, # ruff:ignore[private-member-access] # the wrapper owns this backend hold
_TASKS_THREAD_ID,
mode,
remaining,
blocking=blocking,
)
)
def joinable() -> bool:
self._raise_if_unusable()
return self._backend_holds()
await self._owners.acquire(
mode,
timeout=timeout,
blocking=blocking,
lock_file=self.lock_file,
via=self._via,
enter=enter,
joinable=joinable,
)
async def release(self, *, force: bool = False) -> None:
"""
Release one level of the calling task's hold.
The wrapper releases the database lock once no task holds it.
:param force: if ``True``, end every level the calling task took through this wrapper. Levels taken through
other wrappers on the same path stay. A wrapper that holds no level for the task releases nothing.
:raises RuntimeError: if the calling task holds no level through this wrapper and *force* is ``False``
"""
_ensure_current_process()
if self._inherited: # pragma: needs fork
return
await self._owners.release(
force=force,
lock_file=self.lock_file,
via=self._via,
leave=functools.partial(self._submit, self._leave_backend),
still_held=self._backend_holds,
)
async def close(self) -> None:
"""
End the holds of the tasks that acquired through this instance and release the lock once no task holds it.
After calling this method, the lock instance is no longer usable. With ``is_singleton=False`` it also closes the
underlying SQLite connection. A singleton wrapper leaves the shared :class:`ReadWriteLock` open, so tasks that
acquired through another wrapper and synchronous users of the same path keep their holds. The next acquire
notices a synchronous ``close()`` or forced release of that shared lock. One that lands in the few bytecodes
between that check and the admission still admits the task into the ended hold.
"""
_ensure_current_process()
if self._inherited: # pragma: needs fork
return
if self._closed:
return
# Refuse acquires from here on, so none is admitted into a hold the backend is letting go of.
self._closed = True
try:
await self._owners.retire(
None if self._owns_backend else self._via,
leave=functools.partial(self._submit, self._lock.close if self._owns_backend else self._leave_backend),
still_held=self._backend_holds,
)
except BaseException as error:
# Only a failed backend call can leave the level held, and then a retried close() has to find it.
if not isinstance(error, asyncio.CancelledError) and self._backend_holds():
self._closed = False
else:
self._shutdown_owned_executor()
raise
# Wait for the worker to exit rather than letting it drain in the background: a caller that forks right
# after closing deserves a single-threaded process, and os.fork warns about any surviving thread.
if self._owns_executor:
await asyncio.to_thread(functools.partial(self._executor.shutdown, wait=True))
async def _run_acquire(self, acquire: Callable[..., AcquireReturnProxy]) -> None:
abandoned = threading.Event()
acquire_future = self._submit(acquire, cancel_check=abandoned.is_set)
try:
await _wait_until_done(acquire_future)
except asyncio.CancelledError as cancellation:
# Stops a backend wait on another process's hold, which would otherwise outlast the caller by as long.
abandoned.set()
try:
await _drain_future(acquire_future)
except Timeout:
# Nothing was taken, so the caller's cancellation is the whole outcome.
raise cancellation from None
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
_raise_cancelled_error(cancellation, error)
try:
await _drain_future(self._submit(self._leave_backend))
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
_raise_cancelled_error(cancellation, error)
raise
_future_result(acquire_future)
def _backend_holds(self) -> bool:
# A sync close() or forced release on a shared singleton ends the tasks' level without the table taking part.
return self._lock._holds_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
def _leave_backend(self) -> None:
# Only the tasks' level: a sync user of the same singleton keeps its own.
self._lock._release_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
def _submit(
self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs
) -> asyncio.Future[_BackendOutcome[_R]]:
return (self._loop or asyncio.get_running_loop()).run_in_executor(
self._executor,
_capture_call,
functools.partial(func, *args, **kwargs),
)
def _shutdown_owned_executor(self) -> None:
if self._owns_executor:
self._executor.shutdown(wait=False)
@property
def _inherited(self) -> bool:
return self._fork_invalidated or os.getpid() != self._creator_pid
def _raise_if_unusable(self) -> None:
_ensure_current_process()
if self._inherited: # pragma: needs fork
msg = f"AsyncReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance"
raise RuntimeError(msg)
if self._closed:
msg = "Cannot operate on a closed database."
raise sqlite3.ProgrammingError(msg)
def _reset_after_fork_in_child(self) -> None: # pragma: forked child
self._fork_invalidated = True
if self._owns_executor:
# The worker did not survive the fork, yet before 3.12 the child's exit still wakes it through its queue. On
# PyPy that queue is pure Python, and its lock may have been held by the worker at fork time, so exit hangs.
# CPython 3.12+ forgets every inherited worker in the child; do the same for the one this lock owns.
# typeshed declares the registry read-only; it is a WeakKeyDictionary the exit hook iterates.
registry = cast(
"MutableMapping[threading.Thread, object]",
executor_threads._threads_queues, # ruff:ignore[private-member-access] # the exit hook's registry
)
for worker in cast("ThreadPoolExecutor", self._executor)._threads: # ruff:ignore[private-member-access] # only the executor lists its workers
registry.pop(worker, None)
def __del__(self) -> None:
# Safety net when close() was never called: shut down the executor we own so its worker thread does not
# outlive the lock. shutdown(wait=False) never blocks.
if os.getpid() == getattr(self, "_creator_pid", None) and getattr(self, "_owns_executor", False):
self._executor.shutdown(wait=False)
class AsyncAcquireReadWriteReturnProxy:
"""Context-aware object that releases the async read/write lock on exit."""
def __init__(self, lock: AsyncReadWriteLock) -> None:
self.lock = lock
async def __aenter__(self) -> AsyncReadWriteLock:
return self.lock
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock.release()
__all__ = [
"AsyncAcquireReadWriteReturnProxy",
"AsyncReadWriteLock",
]
@@ -0,0 +1,76 @@
"""A minimal native lock over a caller-owned file descriptor, contending with :class:`FileLock` on the same file."""
from __future__ import annotations
__lazy_modules__ = ["filelock._unix", "threading", "typing"]
import sys
import time
from threading import TIMEOUT_MAX
from typing import Final
from filelock._api import _seconds
if sys.platform == "win32": # pragma: win32 cover
from filelock._windows import _lock_fd_nonblocking, _unlock_fd
else: # pragma: win32 no cover
from filelock._unix import _lock_fd_nonblocking, _unlock_fd
def lock_descriptor(fd: int, *, blocking: bool = True, poll_interval: float = 0.05) -> bool:
"""
Take the native OS lock on *fd*, a file descriptor the caller opened and owns.
This is the same one-byte exclusive lock :class:`FileLock` uses, so a descriptor lock and a path lock on the same
file contend with each other. Unlike :class:`FileLock` it adds no path handling: it never opens, truncates, closes,
unlinks, chmods, canonicalizes, or falls back. The caller owns *fd* before, during, and after the call, and must
close it. On Windows *fd* must be a synchronous descriptor (its handle not opened with ``FILE_FLAG_OVERLAPPED``).
For timeout, reentrancy, singleton, lifetime, or stale-break behavior, use :class:`FileLock`. There is no async
wrapper. Run this in an executor, or drive ``blocking=False`` from your own polling loop.
:param fd: an open file descriptor the caller owns.
:param blocking: when ``True`` (default), retry the nonblocking attempt every *poll_interval* seconds until it
succeeds; when ``False``, make one attempt.
:param poll_interval: positive seconds between attempts while blocking, at most :data:`threading.TIMEOUT_MAX`;
ignored when *blocking* is ``False``.
:returns: ``True`` once the lock is held, or ``False`` on contention when ``blocking`` is ``False``.
:raises OSError: for a permanent native failure, such as an invalid descriptor, or with ``errno.ENOSYS`` when the
Python build lacks the native locking primitive. The descriptor is left open.
:raises ValueError: if a blocking call receives a *poll_interval* outside that range, including ``nan``.
.. versionadded:: 3.30.0
"""
if not blocking:
return _lock_fd_nonblocking(fd)
# time.sleep raises OverflowError past TIMEOUT_MAX; nan fails both comparisons.
if not 0 < (poll_interval := _seconds("poll_interval", poll_interval)) <= TIMEOUT_MAX:
msg: Final[str] = f"poll_interval must be greater than 0 and at most {TIMEOUT_MAX}, got {poll_interval}"
raise ValueError(msg)
while not _lock_fd_nonblocking(fd):
time.sleep(poll_interval)
return True
def unlock_descriptor(fd: int) -> None:
"""
Release the native OS lock on *fd* without touching the descriptor.
:param fd: the descriptor a prior :func:`lock_descriptor` locked; the caller still owns and must close it.
:raises OSError: if the native unlock fails, including ``errno.ENOSYS`` when the Python build lacks the native
locking primitive; the caller may retry on the same descriptor.
.. versionadded:: 3.30.0
"""
_unlock_fd(fd)
__all__ = [
"lock_descriptor",
"unlock_descriptor",
]
+90
View File
@@ -0,0 +1,90 @@
from __future__ import annotations
__lazy_modules__ = ["typing"]
from typing import Final
class Timeout(TimeoutError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API
"""Raised when the lock could not be acquired in *timeout* seconds."""
def __init__(self, lock_file: str, holder: str | None = None) -> None:
super().__init__()
self._lock_file = lock_file
self._holder = holder
def __reduce__(self) -> tuple[type[Timeout], tuple[str], dict[str, object]]:
# holder rides in the state so a subclass taking only lock_file, and a 4.0.12 reader, can still unpickle it.
return self.__class__, (self._lock_file,), self.__dict__
def __str__(self) -> str: # pragma: needs hard-link
message: Final = f"The file lock '{self._lock_file}' could not be acquired."
if self._holder is None:
return message
return f"{message} It is held by {self._holder}, which this host cannot probe."
def __repr__(self) -> str:
return f"{self.__class__.__name__}({self.lock_file!r})"
@property
def lock_file(self) -> str:
"""The path of the file lock."""
return self._lock_file
class SoftFileLockLifetimeWarning(FutureWarning):
"""
The configured soft-lock lifetime permits overlapping live holders after expiry.
A :class:`FutureWarning` shows by default wherever the lock is configured; a :class:`DeprecationWarning` shows only
in ``__main__``.
"""
class LeaseSettingsMismatch(ValueError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API
"""A lease contender disagrees with the published claim about how long the lease lasts."""
class SoftFileLockProtocolError(OSError):
"""Raised when strict soft-lock state cannot be interpreted without risking overlap."""
def __init__(self, lock_file: str, claim_name: str | None, reason: str) -> None:
self._lock_file = lock_file
self._claim_name = claim_name
self._reason = reason
super().__init__(self.__str__())
def __reduce__(
self,
) -> tuple[
type[SoftFileLockProtocolError], tuple[str, str | None, str], dict[str, object]
]: # pragma: needs hard-link
return self.__class__, (self._lock_file, self._claim_name, self._reason), self.__dict__
def __str__(self) -> str:
location = self._lock_file if self._claim_name is None else f"{self._lock_file}: claim {self._claim_name!r}"
return f"Invalid strict soft-lock state at {location}: {self._reason}"
@property
def lock_file(self) -> str: # pragma: needs hard-link
"""The requested lock path."""
return self._lock_file
@property
def claim_name(self) -> str | None: # pragma: needs hard-link
"""The claim that caused the error, if scanning identified one."""
return self._claim_name
@property
def reason(self) -> str: # pragma: needs hard-link
"""The protocol validation failure."""
return self._reason
__all__ = [
"LeaseSettingsMismatch",
"SoftFileLockLifetimeWarning",
"SoftFileLockProtocolError",
"Timeout",
]
@@ -0,0 +1,328 @@
from __future__ import annotations
__lazy_modules__ = ["hashlib", "pathlib", "socket", "typing"]
import hashlib
import os
import socket
import sys
from errno import EPERM, ESRCH
from pathlib import Path
from typing import Final
#: Every marker format in this package caps the hostname at the RFC 1123 limit and reads a longer one as malformed.
_HOST_NAME_LIMIT: Final[int] = 253
#: The bytes those formats carry verbatim: printable non-space ASCII, less the ``?`` reserved for the escape.
_HOST_NAME_VERBATIM: Final[frozenset[int]] = frozenset(range(0x21, 0x7F)) - {ord("?")}
#: The characters at the end of an over-long name that carry a digest of the part cut off.
_HOST_NAME_DIGEST_LEN: Final[int] = 8
#: The kernel ABI fixes the initial PID namespace's inode (``PID_NS_INIT_INO`` in ``linux/nsfs.h``); every namespace
#: created later draws one from ``0xF0000000`` up, so this value names the host's own namespace.
_INITIAL_PID_NAMESPACE: Final[int] = 0xEFFFFFFC
_NAMESPACE_MARK: Final[str] = "?pidns-"
def host_name() -> str:
"""
The hostname recorded alongside an owner, so a marker written on another machine is never probed here.
Marker formats here hold the field to printable non-space ASCII, while ``socket.gethostname()`` reports whatever
the kernel stores: a space, a newline that forges an extra marker line, or a byte no codec encodes, which Python
hands back as a surrogate. A holder that publishes one raw writes a marker it cannot read back, and loses the lock
to the first peer that ages it out as malformed. ``?`` is illegal in a hostname, so escaping it along with every
out-of-grammar byte as ``?<hex>`` leaves a real name untouched and still keeps two hosts apart, which
:func:`owner_is_stale` relies on to refuse to probe a foreign PID.
Linux appends the PID namespace, as Mercurial's lock prefix does. Two containers in one pod share the hostname and
the boot id but not the PID namespace, so a PID recorded in one names another process, or none, in the other. The
suffix starts with a ``?`` no escape can produce, so ``a`` in a namespace never reads as a host literally named
``a?pidns-...``. The initial namespace adds nothing, which keeps a host's name what older releases recorded.
An over-long name ends in a digest of the part cut to fit the limit. A bare cut gives every name that shares the
kept prefix one identity, and :func:`owner_is_stale` would then probe a local PID for another host's holder.
"""
name = (
"".join(
chr(byte) if byte in _HOST_NAME_VERBATIM else f"?{byte:02x}"
for byte in socket.gethostname().encode("utf-8", "surrogateescape")
)
+ _pid_namespace_suffix()
)
if len(name) > _HOST_NAME_LIMIT:
keep = _HOST_NAME_LIMIT - _HOST_NAME_DIGEST_LEN - 1
name = f"{name[:keep]}-{hashlib.sha256(name[keep:].encode()).hexdigest()[:_HOST_NAME_DIGEST_LEN]}"
# An escape is three characters and a kept byte is never '?', so a bare '?' can only mean an empty hostname.
return name or "?"
def _pid_namespace_suffix() -> str:
try:
namespace: Final = Path("/proc/self/ns/pid").stat().st_ino
except OSError: # only Linux 3.8+ has the file, and a sandbox may hide it; nothing else tells namespaces apart
return ""
return "" if namespace == _INITIAL_PID_NAMESPACE else f"{_NAMESPACE_MARK}{namespace:x}"
def owner_is_stale(pid: int, hostname: str, start_token: int | None) -> bool:
"""
Whether the recorded owner is provably gone, so reclaiming its marker cannot detach a live holder.
Fail closed: return ``True`` only when this process can prove the exact recorded owner is dead. A marker from
another host cannot be probed; a live PID whose start token still matches, or whose token cannot be read, is the
holder or is indistinguishable from it; a live PID whose start token differs is a recycled PID, so the process that
wrote the marker is gone. PostgreSQL, Qt ``QLockFile`` and Mercurial all break a stale lock only on proof of death
and treat an unreadable or foreign owner as still holding.
"""
if hostname != host_name():
return False
if process_alive(pid) and (
start_token is None or (current := process_start_token(pid)) is None or _same_start(current, start_token)
):
return False
# Only a bare hostname can be an older release's, written from a container that shares this host's name.
return _NAMESPACE_MARK in hostname or not _owner_runs_elsewhere(pid, start_token)
def owner_is_current_process(pid: int, hostname: str, start_token: int | None) -> bool:
"""
Whether the recorded owner is this process, so a marker a recycled PID left behind never reads as ours.
Fail closed: a recorded start token must match this process's. One that differs was written by an earlier process
that reused this PID, and when this process cannot read its own token nothing proves the marker is ours. A marker
without a token falls back to the PID and hostname.
"""
if pid != os.getpid() or hostname != host_name():
return False
return start_token is None or start_token == process_start_token(pid)
def _same_start(current: int, recorded: int) -> bool:
# A Linux token carries the boot id in its high 64 bits, or 0 there when the process could not read the boot id.
# systemd's ProcSubset=pid hides /proc/sys from one unit, so one process may lack the boot id while its peer has
# it. A missing boot id then proves nothing, and only starttime is compared. Other platforms' tokens fit in the
# low bits.
return current == recorded or (0 in {current >> 64, recorded >> 64} and current % 2**64 == recorded % 2**64)
if sys.platform == "win32": # pragma: win32 cover
import ctypes
from ctypes import wintypes
_KERNEL32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True)
_KERNEL32.CloseHandle.argtypes = [wintypes.HANDLE]
_KERNEL32.CloseHandle.restype = wintypes.BOOL
_KERNEL32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
_KERNEL32.OpenProcess.restype = wintypes.HANDLE
_KERNEL32.GetProcessTimes.argtypes = [
wintypes.HANDLE,
ctypes.POINTER(wintypes.FILETIME),
ctypes.POINTER(wintypes.FILETIME),
ctypes.POINTER(wintypes.FILETIME),
ctypes.POINTER(wintypes.FILETIME),
]
_KERNEL32.GetProcessTimes.restype = wintypes.BOOL
_WIN_SYNCHRONIZE: Final[int] = 0x100000
_WIN_PROCESS_QUERY_LIMITED_INFORMATION: Final[int] = 0x1000
_WIN_ERROR_INVALID_PARAMETER: Final[int] = 87
_WIN_INHERIT_HANDLE: Final[bool] = False
def process_alive(pid: int) -> bool:
"""Whether a process with this PID exists, treating an access denial as proof it does."""
handle = _KERNEL32.OpenProcess(_WIN_SYNCHRONIZE, _WIN_INHERIT_HANDLE, pid)
if handle:
_KERNEL32.CloseHandle(handle)
return True
return ctypes.get_last_error() != _WIN_ERROR_INVALID_PARAMETER
def process_start_token(pid: int) -> int | None:
"""The process creation FILETIME as a 100ns tick count, or ``None`` when it cannot be read."""
handle = _KERNEL32.OpenProcess(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, _WIN_INHERIT_HANDLE, pid)
if not handle:
return None
creation, exit_time, kernel_time, user_time = (wintypes.FILETIME() for _ in range(4))
try:
if not _KERNEL32.GetProcessTimes(
handle,
ctypes.byref(creation),
ctypes.byref(exit_time),
ctypes.byref(kernel_time),
ctypes.byref(user_time),
):
return None # pragma: no cover # win32 GetProcessTimes failure path; not reproducible on a live handle
finally:
_KERNEL32.CloseHandle(handle)
return (creation.dwHighDateTime << 32) | creation.dwLowDateTime
else: # pragma: win32 no cover
def process_alive(pid: int) -> bool:
"""Whether a process with this PID exists, treating an access denial (``EPERM``) as proof it does."""
try:
os.kill(pid, 0)
except OSError as error:
if error.errno == ESRCH:
return False
if error.errno == EPERM:
return True
raise
return True
if sys.platform in {"linux", "android"}: # pragma: linux cover
# Termux/Android reports sys.platform == "android" but runs the Linux kernel, so /proc/<pid>/stat and the boot
# id are the same reliable start-time source; treat it exactly like Linux rather than the tokenless fallback.
# comm (field 2) is wrapped in parentheses and may itself contain spaces or a ')', so the fixed fields start
# after the final ')'. starttime is field 22 overall, the twentieth of those trailing fields (index 19).
_STARTTIME_INDEX: Final[int] = 19
def _read_boot_id() -> int:
# starttime is measured in clock ticks since boot, so on its own it repeats across a reboot. Folding the
# boot id into the high bits makes the Linux token reboot-safe like the absolute clocks macOS and Windows
# expose, while staying a single integer so a 3.29 reader still parses the third marker line. 0 when this
# process cannot read the boot id leaves the bare starttime, which _same_start compares alone.
try:
boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii")
return int(boot_id.strip().replace("-", ""), 16)
except (OSError, ValueError):
return 0
_BOOT_ID: Final[int] = _read_boot_id()
#: ``TIME_NS_INIT_INO`` in ``linux/proc_ns.h``, fixed by the kernel ABI like the initial PID namespace's inode.
_INITIAL_TIME_NAMESPACE: Final[int] = 0xEFFFFFFA
def process_start_token(pid: int) -> int | None:
"""
The ``/proc/<pid>/stat`` ``starttime`` folded with the boot id.
``None`` when the process is gone, or when this process runs in a time namespace with a boottime offset.
Since Linux 5.6 the kernel shifts ``starttime`` by the reader's offset, so a value read there differs
from what a peer outside reads.
"""
return _read_start_token(pid) if _starttime_unshifted() else None
def _starttime_unshifted() -> bool:
# setns can move this process into another time namespace, so ask on every read.
try:
namespace: Final = Path("/proc/self/ns/time").stat().st_ino
except FileNotFoundError: # no entry means no time namespaces on this kernel
return True
except OSError: # any other failure leaves the namespace unknown, so starttime is not comparable
return False
if namespace == _INITIAL_TIME_NAMESPACE:
return True
# A container runtime may give each container a time namespace without offsets. timens_offsets describes
# the namespace this process's children get, its own unless it unshared one since.
try:
if Path("/proc/self/ns/time_for_children").stat().st_ino != namespace:
return False
offsets: Final = Path("/proc/self/timens_offsets").read_text(encoding="ascii")
except OSError:
return False
# The clock goes by name since Linux 5.8 and by clock id before.
return any(line.split() in (["boottime", "0", "0"], ["7", "0", "0"]) for line in offsets.splitlines())
def _read_start_token(pid: int) -> int | None:
try:
data = Path(f"/proc/{pid}/stat").read_bytes()
except OSError:
return None
# psutil identifies a process by the same (pid, starttime) pair; the boot id extends that across reboots.
fields = data[data.rfind(b")") + 1 :].split()
if len(fields) <= _STARTTIME_INDEX: # pragma: no cover # a truncated /proc read, never seen in practice
return None
try:
starttime = int(fields[_STARTTIME_INDEX])
except ValueError: # pragma: no cover # /proc always renders starttime as an integer
return None
return (_BOOT_ID << 64) | starttime
def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool:
"""
Whether the owner runs in another PID namespace as *pid*.
An older release in a container sharing this hostname recorded its in-container PID, which means nothing
here. Readers without a boottime offset read the same starttime and boot id. A task with the recorded
start token whose innermost PID is *pid* is therefore the owner; one that started in the same tick is not.
A reader with an offset cannot compare start tokens, so the innermost PID alone keeps the marker.
"""
if start_token is None or (_BOOT_ID and start_token >> 64 not in {0, _BOOT_ID}):
return False
unshifted: Final = _starttime_unshifted()
try:
with os.scandir("/proc") as entries:
return any(
entry.name.isdigit()
and (task := int(entry.name)) != pid
and (
not unshifted
or ((token := _read_start_token(task)) is not None and _same_start(token, start_token))
)
and _innermost_pid(task) == pid
for entry in entries
)
except OSError:
# A /proc this process cannot list proves nothing either way, which leaves 4.0.12's verdict.
return False
def _innermost_pid(task: int) -> int | None:
try:
status: Final = Path(f"/proc/{task}/status").read_bytes()
except OSError: # pragma: no cover # the task exited between the stat read and this one
return None
# The kernel ends a line only with "\n". A task names itself, and its Name line keeps "\r", "\v" and the
# other separators that a text-mode read and str.splitlines() split at. A name could then forge a line.
return next(
(int(line.split()[-1]) for line in status.split(b"\n") if line.startswith(b"NSpid:")),
None, # kernels before 4.1 record no namespace PIDs
)
elif sys.platform == "darwin": # pragma: darwin cover
import ctypes
import struct
_LIBC: Final[ctypes.CDLL] = ctypes.CDLL(None, use_errno=True)
_CTL_KERN: Final[int] = 1
_KERN_PROC: Final[int] = 14
_KERN_PROC_PID: Final[int] = 1
# kinfo_proc opens with kp_proc.p_starttime (a struct timeval) at offset 0: int64 seconds, int32 microseconds.
# The offset is fixed by the struct chain kinfo_proc -> extern_proc -> p_un, so a read at 0 is not a guess.
_TIMEVAL_AT_ZERO: Final[str] = "<qi"
_TIMEVAL_SIZE: Final[int] = struct.calcsize(_TIMEVAL_AT_ZERO)
def process_start_token(pid: int) -> int | None:
"""The process start time in microseconds from ``sysctl(KERN_PROC_PID)``, or ``None`` when it is gone."""
mib = (ctypes.c_int * 4)(_CTL_KERN, _KERN_PROC, _KERN_PROC_PID, pid)
length = ctypes.c_size_t(0)
# The size probe reports the kinfo_proc size for any PID, so the read below, not the probe, tells a live
# process from a gone one: a gone PID leaves the fetch a zero-length success, so re-check the length after.
if _LIBC.sysctl(mib, 4, None, ctypes.byref(length), None, 0) != 0:
return None
buffer = (ctypes.c_char * length.value)()
if _LIBC.sysctl(mib, 4, buffer, ctypes.byref(length), None, 0) != 0 or length.value < _TIMEVAL_SIZE:
return None
seconds, microseconds = struct.unpack_from(_TIMEVAL_AT_ZERO, buffer.raw, 0)
return seconds * 1_000_000 + microseconds
else: # pragma: no cover # a POSIX platform without a proven start-time source falls back to fail-closed liveness
def process_start_token(pid: int) -> int | None:
"""No proven start-time source, so the owner carries no token and liveness rests on the PID alone."""
del pid
return None
if sys.platform not in {"linux", "android"}: # pragma: linux no cover # pragma: android no cover
def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool:
"""Without Linux PID namespaces, a PID that is gone or recycled leaves its owner nowhere else to run."""
del pid, start_token
return False
__all__ = [
"host_name",
"owner_is_current_process",
"owner_is_stale",
"process_alive",
"process_start_token",
]
+402
View File
@@ -0,0 +1,402 @@
from __future__ import annotations
__lazy_modules__ = [
"contextlib",
"filelock._error",
"filelock._identity",
"filelock._soft",
"filelock._util",
"math",
"secrets",
"threading",
"weakref",
]
import os
import secrets
import sys
import time
from contextlib import suppress
from dataclasses import dataclass
from math import isfinite
from threading import Event, Thread, current_thread
from typing import TYPE_CHECKING, Final, Literal, cast
from weakref import WeakMethod
from filelock._api import _check_timeout_max, _seconds
from filelock._error import LeaseSettingsMismatch
from filelock._identity import owner_is_stale
from filelock._marker import MarkerSoftFileLock, OwnerMode, OwnerRecord, parse_marker
from filelock._soft import _read_lock_file
from filelock._util import pin_inode, touch
if TYPE_CHECKING:
from collections.abc import Callable, Mapping
from typing import Unpack
from filelock._api import LockOptions, _ExtraValue
#: ``evicted`` is a :class:`~filelock.SoftReadWriteLock` loss: a peer waited out the stale threshold and committed a
#: generation without this holder.
CompromiseReason = Literal["marker-missing", "owner-changed", "refresh-failed", "evicted"]
_RefreshOutcome = Literal["ok", "transient", "marker-missing", "owner-changed"]
_DEFAULT_LEASE_DURATION: Final[float] = 30.0
@dataclass(frozen=True)
class LeaseCompromise:
"""Why a held lease stopped being this process's to hold, and the error that ended its refresh, if any."""
lock_file: str
token: str
reason: CompromiseReason
error: Exception | None = None
@dataclass(frozen=True)
class _Heartbeat:
"""A running heartbeat and the event that stops it, which only ever exist together."""
thread: Thread
stop: Event
@dataclass
class _LeaseClaim:
"""The state of one claim: its token, its heartbeat, and how that claim was lost."""
token: str | None = None
compromise: LeaseCompromise | None = None
heartbeat: _Heartbeat | None = None
class SoftFileLease(MarkerSoftFileLock):
"""
Existence lock whose claim expires, so a peer may take it while the previous holder still runs.
A lease trades mutual exclusion for progress. The holder publishes a claim and refreshes it every
``heartbeat_interval`` seconds; a contender takes the marker once it has seen no refresh for ``lease_duration``
seconds of its own monotonic clock, so clock skew between hosts cannot expire a live claim. Nothing stops the
expired holder: it keeps running, and it keeps using whatever the lock protects. Treat the lease as a hint about
who *should* be working, not as a guarantee that only one worker is.
To make a protected resource reject a superseded holder, that resource must be linearizable and must fence on a
monotonic generation it controls. :attr:`token` names a claim; it does not fence one. Where overlap is unacceptable,
use :class:`StrictSoftFileLock <filelock.StrictSoftFileLock>` instead.
Every contender for a path must agree on ``lease_duration``. A contender that finds a claim published under a
different duration raises :class:`LeaseSettingsMismatch <filelock.LeaseSettingsMismatch>` rather than apply its own
expiry to a peer that never agreed to it.
Expiry reclaims less on Windows, which refuses to rename or delete a file another process holds open. A peer there
takes an expired claim only once the previous holder's process exits and its handle closes; a holder that lives on
but stops refreshing keeps the marker. Unix reclaims the marker either way.
``on_compromise`` fires from the heartbeat thread when a refresh fails, or when the marker vanishes or names another
owner. The holder should stop touching the protected resource when it runs. Because it runs on that thread, a
``release()`` inside it only takes effect when the lease was built with ``thread_local=False``; the default
thread-local context hides the claim from every thread but the one that acquired it, so the release does nothing.
Signal the acquiring thread instead when the context stays thread-local.
.. versionadded:: 3.30.0
"""
_owner_mode: OwnerMode = "lease"
#: lease_duration replaces the legacy age-based lifetime, so accepting both would give one lock two expiry clocks.
_lifetime_supported: bool = False
_lifetime_unsupported_reason: str = "lease_duration sets when a lease expires"
def __init__(
self,
lock_file: str | os.PathLike[str],
*,
lease_duration: float = _DEFAULT_LEASE_DURATION,
heartbeat_interval: float | None = None,
on_compromise: Callable[[LeaseCompromise], None] | None = None,
**kwargs: Unpack[LockOptions],
) -> None:
"""
Create a lease.
:param lease_duration: seconds a contender must see the marker go unrefreshed before it may take the claim.
Every contender for the path must pass the same value.
:param heartbeat_interval: seconds between refreshes. Defaults to a third of ``lease_duration``, leaving room
for two missed refreshes before a peer may take the claim. Must be shorter than ``lease_duration`` and at
most :data:`threading.TIMEOUT_MAX`.
:param on_compromise: called from the heartbeat thread with a :class:`LeaseCompromise` when the claim is lost.
:param kwargs: every other :class:`BaseFileLock <filelock.BaseFileLock>` option, ``timeout`` and ``mode`` among
them. The metaclass passes them all by keyword, and taking them here lets
:class:`AsyncSoftFileLease <filelock.AsyncSoftFileLease>` add the async plumbing a fixed signature would
hide.
"""
# A plain float: the marker records its repr, which NumPy's float64 spells in a form no parser reads.
if not isfinite(lease_duration := _seconds("lease_duration", lease_duration)) or lease_duration <= 0:
msg = f"lease_duration must be positive and finite, got {lease_duration!r}"
raise ValueError(msg)
heartbeat_interval = (
lease_duration / 3 if heartbeat_interval is None else _seconds("heartbeat_interval", heartbeat_interval)
)
if not 0 < heartbeat_interval < lease_duration:
msg = f"heartbeat_interval must be positive and below lease_duration, got {heartbeat_interval!r}"
raise ValueError(msg)
_check_timeout_max("heartbeat_interval", heartbeat_interval)
super().__init__(lock_file, **kwargs)
self._lease_duration = lease_duration
self._heartbeat_interval = heartbeat_interval
self._on_compromise = on_compromise
# Two threads lazily creating a shared claim could each get one, and release() would miss the heartbeat's.
if not self.is_thread_local():
self._context.lease_claim = _LeaseClaim()
def _singleton_extra_mismatches(self, kwargs: Mapping[str, _ExtraValue], /) -> dict[str, tuple[str, str]]:
mismatches = super()._singleton_extra_mismatches(kwargs)
lease_duration: Final = _seconds(
"lease_duration", cast("float", kwargs.get("lease_duration", _DEFAULT_LEASE_DURATION))
)
if lease_duration != self._lease_duration:
mismatches["lease_duration"] = (str(lease_duration), str(self._lease_duration))
# An omitted heartbeat_interval resolves as in __init__; a differing duration is already reported above.
heartbeat_interval: Final = (
self._lease_duration / 3
if (passed := kwargs.get("heartbeat_interval")) is None
else _seconds("heartbeat_interval", cast("float", passed))
)
if heartbeat_interval != self._heartbeat_interval:
mismatches["heartbeat_interval"] = (str(passed), str(self._heartbeat_interval))
# A callback compares by identity, as on_acquired does: two equal callables can close over different state.
if (on_compromise := kwargs.get("on_compromise")) is not self._on_compromise:
mismatches["on_compromise"] = (str(on_compromise), str(self._on_compromise))
return mismatches
@property
def _claim(self) -> _LeaseClaim:
# Per hold, so another thread's failed acquire cannot stop the holder's heartbeat.
state: Final = self._hold_state()
if (claim := state.lease_claim) is None:
claim = state.lease_claim = _LeaseClaim()
return claim
@property
def lease_duration(self) -> float:
"""The staleness in seconds after which a contender may take this claim."""
return self._lease_duration
@property
def token(self) -> str | None:
"""
The token naming the claim this process published.
:returns: the token while the lease is held, ``None`` otherwise. It identifies a claim; it does not fence one.
"""
return self._claim.token
@property
def compromise(self) -> LeaseCompromise | None:
"""
The loss of claim the heartbeat observed.
The value outlives ``release()``, so a holder can check after leaving the ``with`` block whether its work may
have overlapped a successor; the next acquisition resets it, as :attr:`SoftReadWriteLock.compromise
<filelock.SoftReadWriteLock.compromise>` does.
:returns: the :class:`LeaseCompromise` of the latest claim, or ``None`` while that claim holds or held to its
release
"""
return self._claim.compromise
def _acquire(self) -> None:
claim = self._claim
self._stop_heartbeat() # no earlier claim's heartbeat outlives the acquisition of the next one
# The published record reads the token, so it exists before the marker is written and goes back on failure.
claim.token = token = secrets.token_hex(16)
claim.compromise = None
try:
super()._acquire()
except BaseException:
claim.token = None
raise
# The context is thread-local by default, so the heartbeat thread cannot read the descriptor this one just
# published, nor the claim this one owns. Hand it the fd, the inode it verified and the claim instead.
if (fd := self._context.lock_file_fd) is not None and (
identity := self._context.lock_file_fd_identity
) is not None:
self._start_heartbeat(claim, fd, identity, token)
else:
claim.token = None
def _release(self) -> None:
self._stop_heartbeat()
self._claim.token = None
super()._release()
def _published_record(self) -> OwnerRecord:
return super()._published_record()._replace(token=self._claim.token, lease_duration=self._lease_duration)
def _try_break_stale_lock(self) -> None:
# A break can fail for reasons a contender must ride out rather than raise on: a peer broke the marker first,
# or Windows refuses to rename a file whose holder still has it open. Poll again instead.
with suppress(OSError), pin_inode(self._lock_path) as pinned:
self._try_break_pinned_lock(pinned)
def _try_break_pinned_lock(self, pinned: int | None) -> None:
if (peer := self._read_peer()) is None:
# Not a readable protocol 2 lease record: a partial write, a foreign or legacy protocol 1 marker, or the
# strict sentinel. The base self-heal evicts a genuinely malformed marker once it ages past the grace
# window and leaves a legitimate legacy or strict holder in place, so a corrupt marker no longer wedges
# every lease contender until its own timeout.
super()._try_break_stale_lock()
return
owner, marker = peer
if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing
return # replaced before the read, so nothing keeps the number read from reuse
# Only a peer that published a lease agreed to be superseded by age; an exclusive owner is reclaimed only once
# it is provably dead, and a contract this version does not know never is. Raise the mismatch outside the read
# so the suppression cannot swallow it.
if owner.mode == "lease" and owner.lease_duration != self._lease_duration:
msg = (
f"{self.lock_file} holds a lease of {owner.lease_duration!r}s but this contender configured "
f"{self._lease_duration!r}s; every contender for a path must agree on lease_duration"
)
raise LeaseSettingsMismatch(msg)
# A dead or recycled owner is reclaimed at once, whether it held a lease or an exclusive marker; a live lease
# owner whose marker stops being refreshed is superseded on the schedule every contender agreed to.
if owner.mode != "unknown" and owner_is_stale(owner.pid, owner.hostname, owner.start):
self._take_over(marker, owner.pid)
return
if owner.mode == "lease" and self._marker_unchanged_for(marker) >= self._lease_duration:
self._take_over(marker, None)
def _read_peer(self) -> tuple[OwnerRecord, os.stat_result] | None:
with suppress(OSError, ValueError):
content, marker = _read_lock_file(self._lock_path)
if (owner := parse_marker(content)) is not None:
return owner, marker
return None
def _start_heartbeat(self, claim: _LeaseClaim, fd: int, identity: tuple[int, int], token: str) -> None:
# The thread watches the event it was handed rather than whatever the claim names later: a heartbeat that
# outlives its join timeout would otherwise adopt the next acquisition's event and never stop.
stop: Final = Event()
loop: Final = _RefreshLoop(
# A strong reference from a running thread would keep a dropped lease alive, so __del__ would never release
# it and its marker would outlive the process. The callback stops a heartbeat that __del__ could not reach.
report=WeakMethod(self._report_compromise, lambda _: stop.set()),
lock_file=self._lock_path,
claim=claim,
fd=fd,
identity=identity,
token=token,
stop=stop,
interval=self._heartbeat_interval,
duration=self._lease_duration,
)
thread: Final = Thread(target=loop.run, name=f"filelock-lease-{os.getpid()}", daemon=True)
# Record the heartbeat before starting the thread so a release racing this acquire on a shared,
# non-thread-local claim always sees it and sets the stop event; the thread then exits at its first wait
# instead of outliving the release. A start that raises leaves the unstarted thread for _stop_heartbeat.
claim.heartbeat = _Heartbeat(thread, stop)
thread.start()
def _stop_heartbeat(self) -> None:
claim = self._claim
if (heartbeat := claim.heartbeat) is None:
return
heartbeat.stop.set()
claim.heartbeat = None
# An unstarted thread has nothing to join, on_compromise may release from the heartbeat thread itself, and
# during finalization a daemon thread is frozen, so joining it raises PythonFinalizationError (3.14+).
if heartbeat.thread.ident is not None and heartbeat.thread is not current_thread() and not sys.is_finalizing():
heartbeat.thread.join(timeout=self._heartbeat_interval)
def _report_compromise(
self,
claim: _LeaseClaim,
reason: CompromiseReason,
error: OSError | None,
token: str,
) -> None:
# Record it on the claim this heartbeat serves, not on self._claim: a thread-local claim read from the
# heartbeat thread is a different, empty one, so the holder would never see the loss it is being told about.
# The token is the one this thread published, not claim.token, which a release may already have cleared.
claim.compromise = LeaseCompromise(lock_file=self.lock_file, token=token, reason=reason, error=error)
if self._on_compromise is not None:
self._on_compromise(claim.compromise)
@dataclass(frozen=True)
class _RefreshLoop:
"""What a heartbeat thread needs to refresh one claim, holding the lease itself only weakly."""
report: WeakMethod[Callable[[_LeaseClaim, CompromiseReason, OSError | None, str], None]]
lock_file: str
claim: _LeaseClaim
fd: int
identity: tuple[int, int]
token: str
stop: Event
interval: float
duration: float
def run(self) -> None:
# A transient error (ESTALE / EIO on the NFS-style filesystems a lease targets) is not a loss. Like restic, call
# the claim unrefreshable once failures run long enough that a contender could take it before the next success.
last_success, missing = time.monotonic(), False
while not self.stop.wait(self.interval):
outcome, error = self._refresh()
if outcome == "ok":
last_success, missing = time.monotonic(), False
continue
# A peer's stale break exchanges a live marker aside for the length of its check, so only a second miss or
# change in a row is a loss.
if outcome in {"marker-missing", "owner-changed"} and not missing:
missing = True
continue
if outcome == "transient" and time.monotonic() - last_success < self.duration - self.interval:
continue
if (report := self.report()) is not None:
report(self.claim, "refresh-failed" if outcome == "transient" else outcome, error, self.token)
return
def _refresh(self) -> tuple[_RefreshOutcome, OSError | None]:
try:
st = os.lstat(self.lock_file)
except FileNotFoundError as error:
return self._refresh_unnamed(error)
except OSError as error:
return "transient", error
# A peer that took the expired claim replaced the marker, so the pathname now names its inode, not ours.
if (st.st_dev, st.st_ino) != self.identity:
return "owner-changed", None
return self._touch()
def _refresh_unnamed(self, name_error: FileNotFoundError) -> tuple[_RefreshOutcome, OSError | None]:
# A renamed directory takes the marker with it; the name is gone but the inode behind the descriptor keeps its
# link.
try:
linked = os.fstat(self.fd).st_nlink > 0
except OSError as error:
return "transient", error
if not linked:
return "marker-missing", name_error
return self._touch()
def _touch(self) -> tuple[_RefreshOutcome, OSError | None]:
try:
touch(self.lock_file, fd=self.fd)
except OSError as error:
return "transient", error
return "ok", None
__all__ = [
"CompromiseReason",
"LeaseCompromise",
"SoftFileLease",
"_LeaseClaim",
]
+189
View File
@@ -0,0 +1,189 @@
from __future__ import annotations
__lazy_modules__ = ["contextlib", "filelock._identity", "filelock._util", "math"]
import math
import os
import re
from contextlib import suppress
from typing import Final, Literal, NamedTuple
from filelock._identity import host_name, owner_is_current_process, owner_is_stale, process_start_token
from filelock._soft import SoftFileLock, _read_lock_file, parse_decimal
from filelock._util import pin_inode, write_all
#: Protocol 1 is the legacy ``<pid>\n<hostname>\n[<start_token>\n]`` marker that :class:`SoftFileLock` still writes.
#: Protocol 2 carries the owner mode and the lease claim. A protocol 1 reader treats a protocol 2 marker as malformed
#: and evicts it after its grace period, so the two never guarantee mutual exclusion against each other.
_PROTOCOL: Final[str] = "filelock/2"
_MAX_PID: Final[int] = 2**31 - 1
#: The forms ``repr`` gives a non-negative finite float or int; ``float()`` would also accept whitespace, underscores
#: and non-ASCII digits.
_DURATION: Final[re.Pattern[str]] = re.compile(r"[0-9]+(?:\.[0-9]+)?(?:e[+-][0-9]+)?", re.ASCII)
#: Preserve unknown contracts so contenders cannot mistake them for malformed, reclaimable markers.
OwnerMode = Literal["lease", "exclusive", "unknown"]
class MarkerSoftFileLock(SoftFileLock):
"""An existence lock whose marker carries a protocol 2 owner record."""
#: Age-based expiry requires a lease contract; an exclusive holder grants none.
_owner_mode: OwnerMode = "exclusive"
@property
def owner(self) -> OwnerRecord | None:
"""
The owner named by the marker on disk.
:returns: the published record, or ``None`` when no marker exists or its record is malformed or protocol 1
"""
return self._read_owner()
@property
def pid(self) -> int | None:
"""
The PID of the process holding this lock, read from the marker.
:returns: the PID, or ``None`` when no marker exists or its record is unreadable
"""
return None if (owner := self._read_owner()) is None else owner.pid
@property
def is_lock_held_by_us(self) -> bool:
"""
Whether the marker on disk names this process.
:returns: ``True`` when the marker names this process's PID, hostname and, when recorded, start token
"""
owner = self._read_owner()
return owner is not None and owner_is_current_process(owner.pid, owner.hostname, owner.start)
def force_break(self) -> None:
"""
Remove the marker whoever holds it, so a later contender can acquire.
Forced breaking voids mutual exclusion: the previous holder keeps running and keeps using whatever the lock
protects. Reserve it for an operator clearing a marker whose holder is known to be gone.
"""
self.break_lock()
def _try_break_stale_lock(self) -> None:
with suppress(OSError, ValueError), pin_inode(self._lock_path) as pinned:
content, marker = _read_lock_file(self._lock_path)
owner: Final[OwnerRecord | None]
if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing
return # replaced before the read, so nothing keeps the number read from reuse
if (owner := parse_marker(content)) is None:
super()._try_break_stale_lock()
elif owner.mode != "unknown" and owner_is_stale(owner.pid, owner.hostname, owner.start):
self._take_over(marker, owner.pid)
@classmethod
def _recorded_holder(cls, content: str | None) -> tuple[int, str] | None:
if (owner := parse_marker(content)) is not None:
return owner.pid, owner.hostname
return super()._recorded_holder(content)
@classmethod
def _marker_is_live(cls, content: str | None, breaker_pid: int) -> bool: # pragma: win32 no cover
if (owner := parse_marker(content)) is None:
return super()._marker_is_live(content, breaker_pid)
return owner.mode == "unknown" or (
(owner.pid != breaker_pid or owner.hostname != host_name())
and not owner_is_stale(owner.pid, owner.hostname, owner.start)
)
def _read_owner(self) -> OwnerRecord | None:
with suppress(OSError, ValueError):
return parse_marker(_read_lock_file(self._lock_path)[0])
return None
def _write_lock_info(self, fd: int) -> None:
write_all(fd, encode_marker(self._published_record()))
def _published_record(self) -> OwnerRecord:
return OwnerRecord(
pid=os.getpid(),
hostname=host_name(),
mode=self._owner_mode,
start=process_start_token(os.getpid()),
)
class OwnerRecord(NamedTuple):
"""Keep optional metadata when reading unknown lock modes."""
pid: int
hostname: str
mode: OwnerMode
token: str | None = None
lease_duration: float | None = None
start: int | None = None
def encode_marker(record: OwnerRecord) -> bytes:
"""Render an owner record as the bytes a protocol 2 marker holds."""
lines = [_PROTOCOL, f"pid={record.pid}", f"host={record.hostname}", f"mode={record.mode}"]
if record.token is not None:
lines.append(f"token={record.token}")
if record.lease_duration is not None:
lines.append(f"duration={record.lease_duration!r}")
if record.start is not None:
lines.append(f"start={record.start}")
return "".join(f"{line}\n" for line in lines).encode()
def parse_marker(content: str | None) -> OwnerRecord | None:
"""Return the owner a protocol 2 marker names, or ``None`` when the record is malformed or protocol 1."""
# A torn write may end in a prefix like "mode=exclu", an unknown contract nobody reclaims; malformed, it ages out.
if not content or content[-1] != "\n" or not (lines := content.strip().splitlines()) or lines[0] != _PROTOCOL:
return None
fields: dict[str, str] = {}
for line in lines[1:]:
key, separator, value = line.partition("=")
if not separator:
return None
fields[key] = value
return _build_record(fields)
def _build_record(fields: dict[str, str]) -> OwnerRecord | None:
# An unknown key is a field a newer filelock published, so ignore it rather than read the record as malformed. An
# unrecognized mode is the same story one level up: a contract this version does not implement. Reading it as
# malformed would age the marker out of a live owner's hands, so keep it and let the caller refuse to reclaim it.
# A record naming no mode at all states no contract and stays malformed.
if (published := fields.get("mode")) is None:
return None
mode: Final[OwnerMode] = published if published in {"lease", "exclusive"} else "unknown"
hostname = fields.get("host")
if not hostname or "pid" not in fields or ("duration" in fields and not _DURATION.fullmatch(fields["duration"])):
return None
try:
pid = parse_decimal(fields["pid"])
duration = float(fields["duration"]) if "duration" in fields else None
start = parse_decimal(fields["start"]) if "start" in fields else None
except ValueError:
return None
if not 1 <= pid <= _MAX_PID:
return None
token = fields.get("token")
# "1e400" passes the grammar but overflows to inf, which mismatches every configured duration and so wedges reclaim,
# where a malformed marker ages out.
if mode == "lease" and (not token or duration is None or not (math.isfinite(duration) and duration > 0)):
return None
return OwnerRecord(pid=pid, hostname=hostname, mode=mode, token=token, lease_duration=duration, start=start)
__all__ = [
"MarkerSoftFileLock",
"OwnerMode",
"OwnerRecord",
"encode_marker",
"parse_marker",
]
File diff suppressed because it is too large. Load diff
+643
View File
@@ -0,0 +1,643 @@
from __future__ import annotations
__lazy_modules__ = [
"contextlib",
"filelock._identity",
"filelock._soft_protocol",
"filelock._util",
"itertools",
"pathlib",
"re",
"secrets",
"typing",
]
import os
import re
import secrets
import stat
import sys
import threading
import time
from contextlib import suppress
from errno import EACCES, EEXIST, EPERM
from itertools import islice
from pathlib import Path
from typing import Final
from filelock._api import BaseFileLock, _raise_grouped_errors
from filelock._identity import host_name, owner_is_current_process, owner_is_stale, process_start_token
from filelock._soft_protocol import STRICT_SOFT_SENTINEL_RECORD
from filelock._util import (
break_lock_file,
exchange_names,
exchange_supported,
pin_inode,
raise_on_not_writable_file,
remove_own_file,
write_all,
)
if sys.platform == "win32": # pragma: win32 cover
from filelock._windows import open_read_without_oplock_wait
from filelock._windows import unlink_without_oplock_wait as unlink_path
else: # pragma: win32 no cover
from filelock._util import unlink_path
MALFORMED_LOCK_AGE_THRESHOLD: Final[float] = 2.0
_MAX_LOCK_FILE_SIZE: Final[int] = 1024
_UNLINK_MAX_RETRIES: Final[int] = 10
_MARKER_WITH_START_TOKEN_LINE_COUNT: Final[int] = 3
#: Contenders act on the record in the marker, so no other account may write it, whatever the mode or umask.
_OTHERS_WRITE: Final[int] = stat.S_IWGRP | stat.S_IWOTH
#: All a SoftFileLock before 3.22 writes, and what a current one shows between creating its marker and writing it.
_RECORDLESS_MARKER: Final[str] = ""
_CLAIMS_IN_PROGRESS: Final[set[str]] = set()
_CLAIMS_LOCK: Final[threading.Lock] = threading.Lock()
class SoftFileLock(BaseFileLock):
"""
Cooperative file lock based on a shared existence marker.
Unlike :class:`UnixFileLock <filelock.UnixFileLock>` and :class:`WindowsFileLock <filelock.WindowsFileLock>`, this
lock does not use OS-level locking primitives. Instead, it creates the lock file with ``O_CREAT | O_EXCL`` and
treats its existence as the lock indicator. The filesystem must provide coherent exclusive creation and directory
updates to each participating process. A crash can leave the marker behind.
The marker contains the holder's PID and hostname. A contender may remove it when it can no longer find a same-host
process with that PID. A configured :attr:`~filelock.BaseFileLock.lifetime` also permits removal based on marker
age, including while the holder remains alive. Age-based expiry can overlap protected operations and does not
provide strict mutual exclusion.
"""
#: Existence locks reclaim by unlinking a pathname, so an age-based lease may break one; a native inode lock cannot.
_lifetime_supported: bool = True
#: Age-based expiry preserves historical behavior but does not provide strict mutual exclusion.
_lifetime_replacements: tuple[str, str] | None = ("StrictSoftFileLock", "SoftFileLease")
#: An existence lock unlinks its marker to release, so it cannot promise to keep the pathname.
_preserve_lock_file_supported: bool = False
#: An existence lock keeps protocol state in its marker, so it cannot lend the descriptor to an on_acquired hook.
_on_acquired_supported: bool = False
#: The marker's (mtime, inode, ctime) when this instance last saw it change, and when, on its monotonic clock.
_marker_seen: tuple[tuple[float, int, int], float] | None = None
#: The identity of a marker a Windows release relinquished but has not unlinked yet.
_marker_cleanup: tuple[int, int] | None = None
#: The descriptor of that marker while its close has not run yet, as a list so one C call can take it.
_marker_fds: list[int]
def _acquire(self) -> None:
self._finish_marker_cleanup()
raise_on_not_writable_file(self._lock_path)
# O_CREAT | O_EXCL makes the create fail with EEXIST when the file already exists, so a successful open
# means this process now holds the lock.
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: # pragma: needs o-nofollow
flags |= o_nofollow
try:
fd = os.open(self._lock_path, flags, self._open_mode() & ~_OTHERS_WRITE)
except OSError as exception:
if not (
exception.errno == EEXIST or (exception.errno == EACCES and sys.platform == "win32")
): # pragma: win32 no cover
raise
self._try_break_stale_lock()
return
# Publish held state only once the record is on disk. On failure, cancellation included, close the descriptor
# and unlink the path only while it still names the file we opened: after an expired lease a successor's
# marker may sit at the same path. The guard starts at the open, so a signal handler that raises before
# _mark_descriptor_owned leaves no marker behind.
identity: tuple[int, int] | None = None
try:
identity = _file_identity(os.fstat(fd))
self._mark_descriptor_pending(fd)
self._apply_explicit_mode(fd)
self._write_lock_info(fd)
self._mark_descriptor_owned(fd, identity)
except BaseException:
self._mark_descriptor_released()
try:
if sys.platform != "win32": # pragma: win32 no cover
# Before the close: our open descriptor keeps the inode allocated, so a successor's marker cannot
# carry its number.
with suppress(OSError):
remove_own_file(self._lock_path, fd)
finally:
try:
os.close(fd)
finally:
if sys.platform == "win32": # pragma: win32 cover
self._unlink_held_marker(identity)
raise
def _apply_explicit_mode(self, fd: int) -> None:
# os.open filters the mode through the umask, so an explicit mode lands as given only through fchmod, the way
# UnixFileLock applies it. Windows maps the mode to a read-only bit that os.open already set.
if self.has_explicit_mode and sys.platform != "win32": # pragma: win32 no cover
with suppress(PermissionError):
os.fchmod(fd, self._context.mode & ~_OTHERS_WRITE)
def _try_break_stale_lock(self) -> None:
with suppress(OSError, ValueError), pin_inode(self._lock_path) as pinned:
content, marker = _read_lock_file(self._lock_path)
if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing
return # another node replaced the pinned one before the read, and nothing keeps its number from reuse
if content == STRICT_SOFT_SENTINEL_RECORD: # pragma: needs hard-link
return
# Such a marker is held for as long as its owner runs, so being unchanged proves nobody gone.
if content == _RECORDLESS_MARKER:
return
holder = _parse_lock_holder(content)
if holder is None:
# Unparsable: wrong line count, a non-integer PID or start token, empty, oversized or not UTF-8.
# Self-heal only once the file is clearly not a half-written fresh lock (a peer between O_EXCL and
# _write_lock_info), so the brief create-then-write window is never mistaken for a stale lock.
if self._marker_unchanged_for(marker) >= MALFORMED_LOCK_AGE_THRESHOLD:
self._take_over(marker, _placeholder_breaker(content))
return
if owner_is_stale(*holder):
self._take_over(marker, holder[0])
def _try_break_expired_lock(self) -> None:
if (lifetime := self._context.lifetime) is None:
return
with suppress(OSError), pin_inode(self._lock_path) as pinned:
# _read_lock_file takes the lstat, not stat: a symlink swapped in at the path must report its own mtime.
content, marker = _read_lock_file(self._lock_path)
if pinned not in {None, marker.st_ino} or time.time() - marker.st_mtime < lifetime:
return
holder = self._recorded_holder(content)
self._take_over(marker, None if holder is None else holder[0])
def _take_over(self, marker: os.stat_result, breaker_pid: int | None) -> None:
# Swap the judged marker for a complete marker of ours in one exchange, so the lock path never stands empty.
# Once the file swapped out proves to be the one judged, this lock holds the path; the caller's pin keeps that
# file's inode number from naming anything else. Only a peer's exchange can move our marker after that, and
# the peer puts it back because what it took is not the file it judged. Every failure before the handoff takes
# our marker off the path and puts back what the exchange took. Without an exchange, the portable break decides.
if exchange_supported(): # pragma: win32 no cover
self._take_over_by_exchange(marker, breaker_pid)
else:
break_lock_file(self._lock_path, marker.st_mtime, marker.st_ino)
def _take_over_by_exchange(self, marker: os.stat_result, breaker_pid: int | None) -> None: # pragma: win32 no cover
if (current := os.lstat(self._lock_path)).st_mtime != marker.st_mtime or current.st_ino != marker.st_ino:
return
if breaker_pid is not None and self._restore_stranded_marker(breaker_pid, marker):
return
aside: Final = f"{self._lock_path}.break.{os.getpid()}.{secrets.token_hex(16)}"
flags: Final = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
fd: Final = os.open(aside, flags, self._open_mode() & ~_OTHERS_WRITE)
self._mark_descriptor_pending(fd)
exchanged = held = False
try:
identity: Final = _file_identity(os.fstat(fd))
self._apply_explicit_mode(fd)
self._write_lock_info(fd)
if not exchange_names(aside, self._lock_path):
return # NFS, SMB and the like refuse it; renaming the marker aside would empty the path
exchanged = True
if (taken := os.lstat(aside)).st_mtime != marker.st_mtime or taken.st_ino != marker.st_ino:
return # a peer recreated the marker, which the cleanup below swaps back
Path(aside).unlink()
exchanged = False
self._mark_descriptor_owned(fd, identity)
held = True
finally:
if not held:
self._abandon_take_over(fd, aside, put_back=exchanged)
def _restore_stranded_marker(self, breaker_pid: int, marker: os.stat_result) -> bool: # pragma: win32 no cover
# A breaker killed between its exchange and the put-back leaves its own marker at the path and the live
# holder's marker under its break name. A recoverer first renames that record to a name of its own. Only one
# contender wins the rename, so two of them never exchange the same record and reverse each other. A record
# a dead recoverer holds is claimable again. A claim of this process that no thread is working on was
# abandoned by a failed put-back, and the next poll takes it up again. Where the marker cannot be put back
# the lock counts as held, as it does wherever no exchange is possible.
if (candidates := self._stranded_candidates(breaker_pid)) is None:
return True
return any(self._recover_claimed(path, breaker_pid, marker, adopt=abandoned) for path, abandoned in candidates)
def _stranded_candidates(self, breaker_pid: int) -> list[tuple[str, bool]] | None: # pragma: win32 no cover
directory, name = os.path.split(self._lock_path)
stranded: Final = re.compile(rf"{re.escape(name)}\.break\.{breaker_pid}\.[0-9a-f]{{32}}")
claimed: Final = re.compile(rf"{re.escape(name)}\.restore\.(\d+)\.[0-9a-f]{{32}}")
candidates: list[tuple[str, bool]] = []
with os.scandir(directory or os.curdir) as entries:
for entry in entries:
if not entry.is_file(follow_symlinks=False):
continue
path = str(Path(directory) / entry.name)
if stranded.fullmatch(entry.name):
candidates.append((path, False))
elif found := claimed.fullmatch(entry.name):
ours = int(found[1]) == os.getpid()
if not (ours or owner_is_stale(int(found[1]), host_name(), None)):
return None
candidates.append((path, ours))
return sorted(candidates)
def _recover_claimed(
self, path: str, breaker_pid: int, marker: os.stat_result, *, adopt: bool
) -> bool: # pragma: win32 no cover
# One try/finally owns the registration from the first step to the last, so no interruption can leave it set.
# Registered before the rename publishes the name, so a thread that scans meanwhile sees a claim in progress.
own: Final = path if adopt else f"{self._lock_path}.restore.{os.getpid()}.{secrets.token_hex(16)}"
if not _begin_claim(own):
return True
try:
if not adopt:
try:
Path(path).rename(own)
except OSError: # a peer that won the rename now holds the record, so the lock counts as held
return True
return self._restore_claimed(own, breaker_pid, marker)
finally:
_end_claim(own)
def _restore_claimed(self, own: str, breaker_pid: int, marker: os.stat_result) -> bool: # pragma: win32 no cover
try:
content, _ = _read_lock_file(own)
except OSError:
self._return_claim(own, breaker_pid)
return True
if not self._marker_is_live(content, breaker_pid):
with suppress(OSError):
Path(own).unlink()
return False
# The inode of the marker judged dead must still be at the path, or a peer has acted since.
with suppress(OSError):
current = os.lstat(self._lock_path)
if (current.st_ino, current.st_mtime) == (marker.st_ino, marker.st_mtime) and exchange_names(
own, self._lock_path
):
if os.lstat(own).st_ino == marker.st_ino:
Path(own).unlink()
return True
# A peer replaced the path after the check, so the exchange took its marker: swap it straight back.
exchange_names(own, self._lock_path)
self._return_claim(own, breaker_pid)
return True
def _return_claim(self, own: str, breaker_pid: int) -> None: # pragma: win32 no cover
# A refused rename leaves the claim under this process's PID, and the next poll retries it.
with suppress(OSError):
Path(own).rename(f"{self._lock_path}.break.{breaker_pid}.{secrets.token_hex(16)}")
@classmethod
def _marker_is_live(cls, content: str | None, breaker_pid: int) -> bool: # pragma: win32 no cover
if content == STRICT_SOFT_SENTINEL_RECORD: # pragma: needs hard-link
return True
if content == _RECORDLESS_MARKER:
# An empty marker is a pre-3.22 holder for as long as it exists, as the stale check treats it.
return True
# The breaker's own marker names its PID on this host; the same number on another host is a holder.
return (
(holder := _parse_lock_holder(content)) is not None
and (holder[0] != breaker_pid or holder[1] != host_name())
and not owner_is_stale(*holder)
)
def _abandon_take_over(self, fd: int, aside: str, *, put_back: bool) -> None: # pragma: win32 no cover
self._mark_descriptor_released()
try:
if put_back:
exchange_names(aside, self._lock_path)
finally:
try:
# Whichever name still holds our marker loses it: the descriptor keeps its inode number ours.
own: Final = os.fstat(fd)
for name in (self._lock_path, aside):
with suppress(FileNotFoundError):
if os.path.samestat(os.lstat(name), own):
Path(name).unlink()
finally:
os.close(fd)
def _marker_unchanged_for(self, marker: os.stat_result) -> float:
# The mtime is the writer's or file server's clock, so against time.time() a contender whose clock runs ahead
# would expire a fresh marker. Only watch it for change, timed on this process's monotonic clock. The owner of a
# file sets its mtime, and ext4 reuses a freed inode number, so a new marker can repeat both. The change time
# cannot be set and moves with every create, which keeps a new marker from inheriting an old one's age.
now: Final = time.monotonic()
key: Final = marker.st_mtime, marker.st_ino, marker.st_ctime_ns
if (seen := self._marker_seen) is None or seen[0] != key:
self._marker_seen = key, now
return 0.0
return now - seen[1]
def _unprobeable_holder(self) -> str | None:
# This host cannot prove such a holder dead, so whoever decides whether it is gone needs its name.
with suppress(OSError, ValueError):
if (content := _read_lock_file(self._lock_path)[0]) == _RECORDLESS_MARKER:
return "an owner that recorded no PID (a filelock release before 3.22, or one still writing its record)"
if (holder := self._recorded_holder(content)) and holder[1] != host_name():
return f"pid {holder[0]} on {holder[1]}"
return None
@classmethod
def _recorded_holder(cls, content: str | None) -> tuple[int, str] | None:
return None if (holder := _parse_lock_holder(content)) is None else holder[:2]
@staticmethod
def _write_lock_info(fd: int) -> None:
# No suppression: a write failure must reach the acquisition rollback so it never publishes a half-written
# marker as held state. The optional third line is this process's start token, absent when the platform
# exposes no proven start time, in which case a reader falls back to PID-only liveness.
info = f"{os.getpid()}\n{host_name()}\n"
if (token := process_start_token(os.getpid())) is not None:
info += f"{token}\n"
write_all(fd, info.encode())
@property
def pid(self) -> int | None:
"""
The PID of the process holding this lock, read from the lock file.
:returns: the PID as an integer, or ``None`` if the lock file does not exist or cannot be parsed
"""
with suppress(OSError, ValueError):
holder = _parse_lock_holder(_read_lock_file(self._lock_path)[0])
if holder is not None:
return holder[0]
return None
@property
def is_lock_held_by_us(self) -> bool:
"""
Whether this lock is held by the current process.
:returns: ``True`` if the lock file exists and names the current process's PID, hostname and, when recorded,
start token
"""
with suppress(OSError, ValueError):
if (holder := _parse_lock_holder(_read_lock_file(self._lock_path)[0])) is not None:
return owner_is_current_process(*holder)
return False
def break_lock(self) -> None:
"""Forcibly break the lock by removing the lock file, regardless of who holds it."""
with suppress(OSError):
Path(self._lock_path).unlink()
def _release_current(self, *, force: bool) -> None:
try:
super()._release_current(force=force)
finally:
# A release a handler cut short after the descriptor closed still owes the unlink on Windows.
self._finish_marker_cleanup()
def _release(self) -> None:
fd = self._context.lock_file_fd
assert fd is not None # ruff:ignore[assert] # _release runs only while held, so the descriptor is set
# Capture the held file's identity before closing so cleanup can refuse to unlink a successor's marker. A
# supported lifetime lease lets a peer break our expired marker and create its own at this path before we
# release; unlinking by path alone would then delete the successor's lock.
identity: tuple[int, int] | None = None
with suppress(OSError):
identity = _file_identity(os.fstat(fd))
if sys.platform == "win32": # pragma: win32 cover
self._release_then_unlink(fd, identity)
else: # pragma: win32 no cover
# Our open descriptor keeps the inode allocated, so its number cannot name a successor's marker yet, and an
# interrupt before the unlink lands leaves the lock held for the retried release to finish.
unlinked: Final = self._unlink_open_marker(fd, identity)
# A failed close may already have released and recycled the descriptor number. Relinquish it before the
# one close attempt so no later release can close an unrelated descriptor that reused the same integer.
self._mark_descriptor_released()
try:
self._close_released_fd(fd, default_suppresses=False)
finally:
# A share that refuses to delete an open file gets the unlink after the close; the marker still exists
# until then, so its inode cannot be reused either.
if not unlinked:
self._unlink_held_marker(identity)
def _release_then_unlink(self, fd: int, identity: tuple[int, int] | None) -> None: # pragma: win32 cover
# Windows cannot unlink an open file, so the unlink outlives the hold until it lands, here or on a later acquire
# or release. Only the gate's holder touches the pending record, so two threads never overwrite each other's.
with self._transition_lock:
self._marker_fds = [fd]
self._marker_cleanup = identity
self._mark_descriptor_released()
try:
self._close_marker_fd()
# Marker cleanup must also run for control-flow exceptions, and both failures must remain observable.
except BaseException as close_error:
try:
self._finish_marker_cleanup()
except BaseException as cleanup_error: # ruff:ignore[blind-except] # keep control-flow failures
_raise_grouped_errors(
"lock descriptor close and marker cleanup both failed",
close_error,
cleanup_error,
)
raise
self._finish_marker_cleanup()
def _marker_names(self) -> tuple[str, ...]:
# After a rename of the lock's directory only the caller's spelling, resolved through the working directory,
# still reaches the marker this hold opened.
return tuple(dict.fromkeys((self._lock_path, self.lock_file)))
def _unlink_open_marker(self, fd: int, identity: tuple[int, int] | None) -> bool: # pragma: win32 no cover
if identity is None:
return True
replaced = False
for name in self._marker_names():
try:
if _file_identity(os.lstat(name)) == identity:
Path(name).unlink()
return True
replaced = True
except FileNotFoundError:
continue
except OSError:
return False
if replaced:
try:
_leave_released_record(fd)
except OSError:
return False
return True
def _finish_marker_cleanup(self) -> None:
# Never waits: whoever holds the gate is acquiring or releasing this lock and finishes the cleanup itself.
if self._marker_cleanup is not None and self._transition_lock.acquire(blocking=False): # pragma: win32 cover
try:
# A close error here belongs to the release an interrupt cut short, not to this caller.
with suppress(OSError):
self._close_marker_fd()
if (identity := self._marker_cleanup) is not None: # pragma: no branch # the gate's holder clears it
self._windows_unlink_if_ours(identity)
finally:
self._transition_lock.release()
def _close_marker_fd(self) -> None: # pragma: win32 cover
# Popping and closing is one chain of C calls that no signal handler can split, so a retried release closes the
# descriptor at most once; a second close could hit an unrelated file that reused the number.
try:
list(map(os.close, islice(iter(vars(self).setdefault("_marker_fds", []).pop, None), 1)))
except IndexError:
return
except OSError:
if not self._suppresses_close_error(default_suppresses=False):
raise
def _unlink_held_marker(self, identity: tuple[int, int] | None) -> None:
if sys.platform == "win32": # pragma: win32 cover
if identity is not None:
self._windows_unlink_if_ours(identity)
else: # pragma: win32 no cover
for name in self._marker_names():
with suppress(OSError):
if _file_identity(os.lstat(name)) == identity:
Path(name).unlink()
return
def _windows_unlink_if_ours(self, identity: tuple[int, int]) -> None: # pragma: win32 cover
retry_delay = 0.001
for attempt in range(_UNLINK_MAX_RETRIES):
# Windows doesn't immediately release file handles after close, causing EACCES/EPERM on unlink. Recheck
# identity each attempt: a failed unlink leaves a window for a successor to replace the marker at this path.
try:
for name in self._marker_names():
with suppress(FileNotFoundError):
if _file_identity(os.lstat(name)) == identity:
# Forgotten before the unlink: Windows does not promise unique file IDs, so no retry may
# reach a successor that reused ours.
self._marker_cleanup = None
unlink_path(name)
break
except OSError as exc:
if exc.errno in {EACCES, EPERM}:
# The file is still ours, so the next acquire or release tries again if these attempts run out.
self._marker_cleanup = identity
if attempt < _UNLINK_MAX_RETRIES - 1:
time.sleep(retry_delay)
retry_delay *= 2
continue
self._marker_cleanup = None
return
def _file_identity(st: os.stat_result) -> tuple[int, int]:
# (st_dev, st_ino) names the concrete inode behind a path, so a marker recreated at the same pathname after an
# expired lease reads as a different file. CPython populates both on Windows from the volume serial and file index.
return st.st_dev, st.st_ino
def _begin_claim(name: str | Path) -> bool: # pragma: win32 no cover
# Keyed by the file name alone: its random token identifies the claim, and the spelling of the directory varies.
with _CLAIMS_LOCK:
if (key := Path(name).name) in _CLAIMS_IN_PROGRESS:
return False
_CLAIMS_IN_PROGRESS.add(key)
return True
def _end_claim(name: str | Path) -> None: # pragma: win32 no cover
with _CLAIMS_LOCK:
_CLAIMS_IN_PROGRESS.discard(Path(name).name)
def _placeholder_breaker(content: str | None) -> int | None: # pragma: win32 no cover
# The placeholder a break leaves at the path while it works names the breaker that may have died there.
if content is None or not content.startswith(placeholder := "filelock/break\n") or not content.endswith("\n"):
return None
with suppress(ValueError):
return parse_decimal(content.removeprefix(placeholder)[:-1])
return None
def _leave_released_record(fd: int) -> None:
# A peer's exchange holds our marker under its break name and will put it back. Overwriting the record lets
# contenders age it out instead of waiting on this live PID. The write comes before the truncate so the file is
# never empty.
if sys.platform != "win32": # pragma: win32 no cover
os.pwrite(fd, released := b"filelock/released\n", 0)
os.ftruncate(fd, len(released))
def _read_lock_file(path: str) -> tuple[str | None, os.stat_result]:
# A legitimate lock file is always a regular file. Classify the path with lstat first, so any other node (symlink,
# FIFO, socket, device) is reported as a malformed lock the caller can evict, without an os.open that would follow
# a symlink, stall on a FIFO, or fail on a socket and leave acquisition wedged. The stat result still flows back
# for the identity-checked stale break. lstat, not stat, so a hostile symlink is never followed onto its target.
st = os.lstat(path)
if not stat.S_ISREG(st.st_mode): # pragma: needs fifo
return None, st
# Re-check on the opened handle: O_NOFOLLOW refuses a symlink swapped in after the lstat, O_NONBLOCK stops a FIFO
# swapped in from stalling the open, and the fstat catches any other non-regular replacement race before we read.
# The capped read stops a huge regular file (e.g. one filled from /dev/zero) from exhausting memory.
if sys.platform == "win32": # pragma: win32 cover
# os.open would wait for an oplock holder to answer its break, past the caller's deadline.
fd = open_read_without_oplock_wait(path)
else: # pragma: win32 no cover
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
try:
st = os.fstat(fd)
if not stat.S_ISREG(st.st_mode): # pragma: no cover # only a non-regular node swapped in after the lstat
return None, st
data = os.read(fd, _MAX_LOCK_FILE_SIZE + 1)
finally:
os.close(fd)
if len(data) <= _MAX_LOCK_FILE_SIZE:
with suppress(UnicodeDecodeError):
return data.decode("utf-8"), st
return None, st
def _parse_lock_holder(content: str | None) -> tuple[int, str, int | None] | None:
# A well-formed lock file is "<pid>\n<hostname>\n" with an optional "<start_token>\n" third line naming the
# holder's process start instant (a filelock 3.29 marker wrote this only on Windows; every platform writes it now).
# Anything else (wrong line count, a non-integer PID or start token, empty or unreadable content) is unparsable;
# returning None lets the caller treat it as a malformed lock to self-heal rather than a holder.
# Every writer since 3.22 ends each line with "\n", so a record without one was cut by a crash or read mid-write. As
# a holder it would name a foreign host no one reclaims, or a start token that lets a contender break a live lock.
if not content or not content.endswith("\n") or len(lines := content.strip().splitlines()) not in {2, 3}:
return None
try:
pid = parse_decimal(lines[0])
start_token = parse_decimal(lines[2]) if len(lines) == _MARKER_WITH_START_TOKEN_LINE_COUNT else None
except ValueError:
return None
# A pid outside the valid range is a malformed lock, not a holder. Without this, a non-positive pid
# reaches os.kill() where 0 / -1 mean "the caller's own process group / every process" so a dead
# holder reads as alive and the lock is never reclaimed, while an oversized pid raises OverflowError
# (not OSError/ValueError) out of the self-heal path. _parse_marker_bytes already enforces this range.
if not 1 <= pid <= 2**31 - 1:
return None
return pid, lines[1], start_token
def parse_decimal(text: str) -> int:
"""
Read a PID or start token as a writer formats it: ASCII digits only.
:raises ValueError: for anything else ``int()`` would accept, such as ``1_000``, surrounding whitespace, a sign or
non-ASCII digits; no filelock writes those
"""
if not (text.isascii() and text.isdigit()):
msg = f"not a decimal integer: {text!r}"
raise ValueError(msg)
return int(text)
__all__ = [
"MALFORMED_LOCK_AGE_THRESHOLD",
"SoftFileLock",
"parse_decimal",
]
@@ -0,0 +1,11 @@
from __future__ import annotations
__lazy_modules__ = ["typing"]
from typing import Final
STRICT_SOFT_SENTINEL_RECORD: Final[str] = "1\nfilelock-strict-v1\x00\n0\n"
__all__ = [
"STRICT_SOFT_SENTINEL_RECORD",
]
@@ -0,0 +1,14 @@
"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots."""
from __future__ import annotations
__lazy_modules__ = ["filelock._soft_rw._async"]
from filelock._soft_rw._async import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock
from filelock._soft_rw._sync import SoftReadWriteLock
__all__ = [
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncSoftReadWriteLock",
"SoftReadWriteLock",
]
@@ -0,0 +1,373 @@
"""Async wrapper around :class:`SoftReadWriteLock` for use with ``asyncio``."""
from __future__ import annotations
__lazy_modules__ = ["asyncio", "filelock._async", "filelock._error", "functools", "threading"]
import asyncio
import functools
import os
import threading
from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar
from filelock._async import (
_TASKS_THREAD_ID,
_BackendOutcome,
_capture_call,
_drain_future,
_future_result,
_raise_cancelled_error,
_task_owners_for,
_wait_until_done,
)
from filelock._error import Timeout
from filelock._soft_rw._sync import SoftReadWriteLock, _lost_hold_error
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable
from concurrent import futures
from types import TracebackType
from filelock._api import AcquireReturnProxy
from filelock._lease import LeaseCompromise
_P = ParamSpec("_P")
_R = TypeVar("_R")
class AsyncSoftReadWriteLock:
"""
Async wrapper around :class:`SoftReadWriteLock` for ``asyncio`` applications.
The sync class's blocking filesystem operations run on a thread pool via ``loop.run_in_executor()``. Each
``asyncio`` task owns its own hold. Tasks share a read lock, and a task asking for the write lock waits until the
other holders release, with waiting writers ahead of new readers. Upgrading or downgrading a held lock raises
:class:`RuntimeError`. The underlying :class:`SoftReadWriteLock` handles forks, the heartbeat, and stale eviction.
Singleton wrappers share one sync lock and its task holds, and closing one ends only the holds taken through it.
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
:param is_singleton: if ``True``, reuse existing :class:`SoftReadWriteLock` instances per resolved path
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
:param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to
``3 * heartbeat_interval``
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
:param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is
lost
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
:param executor: executor for ``run_in_executor``; ``None`` uses the default executor
.. versionadded:: 3.27.0
"""
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
on_compromise: Callable[[LeaseCompromise], None] | None = None,
loop: asyncio.AbstractEventLoop | None = None,
executor: futures.Executor | None = None,
) -> None:
self._creator_pid = os.getpid()
self._closed = False
self._owns_backend = not is_singleton
#: Marks the task holds taken through this wrapper, which its close() ends.
self._via: Final = object()
self._lock = SoftReadWriteLock(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
on_compromise=on_compromise,
)
self._owners = _task_owners_for(self._lock)
self._loop = loop
self._executor = executor
@property
def lock_file(self) -> str:
"""The path to the lock file passed to the constructor."""
return self._lock.lock_file
@property
def timeout(self) -> float:
"""The default timeout applied when ``acquire_read`` / ``acquire_write`` is called without one."""
return self._lock.timeout
@property
def blocking(self) -> bool:
"""Whether ``acquire_*`` defaults to blocking; ``False`` makes contention raise immediately."""
return self._lock.blocking
@property
def generation(self) -> int | None:
"""The generation the current hold was granted at, a fencing token; ``None`` when no lock is held."""
return self._lock.generation
@property
def compromise(self) -> LeaseCompromise | None:
"""How the current hold was lost, or ``None`` while it stands; acquiring raises until the hold is released."""
return self._lock.compromise
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
"""The event loop used for ``run_in_executor``, or ``None`` for the running loop."""
return self._loop
@property
def executor(self) -> futures.Executor | None:
"""The executor used for ``run_in_executor``, or ``None`` for the default executor."""
return self._executor
@asynccontextmanager
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases a shared read lock.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if the calling task already holds the write lock
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
await self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
@asynccontextmanager
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
"""
Async context manager that acquires and releases an exclusive write lock.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if the calling task already holds the read lock
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
await self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
await self.release()
async def acquire_read(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireSoftReadWriteReturnProxy:
"""
Acquire a shared read lock.
See :meth:`SoftReadWriteLock.acquire_read` for reentrancy / upgrade / fork semantics. The blocking work runs
inside ``run_in_executor`` so other coroutines on the same loop keep progressing while this call waits.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:returns: a proxy usable as an async context manager to release the lock
:raises RuntimeError: if the calling task already holds the write lock, if this instance was invalidated by
:func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
self._raise_if_unusable()
await self._acquire("read", timeout, blocking=blocking)
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
async def acquire_write(
self, timeout: float | None = None, *, blocking: bool | None = None
) -> AsyncAcquireSoftReadWriteReturnProxy:
"""
Acquire an exclusive write lock.
See :meth:`SoftReadWriteLock.acquire_write` for the writer-preferring semantics. The blocking work runs inside
``run_in_executor``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:returns: a proxy usable as an async context manager to release the lock
:raises RuntimeError: if the calling task already holds the read lock, if this instance was invalidated by
:func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
self._raise_if_unusable()
await self._acquire("write", timeout, blocking=blocking)
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
async def _acquire(self, mode: Literal["read", "write"], timeout: float | None, *, blocking: bool | None) -> None:
blocking = self._lock.blocking if blocking is None else blocking
async def enter(remaining: float) -> None:
# A close() may have landed while this task waited for the table.
self._raise_if_unusable()
await self._run_acquire(
functools.partial(
self._lock._acquire_level_of, # ruff:ignore[private-member-access] # the wrapper owns this backend hold
_TASKS_THREAD_ID,
mode,
remaining,
blocking=blocking,
)
)
def joinable() -> bool:
self._raise_if_unusable()
if (compromise := self._lock.compromise) is not None:
raise _lost_hold_error(self.lock_file, mode, compromise)
return self._backend_holds()
await self._owners.acquire(
mode,
timeout=self._lock.timeout if timeout is None else timeout,
blocking=blocking,
lock_file=self.lock_file,
via=self._via,
enter=enter,
joinable=joinable,
)
async def release(self, *, force: bool = False) -> None:
"""
Release one level of the calling task's hold.
The wrapper releases the lock once no task holds it.
:param force: if ``True``, end every level the calling task took through this wrapper. Levels taken through
other wrappers on the same path stay. A wrapper that holds no level for the task releases nothing.
:raises RuntimeError: if the calling task holds no level through this wrapper and *force* is ``False``
"""
if self._creator_pid == os.getpid():
await self._owners.release(
force=force,
lock_file=self.lock_file,
via=self._via,
leave=functools.partial(self._submit, self._leave_backend),
still_held=self._backend_holds,
)
async def close(self) -> None:
"""
End the holds of the tasks that acquired through this instance and release the lock once no task holds it.
Idempotent. After calling this method the instance can no longer acquire locks. With ``is_singleton=False`` it
also closes the underlying :class:`SoftReadWriteLock`. A singleton wrapper leaves the shared one open, so tasks
that acquired through another wrapper and synchronous users of the same path keep their holds. The next acquire
notices a synchronous ``close()`` or forced release of that shared lock. One that lands in the few bytecodes
between that check and the admission still admits the task into the ended hold.
"""
if self._creator_pid != os.getpid() or self._closed:
return
# Refuse acquires from here on, so none is admitted into a hold the backend is letting go of.
self._closed = True
try:
await self._owners.retire(
None if self._owns_backend else self._via,
leave=functools.partial(self._submit, self._lock.close if self._owns_backend else self._leave_backend),
still_held=self._backend_holds,
)
except BaseException as error:
# Only a failed backend call can leave the level held, and then a retried close() has to find it.
if not isinstance(error, asyncio.CancelledError) and self._backend_holds():
self._closed = False
raise
def _raise_if_unusable(self) -> None:
if self._creator_pid != os.getpid(): # pragma: forked child
msg = f"AsyncSoftReadWriteLock on {self.lock_file} was inherited across fork; construct a new instance"
raise RuntimeError(msg)
if self._closed:
msg = f"AsyncSoftReadWriteLock on {self.lock_file} has been closed"
raise RuntimeError(msg)
def _backend_holds(self) -> bool:
# A sync close() or forced release on a shared singleton ends the tasks' level without the table taking part.
return self._lock._holds_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
def _leave_backend(self) -> None:
# Only the tasks' level: a sync user of the same singleton keeps its own.
self._lock._release_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
async def _run_acquire(self, acquire: Callable[..., AcquireReturnProxy]) -> None:
# run_in_executor cannot recall work the pool already started. A sync acquire that won the lock before it saw
# the cancellation has published its claim and started the heartbeat, which keeps the record fresh so no peer
# evicts it. Wait the call out and hand the claim back, as AsyncReadWriteLock does.
abandoned = threading.Event()
acquire_future = self._submit(acquire, cancel_check=abandoned.is_set)
try:
await _wait_until_done(acquire_future)
except asyncio.CancelledError as cancellation:
# Stops a backend wait on another process's hold, which would otherwise outlast the caller by as long.
abandoned.set()
try:
await _drain_future(acquire_future)
except Timeout:
# Nothing was taken, so the caller's cancellation is the whole outcome.
raise cancellation from None
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
_raise_cancelled_error(cancellation, error)
try:
await _drain_future(self._submit(self._leave_backend))
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
_raise_cancelled_error(cancellation, error)
raise
_future_result(acquire_future)
def _submit(
self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs
) -> asyncio.Future[_BackendOutcome[_R]]:
loop = self._loop or asyncio.get_running_loop()
return loop.run_in_executor(self._executor, _capture_call, functools.partial(func, *args, **kwargs))
class AsyncAcquireSoftReadWriteReturnProxy:
"""Async context-aware object that releases an :class:`AsyncSoftReadWriteLock` on exit."""
def __init__(self, lock: AsyncSoftReadWriteLock) -> None:
self.lock = lock
async def __aenter__(self) -> AsyncSoftReadWriteLock:
return self.lock
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock.release()
__all__ = [
"AsyncAcquireSoftReadWriteReturnProxy",
"AsyncSoftReadWriteLock",
]
@@ -0,0 +1,638 @@
"""
The generation-log protocol behind :class:`~filelock.SoftReadWriteLock`.
The lock's whole state is a sequence of immutable snapshots, ``gen/<N>``, each naming the writer and the readers that
hold the lock at generation ``N``. Every transition is one atomic publication of the next snapshot: a participant reads
the latest generation, derives the successor it wants, and links a fully written temporary file to ``gen/<N+1>``. The
link fails if a peer got there first, so the participant re-reads and tries again. No step is ever half done: a crash
leaves at most an unlinked temporary file, never a state that another participant must repair before it can proceed.
Liveness rests on a nonce, not a clock. Each participant keeps ``holders/<token>`` and rewrites its nonce every
heartbeat. A contender records the bytes it read there and the moment it read them on its own monotonic clock; a
member whose bytes have not changed for ``stale_threshold`` is evicted by committing a snapshot without it. Two hosts
never compare clocks, so a skewed file server or a suspended virtual machine cannot make a live holder look dead.
The generation number is monotonic and unique to one transition within one incarnation of the log, so the generation at
which a hold was granted fences it: a resource that rejects anything carrying a lower generation than the highest it has
accepted refuses a holder that paused past its expiry and resumed. A random epoch file names the incarnation, so a
participant that outlives a removal of the log reads the new one instead of continuing the old numbering into it.
"""
from __future__ import annotations
__lazy_modules__ = ["contextlib", "filelock._error", "filelock._identity", "math", "pathlib", "secrets"]
import os
import secrets
from contextlib import suppress
from dataclasses import dataclass
from math import isfinite
from pathlib import Path
from typing import TYPE_CHECKING, Final, Literal, Protocol
from filelock._error import SoftFileLockProtocolError
from filelock._identity import host_name
if TYPE_CHECKING:
from collections.abc import Callable
Mode = Literal["read", "write"]
HeartbeatOutcome = Literal["ok", "lost", "transient"]
_PROTOCOL: Final[str] = "filelock-rw/1"
_GENERATION_DIGITS: Final[int] = 20
_TOKEN_HEX_LENGTH: Final[int] = 32
#: A snapshot line is ``reader=`` plus a token, so this bounds a hostile record without capping real reader counts.
_MAX_RECORD_SIZE: Final[int] = 1 << 20
#: Snapshots older than this many generations behind the latest are removed by whoever commits. The window is also how
#: far ``latest`` probes past a gap before it trusts that nothing newer exists; ``gen/HEAD``, not the window, is what
#: keeps a listing served further behind from passing an old generation off as the head.
_RETAINED_GENERATIONS: Final[int] = 64
_GENERATIONS_DIRECTORY: Final[str] = "gen"
#: A random token created once per log: removing ``gen/`` removes it, so a participant can tell a log that was wiped and
#: restarted from the one it remembers.
_EPOCH_NAME: Final[str] = "epoch"
#: The epoch plus a copy of the newest snapshot, renamed into place after each commit. NFS revalidates a file opened by
#: name but may serve a listing from cache, so this is the head a stale client cannot miss; it may lag, never lead.
_HEAD_NAME: Final[str] = "HEAD"
_HOLDERS_DIRECTORY: Final[str] = "holders"
_TEMPORARY_PREFIX: Final[str] = ".commit-"
class TreeReplacedError(RuntimeError):
"""
Raised by a ``Files`` implementation when a protocol directory is not a real directory (a symlink, a junction, a
file) or belongs to another account.
"""
class Files(Protocol):
"""The filesystem operations the protocol runs on; :mod:`._storage` implements them over :mod:`os`."""
def read(self, path: str) -> bytes | None:
"""The file's bytes, ``None`` when it does not exist, or ``b""`` when it is not a regular file."""
def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]:
"""
Create the file exclusively with the whole record written; raise ``FileExistsError`` when it exists.
The returned device and inode (the kernel's identity for a file, independent of its name) let ``overwrite``
check that the name still leads to this file.
*durable* flushes it to stable storage first, for a record that outlives the host that wrote it: a generation a
crash left empty would wedge every other participant. A holder record dies with its writer, so it skips that.
"""
def link(self, source: str, target: str) -> bool:
"""Hard-link *source* to *target* without replacing; ``True`` when *target* names *source*'s file afterwards."""
def replace(self, source: str, target: str) -> bool:
"""Rename *source* over *target* atomically; ``False`` when the platform refused because *target* is in use."""
def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool:
"""
Rewrite from offset zero the file ``create`` returned *identity* for.
``False`` when *path* no longer leads to that file.
"""
def unlink(self, path: str) -> None:
"""Remove the file, tolerating one that is already gone."""
def listdir(self, path: str) -> list[str]:
"""The names in the directory, or an empty list when it does not exist."""
def prepare(self, root: str, *, force: bool = False) -> None:
"""
Create the protocol directories under *root*, refusing anything at those paths that is not a directory.
Done once per lock; *force* repeats it after a write found a directory missing.
"""
@dataclass(frozen=True)
class Snapshot:
"""The lock's state at one generation: which token writes, which tokens read."""
generation: int
writer: str | None
readers: frozenset[str]
@property
def members(self) -> frozenset[str]:
"""Every token this generation names."""
return self.readers if self.writer is None else self.readers | {self.writer}
def encode(self) -> bytes:
lines = [_PROTOCOL, f"generation={self.generation}"]
if self.writer is not None:
lines.append(f"writer={self.writer}")
lines.extend(f"reader={token}" for token in sorted(self.readers))
return "".join(f"{line}\n" for line in lines).encode("ascii")
def parse_snapshot(data: bytes) -> Snapshot | None:
"""The snapshot a ``gen/<N>`` record encodes, or ``None`` when the record is malformed."""
if not data or len(data) > _MAX_RECORD_SIZE:
return None
try:
lines = data.decode("ascii").split("\n")
except UnicodeDecodeError:
return None
if lines[0] != _PROTOCOL or lines[-1]:
return None
generation: int | None = None
writer: str | None = None
readers: set[str] = set()
for line in lines[1:-1]:
key, _, value = line.partition("=")
if (
key == "generation"
and generation is None
and len(value) <= _GENERATION_DIGITS
and value.isdigit()
and str(int(value)) == value
):
generation = int(value)
elif key == "writer" and writer is None and is_token(value):
writer = value
elif key == "reader" and is_token(value) and value not in readers:
readers.add(value)
else:
return None
return None if generation is None else Snapshot(generation=generation, writer=writer, readers=frozenset(readers))
def is_token(value: str) -> bool:
"""Whether *value* is a claim token: exactly 32 lowercase hex digits."""
return len(value) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in value)
def new_token() -> str:
"""A fresh claim token."""
return secrets.token_hex(_TOKEN_HEX_LENGTH // 2)
def encode_holder(token: str, nonce: str, lease: float) -> bytes:
"""
The ``holders/<token>`` record; only the nonce changes between heartbeats, so the length stays constant.
*lease* is the writer's own ``stale_threshold``: a peer configured tighter must not read this holder's longer
heartbeat interval as death, so evictors wait the longer of their threshold and this one.
"""
return (
f"{_PROTOCOL}\ntoken={token}\npid={os.getpid()}\nhost={host_name()}\nlease={float(lease)!r}\nnonce={nonce}\n"
).encode("ascii")
def holder_lease(record: bytes | None) -> float | None:
"""The ``lease`` a holder record carries, or ``None`` when it has none (an older release) or it is malformed."""
try:
text = next(
line for line in (record or b"").decode("ascii").split("\n") if line.startswith("lease=")
).removeprefix("lease=")
lease = float(text)
except (StopIteration, ValueError): # UnicodeDecodeError is a ValueError
return None
# Only the exact spelling repr() writes: float() alone also takes "1_000", " 1", and non-ASCII digits.
return lease if isfinite(lease) and lease > 0 and repr(lease) == text else None
class Ledger:
"""
How long each observed value has stayed the same, measured on this observer's monotonic clock.
A value that has not changed for ``stale_threshold`` belongs to a participant that stopped refreshing it. The clock
is the observer's own and only ever measures the gap between two of its own observations, which is what lets two
hosts agree on liveness without agreeing on the time.
"""
def __init__(self, clock: Callable[[], float]) -> None:
self._clock = clock
self._seen: dict[str, tuple[bytes | None, float]] = {}
def observe(self, key: str, value: bytes | None, *, asked: float | None = None) -> float:
"""
Record *value* for *key* and return for how long it has stayed the same.
*asked* is the clock reading taken before the caller fetched *value*. An unchanged value counts as unchanged up
to *asked*, so a pause between the fetch and this call adds no time.
"""
now = self._clock()
if (seen := self._seen.get(key)) is None or seen[0] != value:
self._seen[key] = (value, now)
return 0.0
return (now if asked is None else asked) - seen[1]
def forget(self, key: str) -> None:
self._seen.pop(key, None)
def retain(self, keys: set[str]) -> None:
"""Drop every key not in *keys*: peers that left would otherwise stay in the ledger for the lock's lifetime."""
self._seen = {key: seen for key, seen in self._seen.items() if key in keys}
class GenerationLog:
"""The ordered snapshots under ``gen/``, found by listing and probing, advanced by linking the next one."""
def __init__(self, files: Files, lock_file: str, root: str) -> None:
self._files = files
self._lock_file = lock_file
self._directory = Path(root, _GENERATIONS_DIRECTORY)
self._epoch_path = str(self._directory / _EPOCH_NAME)
self._head_path = str(self._directory / _HEAD_NAME)
self._epoch: bytes | None = None
self._latest: Snapshot | None = None
def latest(self) -> Snapshot:
"""
The newest snapshot.
Starts from the newest readable generation among those ``gen/HEAD``, the listing, or memory name, then probes
forward by name across the retained window. ``HEAD`` only names a generation, and that generation must still
read back: a committer stalled before its rename can leave ``HEAD`` on a generation compacted since. A
generation compacted between the listing and the read, and a hole left by a committer that died before it
compacted, are covered by the probe. Memory and ``HEAD`` are bound to the log's epoch, so a log wiped and
restarted is never continued from a generation it no longer holds. Only a log that names generations, now or as
remembered, of which none can be read is refused, because that is a client so far behind the log that nothing
it reads can be trusted.
"""
# When everything the first pass named is gone, peers compacted past it while this client looked, or the log was
# replaced; one more pass settles whether this client cannot see the head at all.
if (latest := self._find()) is None and (latest := self._find()) is None:
raise SoftFileLockProtocolError(self._lock_file, None, "generation log names no readable snapshot")
self._latest = latest
return latest
def _find(self) -> Snapshot | None:
if (epoch := self._incarnation()) != self._epoch:
self._epoch, self._latest = epoch, None
named = set(self._list())
# Generation 0 is the empty log, which has no record to read back.
if self._latest is not None and self._latest.generation:
named.add(self._latest.generation)
if head := self._head_generation():
named.add(head)
for generation in sorted(named, reverse=True):
if (start := self._read(generation)) is not None:
# Compaction keeps the surviving generations contiguous, so the first missing name past a readable HEAD
# generation is the end; probing the whole window on every poll cost 64 failed opens.
return self._probe(start, patience=1 if generation == head else _RETAINED_GENERATIONS)
newest = max(named, default=0)
found = self._probe(Snapshot(generation=newest, writer=None, readers=frozenset()))
return found if found.generation > newest or not named else None
def _head_generation(self) -> int | None:
# HEAD is never flushed, so a crash can leave it unreadable; readers verify what it names anyway, so a torn one
# only sends them to the window probe.
if (record := self._head_record()) is None or (snapshot := parse_snapshot(record)) is None:
return None
return snapshot.generation
def _head_record(self) -> bytes | None:
if (data := self._files.read(self._head_path)) is None:
return None
# A HEAD from another epoch landed after its log was removed; a log whose epoch was never written runs without.
epoch, newline, record = data.partition(b"\n")
return record if self._valid_epoch() == epoch + newline else None
def _valid_epoch(self) -> bytes | None:
epoch = self._epoch
return epoch if epoch is not None and is_token(epoch.decode("ascii", "replace").removesuffix("\n")) else None
def _incarnation(self) -> bytes | None:
if (epoch := self._files.read(self._epoch_path)) is None:
# Exclusive creation, not a link: reading the log must keep working where os.link does not exist. A peer
# that reads the file before its token is written takes the empty value for the epoch, and its next commit
# sees the change and re-reads, so a partial read costs one retry.
try:
self._files.create(self._epoch_path, f"{new_token()}\n".encode("ascii"), durable=True)
except FileExistsError:
pass
except FileNotFoundError:
# No gen/ yet: there is no log to tell apart, and the first participant's prepare() creates it.
return None
epoch = self._files.read(self._epoch_path)
return epoch
def _list(self) -> list[int]:
listed = self._listed()
for generation in listed[:-_RETAINED_GENERATIONS]:
self._files.unlink(self._path(generation))
return listed[-_RETAINED_GENERATIONS:]
def _listed(self) -> list[int]:
# isdigit() alone admits Unicode digits: superscripts int() rejects, fullwidth ones it parses to another number.
return sorted(
int(name)
for name in self._files.listdir(str(self._directory))
if len(name) == _GENERATION_DIGITS and name.isascii() and name.isdigit()
)
def _probe(self, start: Snapshot, patience: int = _RETAINED_GENERATIONS) -> Snapshot:
latest = start
generation = start.generation
gap = 0
while gap < patience:
generation += 1
if (snapshot := self._read(generation)) is None:
gap += 1
continue
latest = snapshot
gap = 0
return latest
def _read(self, generation: int) -> Snapshot | None:
if (data := self._files.read(self._path(generation))) is None:
return None
if (snapshot := parse_snapshot(data)) is None or snapshot.generation != generation:
name = self._name(generation)
raise SoftFileLockProtocolError(self._lock_file, name, "malformed generation record")
return snapshot
def commit(self, successor: Snapshot) -> bool:
"""
Publish *successor* as the next generation; ``False`` when a peer published that generation first.
The record is complete before it gets its public name, and a no-replace hard link is the compare-and-swap: only
one link to ``gen/<N+1>`` can ever succeed. The link's return code is not trusted, because an NFS retry can
report a link that did land as failed; the file's identity after the call is what decides. A log whose epoch
changed since the snapshot was read is a different log, so the commit is refused as lost and the caller
re-reads; that narrows a wipe racing this commit to the moment between the epoch check and the link. A slot
compaction freed is refused the same way, before the link and again after it, since a committer can stall in
between. After the link a surviving predecessor proves little: it may be a record another stalled committer
linked into its own freed slot and has yet to remove. A slot comes free once the log is a whole window past it,
and the head stays, so a record the listing or ``HEAD`` leads by a window is refused as well. A record refused
after it landed is removed when nothing was built on it. One that peers lead by less than a window stays, or its
slot would come free early.
"""
if self._files.read(self._epoch_path) != self._epoch or not self._extends_the_log(successor.generation):
return False
if not self._publish(
self._path(successor.generation), data := successor.encode(), self._files.link, durable=True
):
return False
lead = max([*self._listed(), self._head_generation() or 0]) - successor.generation
if lead >= _RETAINED_GENERATIONS or not self._extends_the_log(successor.generation):
if not 0 < lead < _RETAINED_GENERATIONS:
self._files.unlink(self._path(successor.generation))
return False
self._latest = successor
# Readers verify the generation HEAD names, so a failed rename costs a probe, not the commit. A late one would
# move HEAD back past later commits, so HEAD only ever moves forward.
with suppress(OSError):
if (
(epoch := self._valid_epoch()) is not None
and self._files.read(self._epoch_path) == epoch
and (self._head_generation() or 0) < successor.generation
):
self._publish(self._head_path, epoch + data, self._files.replace, durable=False)
if (expired := successor.generation - _RETAINED_GENERATIONS - 1) > 0:
self._compact_through(expired)
return True
def _extends_the_log(self, generation: int) -> bool:
# Compaction removes oldest first, so a missing predecessor means compaction freed this slot. The first
# generation has none; a HEAD of this epoch means the log already moved past it.
if generation > 1:
return self._files.read(self._path(generation - 1)) is not None
return self._head_record() is None
def _publish(self, target: str, data: bytes, place: Callable[[str, str], bool], *, durable: bool) -> bool:
temporary = str(self._directory / f"{_TEMPORARY_PREFIX}{new_token()}")
try:
self._files.create(temporary, data, durable=durable)
except FileNotFoundError:
# No gen/: the log was removed, or nobody has prepared it yet. Either way there is nothing to publish into
# until a participant's prepare() creates it, and the caller re-reads whatever log exists by then.
return False
try:
return place(temporary, target)
finally:
self._files.unlink(temporary)
def _compact_through(self, expired: int) -> None:
# Oldest first, so survivors stay contiguous: a stalled committer's leftover would otherwise sit below a hole.
oldest = expired
while oldest > 1 and self._files.read(self._path(oldest - 1)) is not None:
oldest -= 1
for generation in range(oldest, expired + 1):
self._files.unlink(self._path(generation))
def _path(self, generation: int) -> str:
return str(self._directory / self._name(generation))
@staticmethod
def _name(generation: int) -> str:
return f"{generation:0{_GENERATION_DIGITS}d}"
class Participant:
"""
One acquisition, from publishing a claim through holding the lock to leaving.
``advance`` is polled until it reports the lock granted, ``heartbeat`` runs on the holder's schedule, and ``leave``
commits the participant out. None of them sleep: the caller owns the deadline and the poll interval. The ledger and
the log outlive one acquisition: how long a peer's record has stayed unchanged is knowledge every attempt against
the same lock shares, or a caller whose attempts are each shorter than the stale threshold could never evict.
"""
def __init__( # ruff:ignore[too-many-arguments] # one value per protocol input; the lock builds this once per acquisition
self,
files: Files,
root: str,
mode: Mode,
*,
stale_threshold: float,
clock: Callable[[], float],
ledger: Ledger,
log: GenerationLog,
) -> None:
self.token: Final[str] = new_token()
self.mode: Final[Mode] = mode
self.generation: int | None = None
self._files = files
self._root = root
self._stale_threshold = stale_threshold
self._clock = clock
self._ledger = ledger
self._log = log
self._holder = self._holder_path(self.token)
self._holder_identity: tuple[int, int] | None = None
self._entered = False
self._next_sweep = clock()
def publish(self) -> None:
"""Create the holder record peers watch for liveness; it exists before the token appears in any snapshot."""
self._files.prepare(self._root)
self._create_holder()
def _create_holder(self) -> None:
try:
self._holder_identity = self._files.create(self._holder, self._record())
except FileNotFoundError:
# Removing the whole log takes the directories with it.
self._files.prepare(self._root, force=True)
self._holder_identity = self._files.create(self._holder, self._record())
def _record(self) -> bytes:
return encode_holder(self.token, new_token(), self._stale_threshold)
def advance(self) -> bool:
"""
Take the next step toward holding the lock; ``True`` once the lock is granted.
A reader enters as soon as no live writer is named. A writer enters as soon as no live writer is named, which
blocks every new reader, then waits for the named readers to leave. Stale members are evicted in the same commit
that admits this participant, and a writer's final admission is itself a commit, so no admission can rest on a
snapshot a peer is about to supersede.
"""
# A lost race is not a reason to wait: re-read and try again before handing the poll interval back to the
# caller.
for _ in range(4):
latest = self._log.latest()
self._keep_claim_fresh(latest)
self._sweep(latest)
stale = frozenset(token for token in latest.members - {self.token} if self._is_stale(token))
writer = None if latest.writer in stale else latest.writer
readers = latest.readers - stale
if not self._entered:
if writer is not None:
# A live writer blocks entry in both modes; only an eviction is worth committing.
if stale:
self._commit(latest, writer, readers, stale)
return False
writer, readers = (self.token, readers) if self.mode == "write" else (None, readers | {self.token})
granted = self.mode == "read" or not readers
if self._entered and not stale and not granted:
return False
if (generation := self._commit(latest, writer, readers, stale)) is None:
continue
self._entered = True
if granted:
self.generation = generation
return granted
return False
def _keep_claim_fresh(self, latest: Snapshot) -> None:
# The snapshot is the truth about membership: a commit the filesystem reported lost may have landed, and a peer
# may have evicted this participant while it waited to drain, so both directions are taken from it. A wait can
# outlast the stale threshold, so the nonce changes on every poll; a peer then never reads a patient contender
# as a corpse. A record that has gone missing was taken by an evictor or a sweeper that did, or by a removal of
# the whole log.
self._entered = self.token in latest.members
if not self._refresh_nonce():
with suppress(FileExistsError):
self._create_holder()
def _sweep(self, latest: Snapshot) -> None:
# A participant that crashed before its first commit leaves a record no snapshot names, and a commit that
# crashed between creating and unlinking its temporary file leaves that too. Neither blocks anyone, so they are
# collected on the same unchanged-for-a-threshold rule as a member, and only every half threshold at most.
if (now := self._clock()) < self._next_sweep:
return
self._next_sweep = now + self._stale_threshold / 2
watched = set(latest.members)
holders = Path(self._root, _HOLDERS_DIRECTORY)
for name in self._files.listdir(str(holders)):
if name != self.token and name not in latest.members and is_token(name):
watched.add(key := f"holder:{name}")
self._collect(key, str(holders / name))
generations = Path(self._root, _GENERATIONS_DIRECTORY)
for name in self._files.listdir(str(generations)):
if name.startswith(_TEMPORARY_PREFIX):
watched.add(key := f"temporary:{name}")
self._collect(key, str(generations / name))
self._ledger.retain(watched)
def _collect(self, key: str, path: str) -> None:
if self._unchanged_for_its_lease(key, path):
self._files.unlink(path)
self._ledger.forget(key)
def _commit(
self, latest: Snapshot, writer: str | None, readers: frozenset[str], stale: frozenset[str]
) -> int | None:
successor = Snapshot(generation=latest.generation + 1, writer=writer, readers=readers)
if not self._log.commit(successor):
return None
for token in stale:
self._files.unlink(self._holder_path(token))
self._ledger.forget(token)
return successor.generation
def _is_stale(self, token: str) -> bool:
return self._unchanged_for_its_lease(token, self._holder_path(token))
def _unchanged_for_its_lease(self, key: str, path: str) -> bool:
asked = self._clock()
record = self._files.read(path)
return self._ledger.observe(key, record, asked=asked) >= max(self._stale_threshold, holder_lease(record) or 0.0)
def _holder_path(self, token: str) -> str:
return str(Path(self._root, _HOLDERS_DIRECTORY, token))
def _refresh_nonce(self) -> bool:
if (identity := self._holder_identity) is None:
return False
return self._files.overwrite(self._holder, self._record(), identity)
def heartbeat(self) -> tuple[HeartbeatOutcome, OSError | None]:
"""
Refresh the nonce and confirm the latest snapshot still names this participant.
``lost`` means a peer evicted this participant or removed its holder record: the lock is no longer held, and the
holder must stop using what it protects. ``transient`` is a filesystem error refreshing the nonce, worth
retrying on the next tick. A failure to read the log or to sweep decides nothing about this holder's liveness,
so it is not counted against it.
"""
try:
if not self._refresh_nonce():
return "lost", None
except TreeReplacedError:
return "lost", None
except OSError as error:
return "transient", error
try:
latest = self._log.latest()
self._sweep(latest)
except (OSError, TreeReplacedError):
return "ok", None
return ("ok" if self.token in latest.members else "lost"), None
def leave(self) -> None:
"""Commit this participant out of the latest snapshot, then remove its holder record."""
# The record goes even when the log cannot be read: a contender that never entered then leaves nothing behind,
# and a member still named ages out the same way with or without one.
# A tree that was swapped out leaves no record this participant can reach, so there is nothing to commit out of.
with suppress(TreeReplacedError):
try:
while self.token in (latest := self._log.latest()).members:
successor = Snapshot(
generation=latest.generation + 1,
writer=None if latest.writer == self.token else latest.writer,
readers=latest.readers - {self.token},
)
if self._log.commit(successor):
break
finally:
self._entered = False
self.generation = None
self._files.unlink(self._holder)
__all__ = [
"Files",
"GenerationLog",
"HeartbeatOutcome",
"Ledger",
"Mode",
"Participant",
"Snapshot",
"TreeReplacedError",
"encode_holder",
"holder_lease",
"is_token",
"new_token",
"parse_snapshot",
]
@@ -0,0 +1,278 @@
"""The :class:`~._protocol.Files` implementation over :mod:`os`."""
from __future__ import annotations
__lazy_modules__ = ["filelock._soft_rw._protocol", "filelock._strict", "filelock._util", "pathlib"]
import errno
import os
import stat
import sys
import time
from contextlib import contextmanager, suppress
from errno import ELOOP, ENOENT, ENOTDIR, ENXIO, ESTALE
from pathlib import Path
from typing import TYPE_CHECKING, Final
from filelock._soft_rw._protocol import (
_GENERATIONS_DIRECTORY,
_HOLDERS_DIRECTORY,
_MAX_RECORD_SIZE,
TreeReplacedError,
)
from filelock._strict import _link_no_follow, _pin_directories, _raise_if_hard_links_unsupported
from filelock._util import ensure_directory_exists, foreign_directory_owner, write_all
if TYPE_CHECKING:
from collections.abc import Generator
_O_NOFOLLOW: Final[int] = getattr(os, "O_NOFOLLOW", 0)
#: Probed once at import, as :mod:`filelock._strict` does, because a per-call check flips when a test mocks the
#: function. The set lists os.rename, whose implementation os.replace shares.
_SUPPORTS_DIR_FD: Final[bool] = (
hasattr(os, "link")
and {os.link, os.open, os.stat, os.mkdir, os.rename, os.unlink} <= os.supports_dir_fd
and os.scandir in os.supports_fd
)
#: Windows opens descriptors in text mode by default, which rewrites newlines; every record here is exact bytes.
_O_BINARY: Final[int] = getattr(os, "O_BINARY", 0)
#: ESTALE is what an NFS client reports for a file a peer unlinked out from under its cached handle: gone, not broken.
_MISSING_ERRNOS: Final[frozenset[int]] = frozenset({ENOENT, ESTALE})
#: Windows refuses an open with EACCES while another process is deleting the file or holds it open without sharing;
#: both clear within moments, so an open is retried this long before the refusal is taken as real.
_WINDOWS_OPEN_GRACE: Final[float] = 0.5
class OsFiles:
"""Record files under a lock's protocol root, opened without following symlinks and owned by this UID alone."""
def __init__(self, lock_file: str) -> None:
self._lock_file = lock_file
self._prepared = False
self._identities: dict[str, tuple[int, int]] = {}
def read(self, path: str) -> bytes | None:
# O_NONBLOCK keeps an open of a FIFO planted at the path from blocking; the regular-file check below rejects it.
if (fd := self._open(path, os.O_RDONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY)) is None:
return None
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
return b""
chunks: list[bytes] = []
remaining = _MAX_RECORD_SIZE + 1
while remaining > 0 and (chunk := os.read(fd, remaining)):
chunks.append(chunk)
remaining -= len(chunk)
return b"".join(chunks)
finally:
os.close(fd)
def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]:
with self._relative(path) as (name, dir_fd):
fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_NOFOLLOW | _O_BINARY, 0o600, dir_fd=dir_fd)
try:
write_all(fd, data)
if durable:
os.fsync(fd)
created = os.fstat(fd)
except BaseException:
os.close(fd)
# The name is unique to this call, so removing it cannot take a record from anyone else.
self.unlink(path)
raise
os.close(fd)
return created.st_dev, created.st_ino
def link(self, source: str, target: str) -> bool:
failure: NotImplementedError | OSError | None = None
with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd):
try:
_link_no_follow(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd)
except FileExistsError:
pass
except (NotImplementedError, OSError) as error:
_raise_if_hard_links_unsupported(self._lock_file, error)
failure = error
# The file's identity decides, not the call's result: an NFS retransmit can report a link that landed as
# failed, and an EEXIST can be this very link answered twice. Any other error on a link that did not land is
# a fault, not a peer winning the race, and a retry would spin.
if (identity := _identity(source_name, source_fd)) is not None and identity == _identity(
target_name, target_fd
):
return True
if failure is not None:
raise failure
return False
def replace(self, source: str, target: str) -> bool:
try:
with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd):
os.replace(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd)
except PermissionError:
# Windows refuses to rename over a file a reader holds open; callers tolerate a target one rename behind.
return False
return True
def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool:
# O_NONBLOCK keeps an open of a FIFO (a named pipe) at the path from blocking, as in read. With no reader the
# open fails with ENXIO.
try:
fd = self._open(path, os.O_WRONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY)
except OSError as error:
if error.errno != ENXIO:
raise
return False # pragma: needs fifo
if fd is None:
return False
try:
# O_NOFOLLOW refuses a symlink but not a hard link (a second name for the same file) or another file renamed
# over the name, and a write through either lands in a file that is not the record. The protocol does not
# link holder records, so a second name marks a file that inherited a removed record's inode number.
opened = os.fstat(fd)
if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != identity or opened.st_nlink != 1:
return False
write_all(fd, data)
finally:
os.close(fd)
return True
def unlink(self, path: str) -> None:
try:
with self._relative(path) as (name, dir_fd):
os.unlink(name, dir_fd=dir_fd)
except OSError as error:
# Windows refuses to remove a file another process holds open; the record then outlives its owner until
# that process exits, which only delays the cleanup a later participant repeats.
if error.errno not in _MISSING_ERRNOS and not isinstance(error, PermissionError):
raise
def listdir(self, path: str) -> list[str]:
try:
with self._directory(path) as dir_fd, os.scandir(path if dir_fd is None else dir_fd) as entries:
return [entry.name for entry in entries]
except FileNotFoundError:
return []
def prepare(self, root: str, *, force: bool = False) -> None:
if self._prepared and not force:
return
if force:
self._identities.clear()
ensure_directory_exists(root)
for directory, depth in (
(Path(root), 0),
(Path(root, _GENERATIONS_DIRECTORY), 1),
(Path(root, _HOLDERS_DIRECTORY), 1),
):
with self._relative(str(directory), depth) as (name, dir_fd):
with suppress(FileExistsError):
os.mkdir(name, 0o700, dir_fd=dir_fd)
# mkdir has no O_NOFOLLOW, so check what the name resolves to before the lock creates anything in it.
found = os.stat(name, dir_fd=dir_fd, follow_symlinks=False)
parent = directory.parent.stat() if dir_fd is None else os.fstat(dir_fd)
if (
not stat.S_ISDIR(found.st_mode)
or getattr(found, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT
):
msg = f"{directory} exists but is not a directory or is a symlink; refusing to use it"
raise TreeReplacedError(msg)
owner = foreign_directory_owner(found, parent)
if owner is not None: # pragma: needs foreign-sticky-tmp
msg = f"{directory} belongs to uid {owner}, not to this process's account or the parent's owner"
raise TreeReplacedError(msg)
self._identities[str(directory)] = found.st_dev, found.st_ino
self._prepared = True
def _open(self, path: str, flags: int) -> int | None:
# Missing is the one answer a caller acts on without raising. A refused open is real on POSIX, where nothing
# this lock does makes its own records unreadable; on Windows it is usually another process mid-delete or
# holding the file without sharing, so it is retried through a short grace before it counts, and a file still
# refusing after that has been in delete-pending for longer than any deletion takes, so it is on its way out:
# missing.
deadline: float | None = None
while True:
if isinstance(opened := self._attempt_open(path, flags), int):
return opened
if opened.errno in _MISSING_ERRNOS:
return None
if not isinstance(opened, PermissionError) or sys.platform != "win32":
raise opened
if deadline is None: # pragma: win32 cover
deadline = time.monotonic() + _WINDOWS_OPEN_GRACE
elif time.monotonic() >= deadline: # pragma: win32 cover
return None
time.sleep(0.002) # pragma: win32 cover
def _attempt_open(self, path: str, flags: int) -> int | OSError:
try:
with self._relative(path) as (name, dir_fd):
return os.open(name, flags, dir_fd=dir_fd)
except OSError as error:
return error
@contextmanager
def _relative(self, path: str, depth: int = 2) -> Generator[tuple[str, int | None]]:
directory, name = os.path.split(path)
with self._directory(directory, depth) as dir_fd:
yield (path if dir_fd is None else name), dir_fd
@contextmanager
def _directory(self, path: str, depth: int = 2) -> Generator[int | None]:
# O_NOFOLLOW applies to the last path component. The check prepare() made on <root> and <root>/gen holds until a
# process that can write the lock's directory renames the tree aside and puts one with a symlinked gen in its
# place. Each call therefore opens the root and then the record's directory without following symlinks and
# works inside what it opened. No descriptor outlives the call, so a later call finds a log removed and started
# again by name, as before. None means descriptors cannot anchor a call and the call uses the whole path.
if depth and _SUPPORTS_DIR_FD: # pragma: needs link-dir-fd
with self._relative(path, depth - 1) as (name, parent_fd):
try:
fd = os.open(name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | _O_NOFOLLOW, dir_fd=parent_fd)
except OSError as error:
# NetBSD reports EFTYPE rather than ELOOP when O_NOFOLLOW refuses a symlink.
if error.errno not in {ELOOP, ENOTDIR, getattr(errno, "EFTYPE", ELOOP)}:
raise
msg = f"{path} is not a directory or is a symlink; refusing to use it"
raise TreeReplacedError(msg) from error
try:
self._admit(fd, path, parent_fd)
except BaseException:
os.close(fd)
raise
try:
yield fd
finally:
os.close(fd)
else:
# Without dir_fd, hold each of those directories open instead, so no process can rename or replace one.
try:
pins = _pin_directories(*[Path(path), *Path(path).parents][:depth][::-1])
except NotADirectoryError as error: # pragma: win32 cover
msg = f"{error.filename} is not a directory or is a symlink; refusing to use it"
raise TreeReplacedError(msg) from error
with pins:
yield None
def _admit(self, fd: int, path: str, parent_fd: int | None) -> None: # pragma: needs link-dir-fd
# A directory other than the one prepare() checked may belong to another account: the tree can be renamed aside
# and recreated while this object lives, and the ownership check has to hold for what is opened now.
found = os.fstat(fd)
if (found.st_dev, found.st_ino) == self._identities.get(path):
return
parent = os.fstat(parent_fd) if parent_fd is not None else Path(path).parent.stat()
if (owner := foreign_directory_owner(found, parent)) is not None: # pragma: needs foreign-sticky-tmp
msg = f"{path} belongs to uid {owner}, not to this process's account or the parent's owner"
raise TreeReplacedError(msg)
self._identities[path] = found.st_dev, found.st_ino
def _identity(path: str, dir_fd: int | None) -> tuple[int, int] | None:
try:
st = os.stat(path, dir_fd=dir_fd, follow_symlinks=False)
except OSError:
return None
return st.st_dev, st.st_ino
__all__ = [
"OsFiles",
]
@@ -0,0 +1,793 @@
"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots."""
from __future__ import annotations
__lazy_modules__ = [
"filelock._error",
"filelock._lease",
"filelock._soft_rw._protocol",
"filelock._soft_rw._storage",
"math",
]
import atexit
import os
import threading
import time
from contextlib import contextmanager, suppress
from dataclasses import dataclass
from math import isfinite
from typing import TYPE_CHECKING, ClassVar, Final
from weakref import WeakValueDictionary
from filelock._api import (
AcquireReturnProxy,
_acquire_thread_lock,
_canonical,
_check_timeout_max,
_ensure_current_process,
_register_fork_class,
_register_fork_object,
_resolve_read_write_timeout,
_seconds,
_timeout_behavior,
)
from filelock._error import Timeout
from filelock._lease import LeaseCompromise
from filelock._soft_rw._protocol import GenerationLog, Ledger, Mode, Participant
from filelock._soft_rw._storage import OsFiles
if TYPE_CHECKING:
from collections.abc import Callable, Generator
from filelock._lease import CompromiseReason
_ALL_INSTANCES: Final[WeakValueDictionary[int, SoftReadWriteLock]] = WeakValueDictionary()
_ALL_INSTANCES_LOCK: threading.Lock = threading.Lock()
class _SoftRWMeta(type):
_instances: WeakValueDictionary[str, SoftReadWriteLock]
_instances_lock: threading.RLock
_instances_under_construction: set[str]
def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True,
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
on_compromise: Callable[[LeaseCompromise], None] | None = None,
) -> SoftReadWriteLock:
_ensure_current_process()
timeout = _seconds("timeout", timeout)
# Passed through only when set, so a subclass that declares its own constructor without it keeps working.
extra = {} if on_compromise is None else {"on_compromise": on_compromise}
if not is_singleton:
return super().__call__(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
**extra,
)
# Keyed like the log root, so one instance never spans two logs.
normalized = _canonical(lock_file)
with cls._instances_lock:
instance = cls._instances.get(normalized)
if instance is None:
if normalized in cls._instances_under_construction:
msg = f"Singleton lock construction is already active for {lock_file!s}"
raise RuntimeError(msg)
construction_pid = os.getpid()
construction_registry = cls._instances_under_construction
construction_registry.add(normalized)
try:
instance = super().__call__(
lock_file,
timeout,
blocking=blocking,
is_singleton=is_singleton,
heartbeat_interval=heartbeat_interval,
stale_threshold=stale_threshold,
poll_interval=poll_interval,
**extra,
)
finally:
construction_registry.discard(normalized)
if os.getpid() != construction_pid: # pragma: needs fork
msg = "Lock construction cannot continue after fork; construct a new lock in the child"
raise RuntimeError(msg)
cls._instances[normalized] = instance
elif (
_timeout_behavior(instance.timeout, any_negative=False)
!= _timeout_behavior(timeout, any_negative=False)
or instance.blocking != blocking
):
msg = (
f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking},"
f" cannot be changed to timeout={timeout}, blocking={blocking}"
)
raise ValueError(msg)
elif on_compromise is not None and instance._on_compromise != on_compromise: # ruff: ignore[private-member-access] # the metaclass owns the singleton it compares
# A tuning difference keeps the first caller's values; a different loss callback is a different safety
# contract, and returning the cached instance would silently drop it.
msg = f"Singleton lock created with a different on_compromise callback for {lock_file!s}"
raise ValueError(msg)
else:
_validate_intervals(heartbeat_interval, stale_threshold, poll_interval)
return instance
class SoftReadWriteLock(metaclass=_SoftRWMeta):
"""
Cross-process and cross-host reader/writer lock for shared filesystems.
Use this class instead of :class:`~filelock.ReadWriteLock` when the lock file lives on a network filesystem (NFS,
Lustre, HPC cluster shared storage). ``ReadWriteLock`` is backed by SQLite and cannot run on NFS because SQLite's
``fcntl`` locking is unreliable there.
The lock's state is a log of immutable snapshots under ``foo.lock.rw/gen/<N>``, each naming the writer and the
readers holding the lock at generation ``N``. Every acquire, release, and eviction publishes the next snapshot with
one atomic no-replace hard link, so no transition can be left half done by a crash on any host. Each participant
also keeps ``foo.lock.rw/holders/<token>``, a record whose nonce a daemon heartbeat thread rewrites every
``heartbeat_interval`` seconds. A contender evicts a member whose record has not changed for ``stale_threshold``
seconds of the contender's own monotonic clock; no clock is ever compared across hosts, so skew between hosts or
against the file server cannot make a live holder look dead.
Writer acquire is writer-preferring: a writer enters the snapshot as soon as no live writer is named, which blocks
any new reader, then waits for the named readers to leave. Writer starvation is impossible.
An expired holder is not fenced by the lock: a process paused past ``stale_threshold`` resumes believing it holds
the lock until its next heartbeat reports the loss through ``on_compromise``. :attr:`generation` is a monotonic
fencing token for the protected resource: a store that rejects writes carrying a lower generation than the highest
it has accepted refuses such a holder.
Reentrancy, upgrade/downgrade rules, thread pinning, and singleton caching by canonical path match
:class:`~filelock.ReadWriteLock`.
Forking invalidates the inherited instance in the child so the child cannot double-own the lock with its parent;
``release()`` on that instance is a no-op, and the child must construct a new instance if it needs a lock.
Trust boundary: protects against same-UID non-cooperating processes (one host or cross-host) and same-host
different-UID users via ``0o600`` / ``0o700`` permissions. Does not protect against root compromise or multi-tenant
mounts where hostile co-tenants share the UID.
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
:param is_singleton: if ``True``, reuse existing instances for the same canonical path
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
:param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to
``3 * heartbeat_interval``, matching etcd's ``LeaseKeepAlive`` convention
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
:param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is
lost: a peer evicted it, or refreshes failed for long enough that a peer could have
.. versionadded:: 3.27.0
.. versionchanged:: 3.33.0
The on-disk protocol is a generation log; earlier releases cannot share a lock path with this one.
"""
_instances: WeakValueDictionary[str, SoftReadWriteLock] = WeakValueDictionary()
_instances_lock = threading.RLock()
_instances_under_construction: ClassVar[set[str]] = set()
def __init_subclass__(cls, **kwargs: object) -> None:
"""Give each lock subclass its own singleton registry and lock."""
super().__init_subclass__(**kwargs)
cls._instances = WeakValueDictionary()
cls._instances_lock = threading.RLock()
cls._instances_under_construction = set()
_register_fork_class(cls)
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
is_singleton: bool = True, # ruff:ignore[unused-method-argument] # consumed by _SoftRWMeta.__call__
heartbeat_interval: float = 30.0,
stale_threshold: float | None = None,
poll_interval: float = 0.25,
on_compromise: Callable[[LeaseCompromise], None] | None = None,
) -> None:
self._creator_pid = os.getpid()
heartbeat_interval, stale_threshold, poll_interval = _validate_intervals(
heartbeat_interval, stale_threshold, poll_interval
)
self.lock_file: str = os.fspath(lock_file)
self.timeout: float = timeout
self.blocking: bool = blocking
self.heartbeat_interval: float = heartbeat_interval
self.stale_threshold: float = stale_threshold
self.poll_interval: float = poll_interval
# Resolved once: a relative lock path must keep naming the same log after the process changes directory. The
# final component stays unresolved, as in 4.0.12, so every release shares one log on a symlinked lock path.
self._root = f"{_canonical(self.lock_file)}.rw"
self._files = OsFiles(self.lock_file)
self._ledger = Ledger(time.monotonic)
self._log = GenerationLog(self._files, self.lock_file, self._root)
self._on_compromise = on_compromise
self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock())
self._hold: _Hold | None = None
self._compromise: LeaseCompromise | None = None
self._closed: bool = False
with _ALL_INSTANCES_LOCK:
_ALL_INSTANCES[id(self)] = self
_register_fork_object(self)
@classmethod
def _reset_class_after_fork(cls) -> None: # pragma: forked child
global _ALL_INSTANCES_LOCK # ruff:ignore[global-statement] # rebinds the module lock to a fresh one in the fork child
_ALL_INSTANCES_LOCK = threading.Lock()
cls._instances = WeakValueDictionary()
cls._instances_lock = threading.RLock()
cls._instances_under_construction = set()
@property
def generation(self) -> int | None:
"""
The generation at which the current hold was granted, or ``None`` when no lock is held.
Generations are monotonic across all participants and hosts, so this is a fencing token: pass it to the
protected resource and have the resource reject any operation carrying a lower generation than the highest it
has accepted. That refuses a holder that paused past ``stale_threshold``, was evicted, and resumed. Removing the
``<lock_file>.rw`` directory starts a new log whose generations begin again at 1.
.. versionadded:: 3.33.0
"""
with self._locks.internal:
return None if self._hold is None else self._hold.participant.generation
@property
def compromise(self) -> LeaseCompromise | None:
"""
How the current hold was lost, or ``None`` while it stands.
Set by the heartbeat thread once a peer has evicted this holder or refreshes have failed for long enough that a
peer could have. The holder should stop using the protected resource when it is set. Until the lost hold is
released, acquiring on this instance raises :class:`RuntimeError` instead of joining it.
.. versionadded:: 3.33.0
"""
with self._locks.internal:
return self._compromise
@contextmanager
def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases a shared read lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a write lock is already held on this instance
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
self.acquire_read(timeout, blocking=blocking)
try:
yield
finally:
self.release()
@contextmanager
def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
"""
Context manager that acquires and releases an exclusive write lock.
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
"""
self.acquire_write(timeout, blocking=blocking)
try:
yield
finally:
self.release()
def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
"""
Acquire a shared read lock.
If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a
read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). On the 0→1
transition the reader publishes its holder record, enters the next snapshot once no live writer is named, and
starts a daemon heartbeat thread that rewrites the record's nonce every ``heartbeat_interval`` seconds so peers
on other hosts do not evict it.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
``None`` uses the instance default
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a write lock is already held on this instance, if this instance was invalidated by
:func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
:raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem
refuses the no-replace hard links the protocol commits with
"""
return self._acquire("read", timeout, blocking=blocking)
def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
"""
Acquire an exclusive write lock.
If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant).
Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not
allowed). Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises
:class:`RuntimeError`.
A writer enters the next snapshot as its writer as soon as no live writer is named, which blocks every new
reader on every host, then waits for the readers that snapshot names to leave. Writer starvation is impossible:
new readers see the named writer and wait behind it.
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
``None`` uses the instance default
:returns: a proxy that can be used as a context manager to release the lock
:raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if this
instance was invalidated by :func:`os.fork`, or if :meth:`close` was called
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
:raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem
refuses the no-replace hard links the protocol commits with
"""
return self._acquire("write", timeout, blocking=blocking)
@classmethod
def get_lock(
cls,
lock_file: str | os.PathLike[str],
timeout: float = -1,
*,
blocking: bool = True,
) -> SoftReadWriteLock:
"""
Return the singleton :class:`SoftReadWriteLock` for *lock_file*.
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
:returns: the singleton lock instance
:raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values
"""
return cls(lock_file, timeout, blocking=blocking)
def close(self) -> None:
"""
Release any held lock and mark the instance closed.
Idempotent. After calling this method the instance can no longer acquire locks — subsequent acquires raise
:class:`RuntimeError`, and so does an acquire another thread is still waiting in, which gives up its claim
instead of taking the lock. A fork-invalidated instance is closed without raising.
"""
if self._creator_pid != os.getpid(): # pragma: forked child
return
# Closed first, under the lock an acquire publishes its hold under: an acquire still waiting then sees the flag
# and leaves instead of publishing, and one that published before it is released below.
with self._locks.internal:
self._closed = True
self.release(force=True)
def release(self, *, force: bool = False) -> None:
"""
Release one level of the current lock.
When the lock level reaches zero the heartbeat thread is stopped, the participant is committed out of the next
snapshot, and its holder record is removed. On a fork-invalidated instance (that is, the child of a
:func:`os.fork` call made while the parent held a lock) this method is a no-op so inherited ``with`` blocks can
unwind cleanly in the child.
:param force: if ``True``, release the lock completely regardless of the current lock level
:raises RuntimeError: if no lock is currently held and *force* is ``False``, or if the calling thread holds no
level of the current lock and *force* is ``False``
"""
if self._creator_pid != os.getpid(): # pragma: forked child
return
self._release(threading.get_ident(), force=force)
def _acquire_level_of(
self, holder: int, mode: Mode, timeout: float, *, blocking: bool, cancel_check: Callable[[], bool]
) -> AcquireReturnProxy:
# Records the level under *holder* instead of the calling thread: an async wrapper's executor thread does not
# identify the tasks it holds for. Gives up with Timeout once *cancel_check* is true.
return self._acquire(mode, timeout, blocking=blocking, holder=holder, cancel_check=cancel_check)
def _release_level_of(self, holder: int) -> None:
self._release(holder, force=False)
def _holds_level_of(self, holder: int) -> bool:
# Read without the internal lock: an async wrapper asks from its event loop, which must not wait out a release
# running on another thread.
return (hold := self._hold) is not None and (holder == hold.write_thread_id or holder in hold.read_depths)
def _release(self, thread_id: int, *, force: bool) -> None:
with self._locks.internal:
hold = self._hold
if hold is None:
if force:
return
msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held"
raise RuntimeError(msg)
if force:
hold.level = 0
else:
self._raise_if_not_holder(hold, thread_id)
hold.leave_one_level(thread_id)
if hold.level > 0:
return
self._hold = None
# Stop the heartbeat before leaving, so a late tick cannot re-read a snapshot this participant just left and
# report an eviction that was its own release. on_compromise runs on that thread and may be what called here,
# in which case there is nothing to join.
hold.heartbeat_stop.set()
try:
if hold.heartbeat_thread is not threading.current_thread():
hold.heartbeat_thread.join(timeout=min(self.heartbeat_interval + 1.0, threading.TIMEOUT_MAX))
finally:
# The hold is already gone from this instance, so a join that raises must not strand its record too.
hold.participant.leave()
def _raise_if_not_holder(self, hold: _Hold, thread_id: int) -> None:
# The hold's own heartbeat thread may release it, since on_compromise runs there.
if thread_id == hold.heartbeat_thread.ident:
return
# Another thread's release would leave the generation log under the holder and let a peer in.
if hold.mode == "write" and thread_id != hold.write_thread_id:
msg = (
f"Cannot release write lock on {self.lock_file} (lock id: {id(self)}) "
f"from thread {thread_id} while it is held by thread {hold.write_thread_id}"
)
raise RuntimeError(msg)
# Readers share one participant, so a thread that holds no read level would end another's hold.
if hold.mode == "read" and thread_id not in hold.read_depths:
msg = (
f"Cannot release read lock on {self.lock_file} (lock id: {id(self)}) "
f"from thread {thread_id}, which does not hold it"
)
raise RuntimeError(msg)
def _acquire(
self,
mode: Mode,
timeout: float | None,
*,
blocking: bool | None,
holder: int | None = None,
cancel_check: Callable[[], bool] | None = None,
) -> AcquireReturnProxy:
if self._creator_pid != os.getpid(): # pragma: forked child
msg = f"SoftReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance"
raise RuntimeError(msg)
timeout = self.timeout if timeout is None else timeout
blocking = self.blocking if blocking is None else blocking
holder = threading.get_ident() if holder is None else holder
with self._locks.internal:
if self._closed:
raise self._closed_error()
timeout = _resolve_read_write_timeout(timeout, blocking=blocking)
if self._hold is not None:
return self._validate_reentrant(mode, holder)
deadline = None if timeout == -1 else time.perf_counter() + timeout
if not _acquire_thread_lock(
self._locks.transaction,
blocking=blocking,
timeout=timeout,
poll_interval=self.poll_interval,
cancel_check=cancel_check,
):
raise Timeout(self.lock_file) from None
try:
return self._do_acquire_inner(mode, deadline, holder, blocking=blocking, cancel_check=cancel_check)
finally:
self._locks.transaction.release()
def _closed_error(self) -> RuntimeError:
return RuntimeError(f"SoftReadWriteLock on {self.lock_file} has been closed")
def _do_acquire_inner(
self,
mode: Mode,
deadline: float | None,
holder: int,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
) -> AcquireReturnProxy:
with self._locks.internal:
if self._hold is not None:
return self._validate_reentrant(mode, holder)
participant = Participant(
self._files,
self._root,
mode,
stale_threshold=self.stale_threshold,
clock=time.monotonic,
ledger=self._ledger,
log=self._log,
)
participant.publish()
try:
self._wait_for(participant.advance, deadline=deadline, blocking=blocking, cancel_check=cancel_check)
except BaseException:
# A contender that gives up must not stay named in the snapshot: a writer left there would block every
# reader until a peer waits out the stale threshold. Its own token is all leave() touches, so it cannot
# undo a peer's claim.
with suppress(OSError):
participant.leave()
raise
stop_event = threading.Event()
hold = _Hold(
level=1,
mode=mode,
write_thread_id=holder if mode == "write" else None,
read_depths={holder: 1} if mode == "read" else {},
participant=participant,
heartbeat_thread=_HeartbeatThread(
refresh=lambda: self._refresh(participant, hold),
interval=self.heartbeat_interval,
stop_event=stop_event,
name=f"filelock-heartbeat-{id(self):x}",
),
heartbeat_stop=stop_event,
last_refresh=time.monotonic(),
)
# Publish the hold and start its heartbeat under one internal-lock section, so a concurrent release() never
# observes a hold whose thread has not started and joins it. If the OS refuses the thread, clear the hold and
# leave: left in place, a peer evicts the unrefreshed record and acquires while this instance still believes it
# holds the lock. A close() that landed while this acquire waited gets the same treatment: published, the hold
# would keep its heartbeat and its slot on an instance nobody can release through.
start_error: BaseException | None = None
with self._locks.internal:
if self._closed:
start_error = self._closed_error()
else:
self._hold = hold
self._compromise = None
try:
hold.heartbeat_thread.start()
except BaseException as error: # ruff:ignore[blind-except] # clear the slot below and re-raise
self._hold = None
start_error = error
if start_error is not None:
participant.leave()
raise start_error
return AcquireReturnProxy(lock=self)
def _validate_reentrant(self, mode: Mode, holder: int) -> AcquireReturnProxy:
hold = self._hold
assert hold is not None # ruff:ignore[assert] # callers dispatch here only inside the self._hold is not None branch
if hold.mode != mode:
opposite = "write" if mode == "read" else "read"
direction = "downgrade" if mode == "read" else "upgrade"
msg = (
f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): "
f"already holding a {opposite} lock ({direction} not allowed)"
)
raise RuntimeError(msg)
if mode == "write" and holder != hold.write_thread_id:
msg = (
f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) "
f"from thread {holder} while it is held by thread {hold.write_thread_id}"
)
raise RuntimeError(msg)
# Publishing a hold clears the loss recorded for the one before, so a loss seen here is this hold's. Its
# heartbeat has stopped and a peer may hold the lock, so a new level must not bypass the log.
if (compromise := self._compromise) is not None:
raise _lost_hold_error(self.lock_file, mode, compromise)
if mode == "read":
hold.read_depths[holder] = hold.read_depths.get(holder, 0) + 1
hold.level += 1
return AcquireReturnProxy(lock=self)
def _wait_for(
self,
predicate: Callable[[], bool],
*,
deadline: float | None,
blocking: bool,
cancel_check: Callable[[], bool] | None,
) -> None:
while not predicate():
# The flag is read without the lock here only to stop waiting early; publishing the hold re-checks it.
if self._closed:
raise self._closed_error()
now = time.perf_counter()
if not blocking or (deadline is not None and now >= deadline):
raise Timeout(self.lock_file)
# An abandoned acquire gives up like a timed-out one, so the caller's rollback withdraws its claim.
if cancel_check is not None and cancel_check():
raise Timeout(self.lock_file)
sleep_for = self.poll_interval
if deadline is not None:
sleep_for = min(sleep_for, max(deadline - now, 0.0))
time.sleep(sleep_for)
def _refresh(self, participant: Participant, hold: _Hold) -> bool:
# The loop ends at the first loss, so the holder hears about it once. A transient filesystem error (ESTALE /
# EIO on the NFS-style filesystems this lock targets) is not a loss: retry rather than report a false
# compromise. Report the record unrefreshable only once failures have run long enough that a peer could evict
# it before the next success would land, a margin before the record actually ages out.
try:
outcome, error = participant.heartbeat()
except Exception as failure: # ruff:ignore[blind-except] # the thread would end with the hold still published
self._report_compromise(hold, "refresh-failed", failure)
return False
if outcome == "ok":
hold.last_refresh = time.monotonic()
return True
if outcome == "lost":
self._report_compromise(hold, "evicted", None)
return False
if time.monotonic() - hold.last_refresh >= self.stale_threshold - self.heartbeat_interval:
self._report_compromise(hold, "refresh-failed", error)
return False
return True
def _report_compromise(self, hold: _Hold, reason: CompromiseReason, error: Exception | None) -> None:
# A tick that outlived its release's join must not stamp the old hold's loss onto whatever was acquired since.
with self._locks.internal:
if self._hold is not hold:
return
compromise = LeaseCompromise(
lock_file=self.lock_file, token=hold.participant.token, reason=reason, error=error
)
self._compromise = compromise
if self._on_compromise is not None:
self._on_compromise(compromise)
def _reset_after_fork_in_child(self) -> None: # pragma: forked child
self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock())
self._hold = None
class _HeartbeatThread(threading.Thread):
def __init__(
self,
refresh: Callable[[], bool],
interval: float,
stop_event: threading.Event,
name: str,
) -> None:
super().__init__(name=name, daemon=True)
self._refresh = refresh
self._interval = interval
self._stop_event = stop_event
def run(self) -> None:
# Set on every exit, an on_compromise that raises included, so nothing waits on a thread that is gone.
try:
while not self._stop_event.wait(self._interval):
if not self._refresh():
return
finally:
self._stop_event.set()
def _lost_hold_error(lock_file: str, mode: Mode, compromise: LeaseCompromise) -> RuntimeError:
return RuntimeError(
f"Cannot acquire {mode} lock on {lock_file}: the current hold was lost ({compromise.reason}), "
"release it before acquiring again"
)
def _validate_intervals(
heartbeat_interval: float, stale_threshold: float | None, poll_interval: float
) -> tuple[float, float, float]:
heartbeat_interval = _seconds("heartbeat_interval", heartbeat_interval)
if not isfinite(heartbeat_interval) or heartbeat_interval <= 0:
msg = f"heartbeat_interval must be positive and finite, got {heartbeat_interval}"
raise ValueError(msg)
# The heartbeat thread waits this long between refreshes, and Event.wait overflows past TIMEOUT_MAX.
_check_timeout_max("heartbeat_interval", heartbeat_interval)
stale_threshold = (
heartbeat_interval * 3 if stale_threshold is None else _seconds("stale_threshold", stale_threshold)
)
if not isfinite(stale_threshold) or stale_threshold <= heartbeat_interval:
msg = (
f"stale_threshold must exceed heartbeat_interval ({heartbeat_interval}) "
f"and be finite, got {stale_threshold}"
)
raise ValueError(msg)
if not isfinite(poll_interval := _seconds("poll_interval", poll_interval)) or poll_interval <= 0:
msg = f"poll_interval must be positive and finite, got {poll_interval}"
raise ValueError(msg)
_check_timeout_max("poll_interval", poll_interval)
# A waiting writer refreshes its record once per poll, so a poll slower than the threshold reads as dead.
if poll_interval >= stale_threshold:
msg = f"poll_interval must be below stale_threshold ({stale_threshold}), got {poll_interval}"
raise ValueError(msg)
return heartbeat_interval, stale_threshold, poll_interval
@dataclass
class _Locks:
internal: threading.Lock
transaction: threading.Lock
@dataclass
class _Hold:
"""Everything that exists only while a lock is held; ``None`` when the instance has no lock."""
level: int
mode: Mode
write_thread_id: int | None
#: Read levels per thread, so a thread can give back only the levels it took.
read_depths: dict[int, int]
participant: Participant
heartbeat_thread: _HeartbeatThread
heartbeat_stop: threading.Event
last_refresh: float
def leave_one_level(self, thread_id: int) -> None:
self.level -= 1
if self.mode == "read":
# The heartbeat thread holds no level of its own, so its release gives back one of a reader's.
owner = thread_id if thread_id in self.read_depths else next(iter(self.read_depths))
if depth := self.read_depths.pop(owner) - 1:
self.read_depths[owner] = depth
def _cleanup_all_instances() -> None: # pragma: no cover - runs from atexit at interpreter shutdown
for instance in list(_ALL_INSTANCES.values()):
with suppress(Exception):
instance.release(force=True)
atexit.register(_cleanup_all_instances)
_register_fork_class(SoftReadWriteLock)
__all__ = [
"SoftReadWriteLock",
"_lost_hold_error",
]
File diff suppressed because it is too large. Load diff
+218
View File
@@ -0,0 +1,218 @@
from __future__ import annotations
__lazy_modules__ = ["contextlib", "filelock._util", "pathlib", "typing", "warnings"]
import os
import sys
import warnings
from contextlib import suppress
from errno import EACCES, EAGAIN, ENOSYS, EWOULDBLOCK
from pathlib import Path
from typing import Final, cast
from filelock._api import BaseFileLock
from filelock._util import remove_own_file
has_fcntl = False
if sys.platform == "win32": # pragma: win32 cover
class UnixFileLock(BaseFileLock):
"""Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems."""
def _acquire(self) -> None:
raise NotImplementedError
def _release(self) -> None:
raise NotImplementedError
else: # pragma: win32 no cover
try:
import fcntl
_ = (fcntl.flock, fcntl.LOCK_EX, fcntl.LOCK_NB, fcntl.LOCK_UN)
except (ImportError, AttributeError):
_FCNTL_UNAVAILABLE: Final[str] = "fcntl is unavailable"
def _lock_fd_nonblocking(_fd: int) -> bool:
raise OSError(ENOSYS, _FCNTL_UNAVAILABLE)
def _unlock_fd(_fd: int) -> None:
raise OSError(ENOSYS, _FCNTL_UNAVAILABLE)
else:
has_fcntl = True
# Contention errnos for a nonblocking flock. EAGAIN/EWOULDBLOCK are the usual "held elsewhere" codes; some
# filesystems report EACCES instead, so treat it as contention too rather than a permanent error.
_CONTENTION_ERRNOS: Final[frozenset[int]] = frozenset({EACCES, EAGAIN, EWOULDBLOCK})
def _lock_fd_nonblocking(fd: int) -> bool:
# One nonblocking exclusive flock attempt shared by UnixFileLock and lock_descriptor, so both contend on
# the same lock and classify errors identically. The caller owns fd; this never closes it.
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as exception:
if exception.errno in _CONTENTION_ERRNOS:
return False
raise
return True
def _unlock_fd(fd: int) -> None:
fcntl.flock(fd, fcntl.LOCK_UN)
def _flock_is_missing(fd: int) -> bool:
try:
_lock_fd_nonblocking(fd)
except OSError as exception:
return exception.errno == ENOSYS
return False # a flock the probe won is released when the caller closes fd
class UnixFileLock(BaseFileLock):
"""
Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.
We leave the lock file in place after release. Unlinking a locked file on Unix splits
waiters across inodes and breaks mutual exclusion for processes that coordinate via the
same path.
"""
def _acquire(self) -> None:
if (missing_flock := self._acquire_native()) is not None:
fd, exception, created = missing_flock
self._switch_to_soft_lock(fd, exception, created=created)
def _acquire_native(self) -> tuple[int, OSError, bool] | None:
if (opened := self._open_lock_file()) is None:
return None
fd, created = opened
self._mark_descriptor_pending(fd)
if created:
# Before the flock: a peer may win it first, and only the creator may set the mode. Outside the flock
# handler below: an fchmod that answers ENOSYS says nothing about flock.
try:
self._apply_explicit_mode(fd)
except BaseException:
self._mark_descriptor_released()
os.close(fd)
raise
# One guard covers the flock through the recording of the hold. A signal handler that raises in between
# would otherwise leave an unreferenced descriptor, and its flock, open until the process exits.
locked = False
try:
locked = _lock_fd_nonblocking(fd)
if locked and self._finalize_locked_fd(fd):
return None
except BaseException as exception:
if not locked and isinstance(exception, OSError) and exception.errno == ENOSYS:
return fd, exception, created
# Contention returns False from _lock_fd_nonblocking, so a raise here is a failure.
self._mark_descriptor_released()
os.close(fd)
raise
self._mark_descriptor_released()
os.close(fd) # contention or an unlinked inode; let the retry loop try again
return None
def _open_lock_file(self) -> tuple[int, bool] | None:
# No O_TRUNC: a contender that loses the flock must not truncate the holder's file (#591), and neither may
# the winner, since a file renamed or hard-linked to the lock path is indistinguishable from a lock file.
# O_EXCL tells whether this attempt created the inode: only then does it get the explicit mode, and the
# ENOSYS fallback removes only its own placeholder. O_NONBLOCK: a lease (Linux F_SETLEASE) on the file would
# otherwise block the open for up to fs.lease-break-time, beyond timeout, blocking=False and cancel_check.
open_flags = os.O_RDWR | os.O_NONBLOCK
if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None:
open_flags |= o_nofollow
open_mode = self._open_mode()
try:
return os.open(self._lock_path, open_flags | os.O_CREAT | os.O_EXCL, open_mode), True
except FileExistsError:
pass
except FileNotFoundError:
# On FUSE/NFS a split LOOKUP + CREATE lets a concurrent unlink() race the create, so ENOENT is transient
# contention. An empty or separator-terminated path (Path() strips the separator) fails this way on
# every attempt on macOS, so re-raise rather than retry forever.
if self._lock_path and not self._lock_path.endswith(os.sep) and Path(self._lock_path).parent.exists():
return None
raise
except PermissionError:
# Sticky-bit dirs (e.g. /tmp): O_CREAT fails if another user owns the file (#317).
if not Path(self._lock_path).exists():
raise
return self._open_existing_lock_file(open_flags, open_mode)
def _open_existing_lock_file(self, open_flags: int, open_mode: int) -> tuple[int, bool] | None:
try:
return os.open(self._lock_path, open_flags, open_mode), False
except FileNotFoundError:
return None # unlinked since the create attempt; let the retry loop start over
except BlockingIOError:
return None # a lease is breaking: contention, so the retry loop keeps the deadline
except PermissionError:
# A marker another account left is readable but not writable. Without flock, waiting on it only needs
# reading it; with flock the lock writes its file, so the denial stands.
try:
fd = os.open(self.lock_file, open_flags & ~os.O_RDWR | os.O_RDONLY)
except (FileNotFoundError, BlockingIOError):
return None # unlinked, or leased: the kernel checks permission before the lease
if _flock_is_missing(fd):
return fd, False
os.close(fd)
raise
def _switch_to_soft_lock(self, fd: int, missing_flock: OSError, *, created: bool) -> None:
# The filesystem does not implement flock. Fail closed when the caller opted out of existence-lock
# semantics (#603), asked to preserve the pathname (#605), or set an on_acquired hook (#607), none of which
# a soft lock can honor.
falls_back: Final = self._fallback_to_soft and not self._preserve_lock_file and self._on_acquired is None
try:
# A file this attempt did not create may be a peer's live marker, so it stays. Our own placeholder goes
# before the close: the open descriptor keeps its inode number from a peer's marker, and an exchange
# puts back a peer's marker that replaced it.
if created and falls_back:
with suppress(OSError):
remove_own_file(self._lock_path, fd)
finally:
self._mark_descriptor_released()
os.close(fd)
if not falls_back:
raise missing_flock
self._fallback_to_soft_lock()
self._acquire()
def _finalize_locked_fd(self, fd: int) -> bool:
# Runs with the flock held; the caller closes fd on failure. A concurrent _release() may have unlinked
# the file between our open() and flock(), which st_nlink 0 shows. That lock guards no name, so report
# it and let the retry loop start over.
if (stat_result := os.fstat(fd)).st_nlink == 0:
return False
self._mark_descriptor_owned(fd, (stat_result.st_dev, stat_result.st_ino))
return True
def _apply_explicit_mode(self, fd: int) -> None:
if self.has_explicit_mode:
with suppress(PermissionError):
os.fchmod(fd, self._context.mode)
def _fallback_to_soft_lock(self: BaseFileLock) -> None:
# Import lazily: this runs only on the rare flock fallback, and asyncio imports _unix, so a
# module-level import of it here would cycle.
from filelock._soft import SoftFileLock # ruff:ignore[import-outside-top-level]
warnings.warn("flock not supported on this filesystem, falling back to SoftFileLock", stacklevel=2)
from filelock.asyncio import AsyncSoftFileLock, BaseAsyncFileLock # ruff:ignore[import-outside-top-level]
self.__class__ = AsyncSoftFileLock if isinstance(self, BaseAsyncFileLock) else SoftFileLock
def _release(self) -> None:
fd = cast("int", self._context.lock_file_fd)
# Retain the descriptor until flock succeeds: a failed unlock leaves the kernel lock held, so is_locked
# must keep reporting held for a retry. Once flock commits, clear held state and close as post-unlock
# cleanup; a close failure (EIO on FUSE/Docker bind mounts) does not make the kernel lock held again.
_unlock_fd(fd)
self._mark_descriptor_released()
self._close_released_fd(fd, default_suppresses=True)
if sys.platform == "win32": # pragma: win32 cover
__all__ = ["UnixFileLock", "has_fcntl"]
else: # pragma: win32 no cover
__all__ = ["UnixFileLock", "_lock_fd_nonblocking", "_unlock_fd", "has_fcntl"]
+399
View File
@@ -0,0 +1,399 @@
from __future__ import annotations
__lazy_modules__ = ["pathlib", "platform", "secrets"]
import errno
import os
import platform
import secrets
import stat
import sys
from contextlib import contextmanager, suppress
from errno import EACCES, EINVAL, EIO, EISDIR, ENOSYS, EXDEV
from functools import cache
from pathlib import Path
from typing import TYPE_CHECKING, Final, cast
if TYPE_CHECKING:
import ctypes
from collections.abc import Callable, Generator
from contextlib import AbstractContextManager
def write_all(fd: int, data: bytes) -> None:
"""
Write the whole buffer to *fd*, looping over the short writes POSIX lets ``os.write`` make.
A caller that stopped after one short write would leave a truncated marker behind. A peer that reads between two
writes sees a prefix of the record; each caller has to tolerate that. Lock records need not survive a power loss, so
we skip ``fsync``.
:param fd: file descriptor open for writing.
:param data: bytes to write in full.
:raises OSError: if a write reports zero progress before the record is complete.
"""
remaining = memoryview(data)
while remaining:
if (written := os.write(fd, remaining)) == 0:
raise OSError(EIO, "os.write wrote 0 bytes before the record was complete")
remaining = remaining[written:]
def raise_on_not_writable_file(filename: str) -> None:
"""
Raise an exception if attempting to open the file for writing would fail.
Separates files that can never be written from files that are writable but currently locked.
:param filename: file to check
:raises OSError: as if the file was opened for writing.
"""
try:
# lstat, not stat: settles exists-and-writable in one syscall, and a hostile symlink at the lock path would
# make stat inspect the link target, letting an attacker turn a contended acquire into a misleading
# PermissionError / IsADirectoryError and probe that target's attributes. The real open passes O_NOFOLLOW and
# refuses the symlink anyway.
file_stat = os.lstat(filename)
except OSError:
return # does not exist, or an error the caller cannot act on
if stat.S_ISDIR(file_stat.st_mode):
if sys.platform == "win32": # pragma: win32 cover
raise PermissionError(EACCES, "Permission denied", filename)
raise IsADirectoryError(EISDIR, "Is a directory", filename) # pragma: win32 no cover
# No mtime guard: the old `if st_mtime != 0` skip covered NFS/Linux quirks where os.lstat returned an all-zero
# struct, which it no longer does. Skipping on mtime 0 let a read-only file or a directory at the lock path pass
# as missing, so acquire() blocked forever on an open that cannot succeed.
# Match open() credentials where supported; group permissions and ACLs can grant access without S_IWUSR. Do not
# follow a symlink, as the lstat above does not: one swapped in since would be judged by its target.
if not (file_stat.st_mode & stat.S_IWUSR) and not os.access(
filename,
os.W_OK,
effective_ids=os.access in os.supports_effective_ids,
follow_symlinks=os.access not in os.supports_follow_symlinks,
):
try:
os.lstat(filename)
except FileNotFoundError:
# A holder may unlink its marker between lstat() and access(); let the caller attempt creation.
return
raise PermissionError(EACCES, "Permission denied", filename)
def ensure_directory_exists(filename: Path | str) -> None:
"""
Ensure the directory containing the file exists (create it if necessary).
:param filename: file.
"""
Path(filename).parent.mkdir(parents=True, exist_ok=True)
@contextmanager
def pin_inode(lock_file: str) -> Generator[int | None]:
"""
Keep the file at *lock_file* allocated for the length of the block and yield its inode number.
An inode is the kernel's identity for a file, independent of its name, and its number names a file only while the
inode stays allocated. ext4 hands a freed number to the next file created. Once a stale marker is unlinked, the next
holder's marker can therefore carry the number a contender read a moment ago. A caller that reads the marker inside
the block, gives up unless it read the yielded inode, and breaks the marker before leaving can rely on the number.
:param lock_file: path to the lock file about to be read and perhaps broken.
:returns: the pinned inode number, or ``None`` where nothing can be pinned. Windows cannot rename a file this
process holds open, so it needs no pin. Without ``O_PATH`` a symlink or a socket cannot be opened.
:raises FileNotFoundError: if the path is gone.
"""
if sys.platform == "win32": # pragma: win32 cover
yield None
else: # pragma: win32 no cover
# O_PATH pins any node, a symlink or socket included, without read permission and without the filesystem seeing
# an open; elsewhere a nonblocking read-only open pins what it can.
flags: Final = getattr(os, "O_PATH", os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) | getattr(os, "O_NOFOLLOW", 0)
try:
fd = os.open(lock_file, flags)
except FileNotFoundError:
raise
except OSError:
yield None
return
try:
yield os.fstat(fd).st_ino
finally:
os.close(fd)
def remove_own_file(lock_file: str, fd: int) -> None: # pragma: win32 no cover
"""
Remove *lock_file* while it still names the file open as *fd*; the caller closes the descriptor afterwards.
Where the platform has an atomic exchange, a stale break takes the file through it, so a peer's marker found at the
path goes back. Otherwise the path is unlinked by name if it still names this file. The open descriptor keeps the
inode allocated, so a matching number cannot belong to a successor's marker.
:raises OSError: if the descriptor cannot be read or the unlink fails.
"""
own: Final = os.fstat(fd)
with suppress(FileNotFoundError):
break_lock_file(lock_file, own.st_mtime, own.st_ino)
if os.path.samestat(os.lstat(lock_file), own):
Path(lock_file).unlink()
def foreign_directory_owner(directory: os.stat_result, parent: os.stat_result) -> int | None:
"""
Return the uid that owns *directory* when another account controls it under a sticky parent.
A lock that keeps its ownership records in a directory relies on them staying in place. The sticky bit (a directory
flag that lets only a file's owner remove it) on a parent such as ``/tmp`` stops other accounts from renaming the
directory away. A directory that this account or the parent's owner owns is therefore safe there. One that another
account owns is outside the lock's control. Without the sticky bit, any process that can create the directory can
also rename this account's own one away, so a refusal there would change nothing. Linux applies the same rule for
``fs.protected_regular``.
:param directory: ``lstat`` of the directory found in place.
:param parent: ``stat`` of the directory that holds it.
:returns: the owning uid when another account controls the directory, otherwise ``None``.
"""
if sys.platform != "win32" and parent.st_mode & stat.S_ISVTX: # pragma: needs foreign-sticky-tmp
return None if directory.st_uid in {os.geteuid(), parent.st_uid} else directory.st_uid
return None
def break_lock_file(lock_file: str, mtime_before: float, ino_before: int) -> None:
"""
Remove the lock file a caller found stale with modification time *mtime_before* and inode *ino_before*.
The caller keeps the judged file open until we return, through :func:`pin_inode` or its own descriptor, so no
other file can carry *ino_before* meanwhile. Right before breaking we ``lstat`` the lock again and leave it alone
if a peer recreated it since the caller judged it stale, as Mercurial re-reads a lock under its break lock.
On Linux (``renameat2``) and macOS (``renamex_np``) the platform can exchange two names in one step. There we swap
the marker with a placeholder of our own, so the lock path never stands empty, and a contender that polls meanwhile
finds a file it cannot parse and waits. If the file we took is not the one judged stale, a peer recreated the lock,
and a second exchange puts it back. Otherwise we remove it and then our placeholder. Of several processes racing to
break the same lock, only one takes the stale file.
Windows cannot exchange names, but it refuses to rename a marker its live holder keeps open. There we rename the
marker aside and, if it was recreated after all, link it back. ``link`` never replaces a name, so a marker a third
process created meanwhile keeps the path. Elsewhere without an exchange (NFS, SMB and other filesystems that refuse
it, platforms without the call) a rename would leave the path empty beside a possibly live holder. We then do not
move the marker, and the caller waits for ``timeout`` or ``break_lock()``.
We compare inodes as well as modification times, because NFS and FAT store coarse modification times. The times are
compared for equality, since a file's owner can set any time. We ``lstat`` so a symlink a peer swaps in is never
followed. The break name carries a random token so no other process can guess it.
:param lock_file: path to the lock file to break.
:param mtime_before: modification time the caller saw when it judged the lock stale.
:param ino_before: inode number the caller saw when it judged the lock stale.
:raises OSError: if taking the marker fails, for example because the file vanished, another user owns it in a
sticky directory, or on Windows its holder keeps it open.
"""
if (current := os.lstat(lock_file)).st_mtime != mtime_before or current.st_ino != ino_before:
return
break_path: Final[str] = f"{lock_file}.break.{os.getpid()}.{secrets.token_hex(16)}"
if exchange_supported(): # pragma: win32 no cover
_swap_out(lock_file, break_path, mtime_before, ino_before)
elif _MOVES_WITHOUT_EXCHANGE: # pragma: win32 cover
_break_by_rename(lock_file, break_path, mtime_before, ino_before)
def _swap_out(lock_file: str, break_path: str, mtime_before: float, ino_before: int) -> None: # pragma: win32 no cover
# An atomic exchange with a placeholder takes the marker without the lock path ever being empty. No contender can
# then create a marker while this one is judged, and a live marker goes back the same way.
fd: Final = os.open(break_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600)
try:
try:
# Not a record any lock writes, so a contender that reads it waits, and one left by a crash ages out. The
# PID tells the contender that evicts it whose break names to search for a marker this process left.
write_all(fd, f"filelock/break\n{os.getpid()}\n".encode())
exchanged = exchange_names(break_path, lock_file)
except OSError:
exchanged = False
if not exchanged:
Path(break_path).unlink()
return
if (grabbed := os.lstat(break_path)).st_mtime == mtime_before and grabbed.st_ino == ino_before:
Path(break_path).unlink()
# Our descriptor keeps the placeholder's inode allocated, so a matching number at the path is still ours.
if os.path.samestat(os.lstat(lock_file), os.fstat(fd)):
Path(lock_file).unlink()
else:
exchange_names(break_path, lock_file)
Path(break_path).unlink()
finally:
os.close(fd)
def _break_by_rename(
lock_file: str, break_path: str, mtime_before: float, ino_before: int
) -> None: # pragma: win32 cover
# Renaming the marker aside empties the path until a recreated live marker is put back. link never replaces a name,
# so a marker a third process created meanwhile keeps the path, and the live one waits under the break name.
# Without os.link, or where the filesystem refuses hard links, the live one waits there too.
with _pinned_for_rename(lock_file):
Path(lock_file).rename(break_path)
if (st_after := os.lstat(break_path)).st_mtime == mtime_before and st_after.st_ino == ino_before:
unlink_path(break_path)
elif stat.S_ISREG(st_after.st_mode) and hasattr(os, "link"): # pragma: needs hard-link
with suppress(OSError):
os.link(break_path, lock_file, follow_symlinks=False)
unlink_path(break_path)
if sys.platform == "win32": # pragma: win32 cover
def _pinned_for_rename(path: str) -> AbstractContextManager[object]:
# A rename by path waits on a Level 1 or batch oplock holder until it answers or closes. Opening first with
# FILE_COMPLETE_IF_OPLOCKED turns that wait into BlockingIOError, and the open handle shares delete, so the
# rename then proceeds. Imported here because _windows imports this module.
from filelock._windows import open_for_delete_without_oplock_wait # ruff:ignore[import-outside-top-level]
return os.fdopen(open_for_delete_without_oplock_wait(path), "rb")
def unlink_path(path: str) -> None:
from filelock._windows import unlink_without_oplock_wait # ruff:ignore[import-outside-top-level]
unlink_without_oplock_wait(path)
else: # pragma: win32 no cover
def unlink_path(path: str) -> None:
Path(path).unlink()
def exchange_supported() -> bool:
"""Whether this platform has a call that swaps two names in one step; a filesystem may still refuse it."""
return _rename_call() is not None
def exchange_names(source: str, target: str) -> bool: # pragma: win32 no cover
"""
Swap the names *source* and *target* in one step, so neither ever stands empty, where :func:`exchange_supported`.
:returns: ``False`` where the filesystem cannot, so the caller falls back
:raises OSError: for any other failure, such as a missing name or a denied rename
"""
renamer, exchange_flag = cast("tuple[Callable[[bytes, bytes, int], int], int]", _rename_call())
if not (error := renamer(os.fsencode(source), os.fsencode(target), exchange_flag)):
return True
# EXDEV: overlayfs and some FUSE filesystems refuse an exchange between layers.
if error in {EINVAL, ENOSYS, EXDEV, getattr(errno, "ENOTSUP", EINVAL), getattr(errno, "EOPNOTSUPP", EINVAL)}:
return False
raise OSError(error, os.strerror(error), source, None, target)
@cache
def _rename_call() -> tuple[Callable[[bytes, bytes, int], int], int] | None:
# The call returns 0 or the errno. Looked up on first use, since most processes never break a lock and ctypes is
# slow to import. None where ctypes is missing (GraalPy) or the platform has no exchange.
try:
import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
except ImportError: # pragma: no cover # GraalPy ships without ctypes
return None
if sys.platform in {"linux", "android"}: # pragma: linux cover
if (renameat2 := _renameat2(ctypes.CDLL(None, use_errno=True))) is None:
return None # pragma: no cover # an architecture the CI matrix does not run
return renameat2, 2 # RENAME_EXCHANGE in linux/fs.h
if sys.platform == "darwin": # pragma: darwin cover
renamex_np: Final = ctypes.CDLL(None, use_errno=True).renamex_np
renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint]
renamex_np.restype = ctypes.c_int
def call_renamex_np(source: bytes, target: bytes, flags: int) -> int:
return 0 if renamex_np(source, target, flags) == 0 else ctypes.get_errno()
return call_renamex_np, 0x2 # RENAME_SWAP in sys/stdio.h
return None # pragma: linux no cover # pragma: android no cover # pragma: darwin no cover
def _renameat2(libc: ctypes.CDLL) -> Callable[[bytes, bytes, int], int] | None: # pragma: linux cover
# Reached through syscall(2) rather than the C library's wrapper. glibc 2.28, musl 1.2.2 and bionic API 30 added
# the wrapper, and Termux ships an older bionic; the kernel has had the call since 3.15. The number is the
# architecture's own, so an architecture outside the table gets no exchange rather than another call.
import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
numbers: Final = {
"x86_64": 316,
"i386": 353,
"i686": 353,
"armv7l": 382,
"armv8l": 382,
"aarch64": 276,
"riscv64": 276,
"loongarch64": 276,
"ppc64le": 357,
"s390x": 347,
}
if (number := numbers.get(platform.machine())) is None:
return None # pragma: no cover # an architecture the CI matrix does not run
syscall: Final = libc.syscall
syscall.restype = ctypes.c_long
at_fdcwd: Final = -100
def call_renameat2(source: bytes, target: bytes, flags: int) -> int:
arguments: Final = (
ctypes.c_long(number),
ctypes.c_int(at_fdcwd),
ctypes.c_char_p(source),
ctypes.c_int(at_fdcwd),
ctypes.c_char_p(target),
ctypes.c_uint(flags),
)
return 0 if syscall(*arguments) == 0 else ctypes.get_errno()
return call_renameat2
def touch(name: str, *, fd: int) -> None:
# Prefer the already-open, already-verified fd so a peer that swaps a symlink or a different file in at the
# path after our O_NOFOLLOW read cannot redirect the touch: utime then targets the inode behind the fd.
# Where the platform cannot utime an fd, fall back to a path-based touch that still refuses to follow a
# symlink where supported, matching the O_NOFOLLOW reads used elsewhere here.
if _SUPPORTS_UTIME_FD: # pragma: needs utime-fd
os.utime(fd, None)
return
os.utime(name, None, follow_symlinks=not _SUPPORTS_UTIME_NOFOLLOW) # pragma: lacks utime-fd
# Windows refuses to rename a marker its live holder keeps open, so renaming aside only moves a marker nobody holds.
# Elsewhere without an exchange a contender could take the emptied path beside a live holder, so the marker stays and
# the lock counts as held.
_MOVES_WITHOUT_EXCHANGE: Final[bool] = sys.platform == "win32"
# Retargeting os.utime to an open fd lets a heartbeat refresh the exact inode it verified instead of whatever the
# pathname now names.
_SUPPORTS_UTIME_FD: Final[bool] = sys.platform != "win32" and os.utime in os.supports_fd
# os.utime follows symlinks unless told not to; not every platform can refuse the follow, so probe support.
_SUPPORTS_UTIME_NOFOLLOW: Final[bool] = os.utime in os.supports_follow_symlinks
__all__ = [
"break_lock_file",
"ensure_directory_exists",
"exchange_names",
"exchange_supported",
"foreign_directory_owner",
"pin_inode",
"raise_on_not_writable_file",
"remove_own_file",
"touch",
"unlink_path",
"write_all",
]
+780
View File
@@ -0,0 +1,780 @@
from __future__ import annotations
__lazy_modules__ = ["filelock._util", "msvcrt", "struct"]
import os
import struct
import sys
import time
from contextlib import contextmanager, suppress
from errno import ENOENT, ENOTDIR, EWOULDBLOCK
from functools import cache
from typing import TYPE_CHECKING, Final, cast
from filelock._api import BaseFileLock
from filelock._util import raise_on_not_writable_file
if TYPE_CHECKING:
from collections.abc import Generator
if sys.platform == "win32": # pragma: win32 cover
import ctypes
import msvcrt
from ctypes import wintypes
_GENERIC_READ: Final[int] = 0x80000000
_GENERIC_WRITE: Final[int] = 0x40000000
_SYNCHRONIZE: Final[int] = 0x00100000
_DELETE: Final[int] = 0x00010000
_DESIRED_ACCESS: Final[int] = _GENERIC_READ | _GENERIC_WRITE | _SYNCHRONIZE
_FILE_SHARE_READ_WRITE: Final[int] = (
0x00000001 | 0x00000002
) # read | write; matches os.open (_SH_DENYNO), no delete
_FILE_OPEN: Final[int] = 1 # open the file only if it exists; the NtCreateFile OPEN_EXISTING
_FILE_OPEN_IF: Final[int] = 3 # open the file if it exists, create it otherwise; the NtCreateFile OPEN_ALWAYS
# Attribute-only access skips share-mode checks, so the probe sees a name's deletion state rather than a reader.
_FILE_READ_ATTRIBUTES: Final[int] = 0x00000080
_FILE_SHARE_ALL: Final[int] = 0x00000001 | 0x00000002 | 0x00000004 # read | write | delete
_FILE_ATTRIBUTE_READONLY: Final[int] = 0x00000001
_FILE_ATTRIBUTE_NORMAL: Final[int] = 0x00000080
_FILE_ATTRIBUTE_REPARSE_POINT: Final[int] = 0x00000400
# CreateOptions: keep the handle synchronous (the CRT and msvcrt.locking rely on a maintained file position),
# refuse a directory, return at once instead of waiting for an oplock holder to answer the break, and open a
# reparse point rather than following it so the check below acts on the link itself.
_FILE_SYNCHRONOUS_IO_NONALERT: Final[int] = 0x00000020
_FILE_NON_DIRECTORY_FILE: Final[int] = 0x00000040
_FILE_DELETE_ON_CLOSE: Final[int] = 0x00001000
_FILE_COMPLETE_IF_OPLOCKED: Final[int] = 0x00000100
_FILE_OPEN_REPARSE_POINT: Final[int] = 0x00200000
_CREATE_OPTIONS: Final[int] = (
_FILE_SYNCHRONOUS_IO_NONALERT | _FILE_NON_DIRECTORY_FILE | _FILE_COMPLETE_IF_OPLOCKED | _FILE_OPEN_REPARSE_POINT
)
# Listing access takes part in share-mode checks, unlike attribute-only access, so a handle that holds it without
# FILE_SHARE_DELETE blocks renames and removals of the directory until it closes.
_FILE_LIST_DIRECTORY: Final[int] = 0x00000001
_FILE_DIRECTORY_FILE: Final[int] = 0x00000001
_DIRECTORY_OPTIONS: Final[int] = _FILE_SYNCHRONOUS_IO_NONALERT | _FILE_DIRECTORY_FILE | _FILE_OPEN_REPARSE_POINT
_OBJ_CASE_INSENSITIVE: Final[int] = 0x00000040 # Win32 name lookups are case-insensitive
_OWNER_WRITE: Final[int] = 0o200
_GROUP_AND_OTHERS: Final[int] = 0o077
# The DACL os.mkdir(mode=0o700) applies, less the inheritance flags a file has no use for: protected, so the
# directory's inheritable entries stay out, with full access for SYSTEM, Administrators and the owner.
_OWNER_ONLY_SDDL: Final[str] = "D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;OW)"
_SDDL_REVISION_1: Final[int] = 1
_INVALID_HANDLE_VALUE: Final[int | None] = ctypes.c_void_p(-1).value
# LockFileEx locks a byte range at an offset carried in OVERLAPPED, independent of the descriptor's file position.
# msvcrt.locking starts at the current position instead, so a metadata write between lock and unlock could shift
# the byte a later unlock targets; the explicit offset removes that hazard for both the path lock and #608's
# descriptor lock.
_LOCKFILE_FAIL_IMMEDIATELY: Final[int] = 0x00000001
_LOCKFILE_EXCLUSIVE_LOCK: Final[int] = 0x00000002
_ERROR_ACCESS_DENIED: Final[int] = 5
_ERROR_LOCK_VIOLATION: Final[int] = 33 # another handle holds the byte range
_ERROR_NOT_LOCKED: Final[int] = 158 # the byte range is not locked
# NtCreateFile returns the raw NTSTATUS as its value, where CreateFileW collapses several of these into one
# ERROR_ACCESS_DENIED. Telling them apart is the point (#604): a name pending deletion or a share conflict is
# transient and worth a retry, a real access denial is not.
_STATUS_SUCCESS: Final[int] = 0x00000000
# A success code that comes with a handle: another handle's oplock is breaking and its holder has not answered yet.
_STATUS_OPLOCK_BREAK_IN_PROGRESS: Final[int] = 0x00000108
_STATUS_ACCESS_DENIED: Final[int] = 0xC0000022
_STATUS_OBJECT_NAME_NOT_FOUND: Final[int] = 0xC0000034
_STATUS_OBJECT_PATH_NOT_FOUND: Final[int] = 0xC000003A
_STATUS_SHARING_VIOLATION: Final[int] = 0xC0000043
_STATUS_DELETE_PENDING: Final[int] = 0xC0000056
#: Probes, 2 ms apart, for transient STATUS_ACCESS_DENIED while a peer unlinks the name (#604). A count ensures
#: every probe runs even when the process stalls; oplock breaks use the caller's wait instead.
_ACCESS_DENIED_PROBES: Final[int] = 250
_ACCESS_DENIED_GRACE: Final[float] = 0.5
_ACCESS_DENIED_RETRY: Final[float] = 0.002
_ntdll: Final[ctypes.WinDLL] = ctypes.WinDLL("ntdll")
_kernel32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True)
_advapi32: Final[ctypes.WinDLL] = ctypes.WinDLL("advapi32", use_last_error=True)
class _UNICODE_STRING(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
_fields_ = (
("Length", wintypes.USHORT), # byte length, not character count
("MaximumLength", wintypes.USHORT),
("Buffer", wintypes.LPWSTR),
)
class _OBJECT_ATTRIBUTES(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
_fields_ = (
("Length", wintypes.ULONG),
("RootDirectory", wintypes.HANDLE),
("ObjectName", ctypes.POINTER(_UNICODE_STRING)),
("Attributes", wintypes.ULONG),
("SecurityDescriptor", ctypes.c_void_p),
("SecurityQualityOfService", ctypes.c_void_p),
)
class _IO_STATUS_BLOCK(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
_fields_ = (
("Status", ctypes.c_void_p), # a union of NTSTATUS and PVOID, so it is pointer-sized
("Information", ctypes.c_void_p),
)
class _SECURITY_ATTRIBUTES(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
_fields_ = (
("nLength", wintypes.DWORD),
("lpSecurityDescriptor", ctypes.c_void_p),
("bInheritHandle", wintypes.BOOL),
)
class _OVERLAPPED(ctypes.Structure): # mirrors the Win32 struct name
_fields_ = (
("Internal", ctypes.c_void_p), # ULONG_PTR: pointer-sized, not DWORD, or the x64 layout corrupts Offset
("InternalHigh", ctypes.c_void_p),
("Offset", wintypes.DWORD), # the DUMMYUNIONNAME struct, flattened: low 32 bits of the byte offset
("OffsetHigh", wintypes.DWORD),
("hEvent", wintypes.HANDLE),
)
class _BY_HANDLE_FILE_INFORMATION(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
_fields_ = (
("dwFileAttributes", wintypes.DWORD),
("ftCreationTime", wintypes.FILETIME),
("ftLastAccessTime", wintypes.FILETIME),
("ftLastWriteTime", wintypes.FILETIME),
("dwVolumeSerialNumber", wintypes.DWORD),
("nFileSizeHigh", wintypes.DWORD),
("nFileSizeLow", wintypes.DWORD),
("nNumberOfLinks", wintypes.DWORD),
("nFileIndexHigh", wintypes.DWORD),
("nFileIndexLow", wintypes.DWORD),
)
_ntdll.NtCreateFile.restype = wintypes.LONG # NTSTATUS
_ntdll.NtCreateFile.argtypes = [
ctypes.POINTER(wintypes.HANDLE),
wintypes.DWORD,
ctypes.POINTER(_OBJECT_ATTRIBUTES),
ctypes.POINTER(_IO_STATUS_BLOCK),
ctypes.POINTER(ctypes.c_longlong), # PLARGE_INTEGER AllocationSize
wintypes.ULONG,
wintypes.ULONG,
wintypes.ULONG,
wintypes.ULONG,
ctypes.c_void_p,
wintypes.ULONG,
]
_ntdll.RtlDosPathNameToNtPathName_U_WithStatus.restype = wintypes.LONG # NTSTATUS
_ntdll.RtlDosPathNameToNtPathName_U_WithStatus.argtypes = [
wintypes.LPCWSTR,
ctypes.POINTER(_UNICODE_STRING),
ctypes.c_void_p,
ctypes.c_void_p,
]
_ntdll.RtlFreeUnicodeString.restype = None
_ntdll.RtlFreeUnicodeString.argtypes = [ctypes.POINTER(_UNICODE_STRING)]
_ntdll.RtlNtStatusToDosError.restype = wintypes.ULONG
_ntdll.RtlNtStatusToDosError.argtypes = [wintypes.LONG]
_kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
_kernel32.CloseHandle.restype = wintypes.BOOL
_kernel32.GetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.POINTER(_BY_HANDLE_FILE_INFORMATION)]
_kernel32.GetFileInformationByHandle.restype = wintypes.BOOL
_kernel32.LockFileEx.argtypes = [
wintypes.HANDLE,
wintypes.DWORD,
wintypes.DWORD,
wintypes.DWORD,
wintypes.DWORD,
ctypes.POINTER(_OVERLAPPED),
]
_kernel32.LockFileEx.restype = wintypes.BOOL
_kernel32.UnlockFileEx.argtypes = [
wintypes.HANDLE,
wintypes.DWORD,
wintypes.DWORD,
wintypes.DWORD,
ctypes.POINTER(_OVERLAPPED),
]
_kernel32.UnlockFileEx.restype = wintypes.BOOL
_advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.argtypes = [
wintypes.LPCWSTR,
wintypes.DWORD,
ctypes.POINTER(ctypes.c_void_p),
ctypes.POINTER(wintypes.ULONG),
]
_advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.restype = wintypes.BOOL
_advapi32.GetNamedSecurityInfoW.argtypes = [
wintypes.LPCWSTR,
ctypes.c_int,
wintypes.DWORD,
ctypes.POINTER(ctypes.c_void_p),
ctypes.POINTER(ctypes.c_void_p),
ctypes.POINTER(ctypes.c_void_p),
ctypes.POINTER(ctypes.c_void_p),
ctypes.POINTER(ctypes.c_void_p),
]
_advapi32.GetNamedSecurityInfoW.restype = wintypes.DWORD
_advapi32.OpenThreadToken.argtypes = [
wintypes.HANDLE,
wintypes.DWORD,
wintypes.BOOL,
ctypes.POINTER(wintypes.HANDLE),
]
_advapi32.OpenThreadToken.restype = wintypes.BOOL
_advapi32.OpenProcessToken.argtypes = [wintypes.HANDLE, wintypes.DWORD, ctypes.POINTER(wintypes.HANDLE)]
_advapi32.OpenProcessToken.restype = wintypes.BOOL
_advapi32.GetTokenInformation.argtypes = [
wintypes.HANDLE,
ctypes.c_int,
ctypes.c_void_p,
wintypes.DWORD,
ctypes.POINTER(wintypes.DWORD),
]
_advapi32.GetTokenInformation.restype = wintypes.BOOL
_kernel32.GetCurrentProcess.restype = wintypes.HANDLE
_kernel32.GetCurrentThread.restype = wintypes.HANDLE
_kernel32.LocalFree.argtypes = [ctypes.c_void_p]
_kernel32.LocalFree.restype = ctypes.c_void_p
_kernel32.CreateDirectoryW.argtypes = [wintypes.LPCWSTR, ctypes.POINTER(_SECURITY_ATTRIBUTES)]
_kernel32.CreateDirectoryW.restype = wintypes.BOOL
_kernel32.CreateFileW.argtypes = [
wintypes.LPCWSTR,
wintypes.DWORD,
wintypes.DWORD,
ctypes.c_void_p,
wintypes.DWORD,
wintypes.DWORD,
wintypes.HANDLE,
]
_kernel32.CreateFileW.restype = ctypes.c_void_p
def _lock_fd_nonblocking(fd: int) -> bool:
# One nonblocking exclusive LockFileEx attempt shared by WindowsFileLock and lock_descriptor, over the one-byte
# range at offset 0. True on acquisition, False on contention, raise otherwise. The caller owns fd; the handle
# from get_osfhandle belongs to the CRT descriptor and must not be closed here.
overlapped = _OVERLAPPED() # zero-initialized, so Offset/OffsetHigh/hEvent are 0
flags = _LOCKFILE_EXCLUSIVE_LOCK | _LOCKFILE_FAIL_IMMEDIATELY
if _kernel32.LockFileEx(msvcrt.get_osfhandle(fd), flags, 0, 1, 0, ctypes.byref(overlapped)):
return True
err = ctypes.get_last_error()
if err == _ERROR_LOCK_VIOLATION:
return False
# A non-contention LockFileEx failure is not reproducible in-process.
raise ctypes.WinError(err) # pragma: no cover
def _unlock_fd(fd: int) -> None:
overlapped = _OVERLAPPED() # the same offset 0 and one-byte length the lock used
if ( # pragma: no cover # only an interrupted earlier release leaves our own range unlocked
not _kernel32.UnlockFileEx(msvcrt.get_osfhandle(fd), 0, 1, 0, ctypes.byref(overlapped))
and (err := ctypes.get_last_error()) != _ERROR_NOT_LOCKED
):
raise ctypes.WinError(err)
class WindowsFileLock(BaseFileLock):
"""
Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems.
Lock file cleanup: Windows attempts to delete the lock file after release, but deletion is
not guaranteed in multi-threaded scenarios where another thread holds an open handle. The lock
file may persist on disk, which does not affect lock correctness.
"""
def _acquire(self) -> None:
raise_on_not_writable_file(self._lock_path)
# The reparse test is bound to the opened handle, so a symlink or junction swapped in cannot defeat it
# through a check-then-open TOCTOU race.
try:
fd = _open_non_reparse_fd(self._lock_path, self._open_mode())
except BlockingIOError:
return # an oplock holder has not answered its break; let the retry loop keep its deadline
if fd is None:
return # open contention (share conflict or a name pending deletion); let the retry loop try again
try:
locked = _lock_fd_nonblocking(fd)
if locked:
self._mark_descriptor_owned(fd)
except BaseException:
# Closing drops the byte-range lock, so a hold already recorded for this descriptor must go with it.
self._mark_descriptor_released()
os.close(fd)
raise
if not locked:
os.close(fd) # another holder owns the byte-range lock; let the retry loop try again
def _release(self) -> None:
fd = cast("int", self._context.lock_file_fd)
# Retain the descriptor until the OS unlock succeeds: if UnlockFileEx raises, the byte-range lock is still
# held, so is_locked must keep reporting held rather than losing the fd. Only after the unlock commits do
# close and unlink run as post-unlock cleanup; their failure cannot make the lock held again.
_unlock_fd(fd)
self._mark_descriptor_released()
self._close_released_fd(fd, default_suppresses=False)
if not self._preserve_lock_file: # preserve_lock_file keeps a stable file identity for the caller (#605)
with suppress(OSError):
unlink_without_oplock_wait(self._lock_path)
def _open_non_reparse_fd(path: str, mode: int, *, oplock_wait: float = _ACCESS_DENIED_GRACE) -> int | None:
"""
Open *path* for locking while refusing reparse points, bound to the handle actually locked.
The file is opened through ``NtCreateFile`` with ``FILE_OPEN_REPARSE_POINT`` so a symlink or junction planted
at the path is not followed, and the reparse decision is read from *that* handle via
``GetFileInformationByHandle`` rather than from a prior pathname query. Reading the held handle closes the
check-then-open race: an attacker cannot swap the path between validation and use because both act on the same
handle. Share mode omits delete so a peer cannot unlink or rename the file out from under a live holder,
matching ``os.open``'s ``_SH_DENYNO``.
``NtCreateFile`` is used instead of ``CreateFileW`` because its return value carries the raw ``NTSTATUS``.
Windows collapses a transient delete-pending name and a permanent access denial into the same Win32
``ERROR_ACCESS_DENIED``; the status keeps them apart, so a real denial fails fast instead of spinning until the
caller's timeout (#604).
The reparse option only guards the final path component; Windows still follows reparse points in intermediate
directories. This assumes the lock file sits in a lock directory untrusted users cannot modify. A path with
attacker-controlled parent directories would need component-by-component handle validation.
:param path: the lock file path.
:param mode: the permission mode; as ``os.open`` does on Windows, a cleared owner-write bit creates the file
read-only. A mode without group and other bits creates it with an owner-only DACL (discretionary access
control list, the Windows permission list); any other mode has no Windows counterpart and leaves the file
the entries its directory hands down. Both only take effect when the file is created, not when an existing
one is opened.
:param oplock_wait: seconds to wait for the holder of an oplock on the file to answer its break.
:returns: a file descriptor owning the opened handle, or ``None`` on a sharing violation or a delete-pending
name, which the caller should treat as contention and retry.
:raises BlockingIOError: if an oplock break stays unanswered for *oplock_wait* seconds.
:raises OSError: if the path resolves to a reparse point, or the open fails for any other reason, raised with
the Win32 error the status maps to. An access denial is raised only once it outlasts a run of probes, since
a peer unlinking the file as it releases can make the open answer ``STATUS_ACCESS_DENIED`` for a moment.
"""
# Emit the audit event os.open would, so consumers watching "open" still see the path-level open and can veto.
sys.audit("open", path, None, os.O_RDWR | os.O_CREAT)
read_only = not mode & _OWNER_WRITE
owner_only = not mode & 0o077
handle, status = _nt_open(path, read_only=read_only, owner_only=owner_only)
denied_probes = 0
oplock_deadline = time.monotonic() + oplock_wait
while True:
if status == _STATUS_ACCESS_DENIED:
if denied_probes >= _ACCESS_DENIED_PROBES:
break
denied_probes += 1
elif status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
if time.monotonic() >= oplock_deadline:
break
else:
break
time.sleep(_ACCESS_DENIED_RETRY)
handle, status = _nt_open(path, read_only=read_only, owner_only=owner_only)
if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
msg = "oplock break in progress"
raise BlockingIOError(EWOULDBLOCK, msg, path)
if status != _STATUS_SUCCESS:
if status in {_STATUS_SHARING_VIOLATION, _STATUS_DELETE_PENDING}:
return None
raise _status_error(status, path)
return _descriptor_for_non_reparse(handle, path, os.O_RDWR)
def open_read_without_oplock_wait(path: str) -> int:
"""
Open *path* read-only and return a descriptor, without waiting for an oplock holder to answer its break.
``os.open`` waits inside the system call until the holder of an oplock on the file answers or closes its handle,
so no timeout or ``blocking=False`` ends the wait. A reparse point at *path* is refused rather than followed.
:param path: the file to read.
:returns: a read-only file descriptor.
:raises BlockingIOError: if an oplock break is in progress, which callers treat as contention.
:raises FileNotFoundError: if *path* does not exist.
:raises OSError: if the open fails for any other reason, or *path* is a reparse point.
"""
sys.audit("open", path, None, os.O_RDONLY)
return _open_existing_without_oplock_wait(path, _GENERIC_READ | _SYNCHRONIZE, os.O_RDONLY)
def open_for_delete_without_oplock_wait(path: str) -> int:
"""
Open *path* with delete access and return a descriptor, without waiting for an oplock holder to answer.
The open handle shares read, write and delete, so a rename of *path* can proceed while the caller keeps it.
A holder that took an oplock through another handle is asked to answer by this open, which then raises, so the
caller skips the rename instead of waiting for it.
:param path: the file to rename.
:returns: a file descriptor owning the handle; the caller closes it.
:raises BlockingIOError: if an oplock break is in progress, which callers treat as contention.
:raises FileNotFoundError: if *path* does not exist.
:raises OSError: if the open fails for any other reason, or *path* is a reparse point.
"""
return _open_existing_without_oplock_wait(path, _DELETE | _FILE_READ_ATTRIBUTES | _SYNCHRONIZE, os.O_RDONLY)
def unlink_without_oplock_wait(path: str) -> None:
"""
Delete *path* through a handle that does not wait for an oplock holder to answer its break.
``os.unlink`` waits like ``os.open`` does. Here an unanswered break leaves the file in place, which callers
accept since deleting a lock file is best effort. A symlink at *path* is deleted as a link.
:param path: the file to delete.
:raises FileNotFoundError: if *path* does not exist.
:raises OSError: if the open fails for any other reason, such as a sharing violation.
"""
sys.audit("os.remove", path, None)
handle, status = _nt_create(
path,
_DELETE | _SYNCHRONIZE,
_FILE_ATTRIBUTE_NORMAL,
_FILE_SHARE_ALL,
_FILE_OPEN,
options=_CREATE_OPTIONS | _FILE_DELETE_ON_CLOSE,
)
if status == _STATUS_SUCCESS:
_kernel32.CloseHandle(handle) # deletes the file
elif status != _STATUS_OPLOCK_BREAK_IN_PROGRESS:
raise _status_error(status, path)
def _open_existing_without_oplock_wait(path: str, access: int, flags: int) -> int:
handle, status = _nt_create(path, access, _FILE_ATTRIBUTE_NORMAL, _FILE_SHARE_ALL, _FILE_OPEN)
if status != _STATUS_SUCCESS:
raise _status_error(status, path)
return _descriptor_for_non_reparse(handle, path, flags)
def _status_error(status: int, path: str) -> OSError:
if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
return BlockingIOError(EWOULDBLOCK, "oplock break in progress", path)
if status in {_STATUS_OBJECT_NAME_NOT_FOUND, _STATUS_OBJECT_PATH_NOT_FOUND}:
return FileNotFoundError(ENOENT, os.strerror(ENOENT), path)
winerror = _ntdll.RtlNtStatusToDosError(status)
return OSError(None, ctypes.FormatError(winerror).strip(), path, winerror)
def _descriptor_for_non_reparse(handle: int, path: str, flags: int) -> int:
info = _BY_HANDLE_FILE_INFORMATION()
# Querying an open handle we just created does not fail.
if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover
err = ctypes.get_last_error()
_kernel32.CloseHandle(handle)
raise ctypes.WinError(err)
if info.dwFileAttributes & _FILE_ATTRIBUTE_REPARSE_POINT:
_kernel32.CloseHandle(handle)
msg = f"Lock file is a reparse point (symlink/junction): {path}"
raise OSError(msg)
try:
# O_NOINHERIT mirrors os.open on Windows: the lock fd must not leak into child processes.
return msvcrt.open_osfhandle(handle, flags | os.O_NOINHERIT)
except BaseException: # pragma: no cover # open_osfhandle audits too; a hook raising must not leak the handle
_kernel32.CloseHandle(handle)
raise
def _open_deletable_for_reading(path: str) -> int:
"""
Open *path* read-only without blocking a concurrent unlink and without following a reparse point.
``os.open`` omits ``FILE_SHARE_DELETE``, so while a process has a file open that way, a delete of it fails with
a sharing violation. Every contender reads a strict claim and its owner unlinks it, so a reader that keeps the
claim open past the owner's retries makes the release fail.
"""
sys.audit("open", path, None, os.O_RDONLY)
handle, status = _nt_create(path, _GENERIC_READ | _SYNCHRONIZE, 0, _FILE_SHARE_ALL, _FILE_OPEN)
if status != _STATUS_SUCCESS:
winerror = _ntdll.RtlNtStatusToDosError(status)
raise OSError(None, ctypes.FormatError(winerror).strip(), path, winerror)
try:
return msvcrt.open_osfhandle(handle, os.O_RDONLY | os.O_NOINHERIT)
except BaseException: # pragma: no cover # open_osfhandle audits too; a hook raising must not leak the handle
_kernel32.CloseHandle(handle)
raise
def _pin_directory(path: str) -> int:
"""
Open the directory at *path* so that no process can rename or remove it until :func:`_unpin_directory`.
Windows has no ``dir_fd``, so an operation resolves every path component by name again. A handle on each
directory of a record path, opened without following a reparse point, keeps that name bound to the directory
checked here until the operation ends.
:raises NotADirectoryError: when *path* is a junction, a symlink or anything other than a directory.
"""
sys.audit("open", path, None, os.O_RDONLY)
# GetFileInformationByHandle needs FILE_READ_ATTRIBUTES; listing access is what takes part in share checks.
access = _FILE_LIST_DIRECTORY | _FILE_READ_ATTRIBUTES | _SYNCHRONIZE
handle, status = _nt_create(path, access, 0, _FILE_SHARE_READ_WRITE, _FILE_OPEN, _DIRECTORY_OPTIONS)
if status != _STATUS_SUCCESS:
winerror = _ntdll.RtlNtStatusToDosError(status)
raise OSError(None, ctypes.FormatError(winerror).strip(), path, winerror)
info = _BY_HANDLE_FILE_INFORMATION()
# Querying an open handle we just created does not fail.
if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover
error = ctypes.get_last_error()
_kernel32.CloseHandle(handle)
raise ctypes.WinError(error)
if info.dwFileAttributes & _FILE_ATTRIBUTE_REPARSE_POINT:
_kernel32.CloseHandle(handle)
raise NotADirectoryError(ENOTDIR, "is a junction or symlink, not a directory", path)
return handle
def _unpin_directory(handle: int) -> None:
_kernel32.CloseHandle(handle)
def _is_unlinked(path: str) -> bool:
"""
Report whether *path* is gone or pending deletion, which ``os.open`` reports as the same EACCES as a denial.
NTFS keeps an unlinked name listed, refusing every open with ``STATUS_DELETE_PENDING``, until the last handle
to the file closes. Another process holding such a handle, such as an antivirus scan, can keep it that way for
longer than any retry grace, yet the name's owner has already removed it.
"""
handle, status = _nt_create(path, _FILE_READ_ATTRIBUTES | _SYNCHRONIZE, 0, _FILE_SHARE_ALL, _FILE_OPEN)
if handle:
_kernel32.CloseHandle(handle)
return status in {_STATUS_OBJECT_NAME_NOT_FOUND, _STATUS_DELETE_PENDING}
def _nt_open(path: str, *, read_only: bool, owner_only: bool) -> tuple[int, int]:
"""
Open or create *path* for locking through ``NtCreateFile`` and return ``(handle, status)``.
Without *owner_only* a file this call creates gets no DACL of its own and inherits its directory's ACL.
"""
attributes = _FILE_ATTRIBUTE_READONLY if read_only else _FILE_ATTRIBUTE_NORMAL
descriptor = _owner_only_descriptor() if owner_only else None
return _nt_create(
path, _DESIRED_ACCESS, attributes, _FILE_SHARE_READ_WRITE, _FILE_OPEN_IF, descriptor=descriptor
)
@cache
def _owner_only_descriptor() -> ctypes.c_void_p:
# The DACL os.mkdir(mode=0o700) applies, minus the directory-only inheritance flags: protected (P), so the
# directory's inheritable entries stay out, with full access (FA) for SYSTEM, Administrators and the owner.
# Converted once and never freed: the open retries every few milliseconds while it waits.
descriptor = ctypes.c_void_p()
if not _advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW(
"D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;OW)", 1, ctypes.byref(descriptor), None
): # pragma: no cover # a constant string converts
raise ctypes.WinError(ctypes.get_last_error())
return descriptor
def _nt_create( # ruff:ignore[too-many-arguments, too-many-positional-arguments] # NtCreateFile's own knobs
path: str,
access: int,
attributes: int,
share: int,
disposition: int,
options: int = _CREATE_OPTIONS,
*,
descriptor: ctypes.c_void_p | None = None,
) -> tuple[int, int]:
"""
Open *path* through ``NtCreateFile`` and return ``(handle, status)``.
``RtlDosPathNameToNtPathName_U_WithStatus`` translates the Win32 path to the NT namespace, handling relative,
drive, UNC and extended-length path forms as Win32 itself would, and allocates a buffer that
``RtlFreeUnicodeString`` releases. The handle is ``0`` unless the status is ``STATUS_SUCCESS``: one that comes
with ``STATUS_OPLOCK_BREAK_IN_PROGRESS`` is closed here, since the file is not ours until the oplock holder
answers.
"""
nt_name = _UNICODE_STRING()
status = _ntdll.RtlDosPathNameToNtPathName_U_WithStatus(path, ctypes.byref(nt_name), None, None) & 0xFFFFFFFF
if status != _STATUS_SUCCESS:
return 0, status
try:
object_attributes = _OBJECT_ATTRIBUTES()
object_attributes.Length = ctypes.sizeof(_OBJECT_ATTRIBUTES)
object_attributes.ObjectName = ctypes.pointer(nt_name)
object_attributes.Attributes = _OBJ_CASE_INSENSITIVE
object_attributes.SecurityDescriptor = descriptor
handle = wintypes.HANDLE()
io_status = _IO_STATUS_BLOCK()
status = (
_ntdll.NtCreateFile(
ctypes.byref(handle),
access,
ctypes.byref(object_attributes),
ctypes.byref(io_status),
None,
attributes,
share,
disposition,
options,
None,
0,
)
& 0xFFFFFFFF
)
finally:
_ntdll.RtlFreeUnicodeString(ctypes.byref(nt_name))
if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
_kernel32.CloseHandle(handle)
if status != _STATUS_SUCCESS:
return 0, status
return handle.value or 0, status
def _directory_is_private(path: str) -> bool:
"""
Report whether only the current user, ``SYSTEM`` and ``Administrators`` can create or remove names in *path*.
A directory whose owner is anyone else can have its ACL rewritten by that owner. Entries that only a child
inherits count too, since the DACL says nothing more precise about who may write.
"""
owner, dacl, descriptor = ctypes.c_void_p(), ctypes.c_void_p(), ctypes.c_void_p()
error = _advapi32.GetNamedSecurityInfoW(
path,
1, # SE_FILE_OBJECT
0x1 | 0x4, # OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION
ctypes.byref(owner),
None,
ctypes.byref(dacl),
None,
ctypes.byref(descriptor),
)
if error == _ERROR_ACCESS_DENIED:
# C:\Windows\Temp is one. Anything that cannot be proven private is not.
return False
if error: # pragma: no cover # the directory was opened or listed an instant earlier
raise ctypes.WinError(error)
try:
# SYSTEM, Administrators, CREATOR OWNER and OWNER RIGHTS; the last two stand for the owner.
trusted = {_sid_from_text(text) for text in ("S-1-5-18", "S-1-5-32-544", "S-1-3-0", "S-1-3-4")}
trusted.add(_user_sid())
# A missing DACL grants everyone full access.
return (
bool(dacl.value)
and _sid_at(cast("int", owner.value)) in trusted
and _name_writers(cast("int", dacl.value)) <= trusted
)
finally:
_kernel32.LocalFree(descriptor)
def _name_writers(dacl: int) -> set[bytes]:
(count,) = struct.unpack("<4xH2x", ctypes.string_at(dacl, 8))
writers = set()
offset = 8
for _ in range(count):
ace_type, _flags, ace_size, mask = struct.unpack("<BBHI", ctypes.string_at(dacl + offset, 8))
# Allowed and allowed-callback ACEs share a layout; object ACEs exist only in directory services. The check
# is conservative: any right that can turn into creating a name counts, namely FILE_ADD_FILE,
# FILE_ADD_SUBDIRECTORY, FILE_DELETE_CHILD, WRITE_DAC, WRITE_OWNER, GENERIC_WRITE and GENERIC_ALL.
if ace_type in {0, 9} and mask & (0x2 | 0x4 | 0x40 | 0x40000 | 0x80000 | 0x40000000 | 0x10000000):
writers.add(_sid_at(dacl + offset + 8))
offset += ace_size
return writers
def _user_sid() -> bytes:
token = wintypes.HANDLE()
token_query: Final = 0x0008
open_as_self: Final = True
# An impersonating thread acts as the account it impersonates, which is the one whose names the lock creates.
if not (
_advapi32.OpenThreadToken(_kernel32.GetCurrentThread(), token_query, open_as_self, ctypes.byref(token))
or _advapi32.OpenProcessToken(_kernel32.GetCurrentProcess(), token_query, ctypes.byref(token))
): # pragma: no cover # every process may query its own token
raise ctypes.WinError(ctypes.get_last_error())
try:
size = wintypes.DWORD()
_advapi32.GetTokenInformation(token, 1, None, 0, ctypes.byref(size))
buffer = ctypes.create_string_buffer(size.value)
if not _advapi32.GetTokenInformation(
token, 1, buffer, size, ctypes.byref(size)
): # pragma: no cover # the buffer has the size the first call asked for
raise ctypes.WinError(ctypes.get_last_error())
return _sid_at(cast("int", ctypes.c_void_p.from_buffer(buffer).value))
finally:
_kernel32.CloseHandle(token)
def _sid_from_text(text: str) -> bytes:
revision, authority, *subauthorities = (int(part) for part in text.removeprefix("S-").split("-"))
return (
struct.pack("<BB", revision, len(subauthorities))
+ authority.to_bytes(6, "big")
+ struct.pack(f"<{len(subauthorities)}I", *subauthorities)
)
def _sid_at(address: int) -> bytes:
return ctypes.string_at(address, 8 + 4 * ctypes.string_at(address + 1, 1)[0])
def _make_private_directory(path: str) -> None:
"""Create *path* with the owner-only descriptor, so that nobody else can add a name to it."""
attributes = _SECURITY_ATTRIBUTES(
nLength=ctypes.sizeof(_SECURITY_ATTRIBUTES),
lpSecurityDescriptor=_owner_only_descriptor(),
bInheritHandle=False,
)
if not _kernel32.CreateDirectoryW(path, ctypes.byref(attributes)): # pragma: no cover # the name is fresh
raise ctypes.WinError(ctypes.get_last_error())
@contextmanager
def _pinned(path: str) -> Generator[tuple[int, int]]:
"""Hold *path* open without delete sharing, which keeps its name from being renamed or removed."""
with _opened(path, _FILE_SHARE_READ_WRITE) as handle:
yield _identity_of(handle)
def _identity_at(path: str) -> tuple[int, int]:
with _opened(path, _FILE_SHARE_ALL) as handle:
return _identity_of(handle)
def _identity_of_fd(fd: int) -> tuple[int, int]:
return _identity_of(msvcrt.get_osfhandle(fd))
@contextmanager
def _opened(path: str, share: int) -> Generator[int]:
# The final component is opened as itself, never through a reparse point.
handle = _kernel32.CreateFileW(
path,
_FILE_READ_ATTRIBUTES,
share,
None,
3, # OPEN_EXISTING
0x02000000 | 0x00200000, # FILE_FLAG_BACKUP_SEMANTICS opens a directory; FILE_FLAG_OPEN_REPARSE_POINT
None,
)
if handle in {None, _INVALID_HANDLE_VALUE}: # pragma: no cover # callers open a name they just made or listed
raise ctypes.WinError(ctypes.get_last_error())
try:
yield cast("int", handle)
finally:
_kernel32.CloseHandle(handle)
def _identity_of(handle: int) -> tuple[int, int]:
info = _BY_HANDLE_FILE_INFORMATION()
if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover
raise ctypes.WinError(ctypes.get_last_error())
return info.dwVolumeSerialNumber, info.nFileIndexHigh << 32 | info.nFileIndexLow
else: # pragma: win32 no cover
class WindowsFileLock(BaseFileLock):
"""Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems."""
def _acquire(self) -> None:
raise NotImplementedError
def _release(self) -> None:
raise NotImplementedError
__all__ = [
"WindowsFileLock",
"_directory_is_private",
"_identity_at",
"_identity_of_fd",
"_is_unlinked",
"_make_private_directory",
"_open_deletable_for_reading",
"_open_non_reparse_fd",
"_pin_directory",
"_pinned",
"_unpin_directory",
"open_for_delete_without_oplock_wait",
"open_read_without_oplock_wait",
"unlink_without_oplock_wait",
]
+925
View File
@@ -0,0 +1,925 @@
"""An asyncio-based implementation of the file lock."""
from __future__ import annotations
__lazy_modules__ = [
"asyncio",
"concurrent",
"contextlib",
"filelock._async",
"filelock._error",
"filelock._util",
"functools",
"inspect",
"threading",
]
import asyncio
import contextlib
import functools
import logging
import os
import sys
import time
from concurrent import futures
from dataclasses import dataclass, field, fields
from inspect import iscoroutinefunction
from threading import RLock
from typing import TYPE_CHECKING, Final, NoReturn, TypeVar, cast
from filelock._api import (
_UNSET_FILE_MODE,
BaseFileLock,
CloseErrorPolicy,
ContextErrorPolicy,
FileLockContext,
FileLockMeta,
ThreadLocalFileContext,
_append_exception_context,
_canonical,
_delegate_to_current,
_ExtraValue,
_fork_transition,
_grouped_errors,
_is_system_call_failure,
_raise_body_and_release,
_raise_chained_errors,
_raise_cleanup_errors,
_raise_grouped_errors,
_register_fork_object,
_resolve_poll_interval,
_resolve_timeout,
)
from filelock._async import (
_AsyncTransitionGate,
_AsyncTransitionUnavailableError,
_BackendOutcome,
_capture_awaitable,
_capture_call,
_drain_future,
_future_result,
_run_in_executor,
_wait_until_done,
)
from filelock._error import Timeout
from filelock._lease import SoftFileLease
from filelock._soft import SoftFileLock
from filelock._strict import StrictSoftFileLock
from filelock._unix import UnixFileLock
from filelock._util import ensure_directory_exists
from filelock._windows import WindowsFileLock
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Coroutine, Hashable, Mapping
from types import TracebackType
from typing import Self
_LOGGER: Final[logging.Logger] = logging.getLogger("filelock")
_ASYNC_RELEASE_CANCELLATION_ERRORS: Final[str] = "lock release cancellation and backend release both failed"
_ASYNC_CONTEXT_RELEASE_ERRORS: Final[str] = "context body, release cancellation, and backend release failed"
_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR: Final[str] = "_filelock_async_release_cancellation"
_ASYNC_RELEASE_CANCELLATION_MARKER: Final[list[None]] = []
_FINALIZER_RELEASES: Final[set[asyncio.Task[None] | futures.Future[None]]] = set()
_AT = TypeVar("_AT", bound="BaseAsyncFileLock")
_T = TypeVar("_T")
class AsyncFileLockMeta(FileLockMeta):
def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters
cls: type[_AT], # ruff:ignore[invalid-first-argument-name-for-method] # metaclass __call__ receives the class being constructed
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
context_error_policy: ContextErrorPolicy = "chain",
close_error_policy: CloseErrorPolicy = "default",
fallback_to_soft: bool = True,
preserve_lock_file: bool = False,
on_acquired: Callable[[int], None] | None = None,
loop: asyncio.AbstractEventLoop | None = None,
run_in_executor: bool = True,
executor: futures.Executor | None = None,
**kwargs: _ExtraValue,
) -> _AT:
if thread_local and run_in_executor:
msg = "run_in_executor is not supported when thread_local is True"
raise ValueError(msg)
return super().__call__( # a subclass may add options of its own, as AsyncSoftFileLease does
**kwargs,
lock_file=lock_file,
timeout=timeout,
mode=mode,
thread_local=thread_local,
blocking=blocking,
is_singleton=is_singleton,
poll_interval=poll_interval,
lifetime=lifetime,
context_error_policy=context_error_policy,
close_error_policy=close_error_policy,
fallback_to_soft=fallback_to_soft,
preserve_lock_file=preserve_lock_file,
on_acquired=on_acquired,
loop=loop,
run_in_executor=run_in_executor,
executor=executor,
)
class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
"""
Base class for asynchronous file locks.
.. versionadded:: 3.15.0
"""
_deadlock_holder_desc: str = "BaseAsyncFileLock instance in this task"
_constructor_lifetime_warning_stacklevel: int = 4
@staticmethod
def _deadlock_scope() -> Hashable | None:
# One event loop thread runs every task, so a thread-scoped registry cannot tell a same-task reacquire
# (a real deadlock: the polling task never reaches its own release) from another task queuing behind the
# holder (no deadlock: each poll yields, so the holder runs on and releases). Only the first may fail
# fast, so scope holders to the task.
return asyncio.current_task()
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
self,
lock_file: str | os.PathLike[str],
timeout: float = -1,
mode: int = _UNSET_FILE_MODE,
thread_local: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility
*,
blocking: bool = True,
is_singleton: bool = False,
poll_interval: float = 0.05,
lifetime: float | None = None,
context_error_policy: ContextErrorPolicy = "chain",
close_error_policy: CloseErrorPolicy = "default",
fallback_to_soft: bool = True,
preserve_lock_file: bool = False,
on_acquired: Callable[[int], None] | None = None,
loop: asyncio.AbstractEventLoop | None = None,
run_in_executor: bool = True,
executor: futures.Executor | None = None,
) -> None:
"""
Create a new lock object.
:param lock_file: path to the file
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the
acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a
negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
:param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and
default ACLs, preserving POSIX default ACL inheritance in shared directories.
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to
``False`` (the default) then the lock will be reentrant across threads. When ``True``, **all fields of the
lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``,
``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change
the value seen by another thread; threads that did not perform the write continue to see the value supplied
at construction time. ``mode`` has no setter, so construction is the only place it is ever set. If you need
configuration values to be visible across threads, construct the lock with ``thread_local=False``.
:param blocking: whether the lock should be blocking or not
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock
file. This is useful if you want to use the lock object for reentrant locking without needing to pass the
same object around.
:param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value
in the acquire method, if no poll_interval value (``None``) is given.
:param lifetime: for :class:`AsyncSoftFileLock`, the age in seconds after which a waiting process may delete
the marker, even while its holder remains alive. This legacy expiry mode does not provide strict mutual
exclusion. ``None`` (the default) disables age-based expiry. Native OS locks (:class:`AsyncFileLock`)
cannot be revoked by file age and ignore a non-``None`` ``lifetime`` with a warning.
:param context_error_policy: how a context manager reconciles a failure in its body with a failure while
releasing on exit. ``"chain"`` (the default) keeps Python's behavior: the release error propagates with the
body error in its ``__context__``. ``"group"`` raises a :class:`BaseExceptionGroup` holding the body error
first and the release error second, so neither hides the other.
:param close_error_policy: for native locks (:class:`AsyncFileLock`), what to do with an ``os.close`` failure
after the OS unlock has already committed. ``"default"`` keeps each platform's historical behavior,
``"raise"`` always propagates the ``OSError``, and ``"suppress"`` always ignores it.
:param fallback_to_soft: for :class:`AsyncFileLock`, whether to fall back to soft existence locking when
``flock`` returns ``ENOSYS``. ``True`` (default) keeps the fallback; ``False`` propagates the error.
:param preserve_lock_file: for native locks (:class:`AsyncFileLock`), whether filelock promises not to unlink
the lock pathname on release. ``False`` (default) keeps each backend's cleanup; ``True`` keeps a stable file
identity (Windows skips its unlink, Unix refuses the ``ENOSYS`` soft fallback). :class:`AsyncSoftFileLock`
rejects ``True``.
:param on_acquired: for native locks (:class:`AsyncFileLock`), a callable invoked with the borrowed lock
descriptor once per physical acquisition, after the lock is held but before
:meth:`~BaseAsyncFileLock.acquire` returns. With ``run_in_executor=True`` (the default) it runs in the
backend executor. It must not close or unlock the descriptor; a raise rolls the acquisition back.
:class:`AsyncSoftFileLock` rejects it.
:param loop: the event loop a garbage-collected lock schedules its release on when no loop runs in the
collecting thread. :meth:`~BaseAsyncFileLock.acquire` and :meth:`~BaseAsyncFileLock.release` always run
on the loop that awaits them.
:param run_in_executor: If this is set to ``True`` then the lock will be acquired in an executor.
:param executor: The executor to use. If not specified, the default executor will be used.
"""
self._creator_pid = os.getpid()
self._holder_id = id(self)
self._is_thread_local = thread_local
self._is_singleton = is_singleton
self._context_error_policy = context_error_policy # already validated by the metaclass
self._close_error_policy = close_error_policy # already validated by the metaclass
self._fallback_to_soft = fallback_to_soft
self._preserve_lock_file = preserve_lock_file # already validated by the metaclass
self._on_acquired = on_acquired # already validated by the metaclass
# BaseFileLock.__init__ never runs here, and the finalizer releases thread-local holds through these.
self._transition_lock = RLock()
self._thread_holds = {}
self._thread_pending = {}
self._async_context: Final[AsyncFileLockContext] = (
AsyncThreadLocalFileContext if thread_local else AsyncFileLockContext
)(
lock_file=os.fspath(lock_file),
timeout=timeout,
mode=mode,
blocking=blocking,
poll_interval=poll_interval,
lifetime=lifetime,
loop=loop,
run_in_executor=run_in_executor,
executor=executor,
)
# BaseFileLock declares _context as FileLockContext; narrowing that writable attribute here is unsound.
self._context = self._async_context
_register_fork_object(self)
def _singleton_extra_mismatches(self, kwargs: Mapping[str, _ExtraValue], /) -> dict[str, tuple[str, str]]:
mismatches = super()._singleton_extra_mismatches(kwargs)
# Loops and executors compare by identity: a different one schedules the lock's work elsewhere.
mismatches.update(
(name, (str(kwargs[name]), str(value)))
for name, value in (("loop", self.loop), ("executor", self.executor))
if kwargs[name] is not value
)
if kwargs["run_in_executor"] != self.run_in_executor:
mismatches["run_in_executor"] = (str(kwargs["run_in_executor"]), str(self.run_in_executor))
return mismatches
@property
def run_in_executor(self) -> bool:
"""Whether run in executor."""
return self._async_context.run_in_executor
@property
def executor(self) -> futures.Executor | None:
"""The executor."""
return self._async_context.executor
@executor.setter
def executor(self, value: futures.Executor | None) -> None: # pragma: no cover
"""
Change the executor.
:param futures.Executor | None value: the new executor or ``None``
"""
self._async_context.executor = value
@property
def loop(self) -> asyncio.AbstractEventLoop | None:
"""The event loop."""
return self._async_context.loop
async def acquire( # ty: ignore[invalid-method-override]
self,
timeout: float | None = None,
poll_interval: float | None = None,
*,
blocking: bool | None = None,
cancel_check: Callable[[], bool] | None = None,
) -> AsyncAcquireReturnProxy:
"""
Try to acquire the file lock.
:param timeout: maximum wait time for acquiring the lock, ``None`` means use the default
:attr:`~BaseFileLock.timeout` is and if ``timeout < 0``, there is no timeout and this method will block
until the lock could be acquired
:param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default
:attr:`~BaseFileLock.poll_interval`
:param blocking: whether to retry until this call holds the lock or the timeout expires, ``None`` means use
the default :attr:`~BaseFileLock.blocking`. ``False`` makes one attempt and raises :class:`~Timeout` on
failure.
:param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll
iteration. When triggered, raises :class:`~Timeout` just like an expired timeout.
:returns: a context object that will unlock the file when the context is exited
:raises Timeout: if fails to acquire lock within the timeout period
:raises ValueError: if a blocking call gets a ``nan`` timeout, or *poll_interval* exceeds
:data:`threading.TIMEOUT_MAX`
.. code-block:: python
# You can use this method in the context manager (recommended)
async with await lock.acquire():
pass
# Or use an equivalent try-finally construct:
await lock.acquire()
try:
pass
finally:
await lock.release()
"""
self._raise_if_inherited()
if timeout is None:
timeout = self._context.timeout
if blocking is None:
blocking = self._context.blocking
if poll_interval is None:
poll_interval = self._context.poll_interval
poll_interval = _resolve_poll_interval(poll_interval)
timeout = _resolve_timeout(timeout, blocking=blocking)
start_time = time.perf_counter()
try:
return await self._acquire_with_admission(
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
poll_interval=poll_interval,
start_time=start_time,
)
except _AsyncTransitionUnavailableError:
raise Timeout(self.lock_file) from None
async def _acquire_with_admission(
self,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
poll_interval: float,
start_time: float,
) -> AsyncAcquireReturnProxy:
async with self._async_context.transition_gate.hold_for_acquire(
blocking=blocking,
cancel_check=cancel_check,
deadline=None if timeout < 0 else start_time + timeout,
poll_interval=poll_interval,
):
# Re-entry only counts, so it skips the executor round trip that resolving the path costs.
if self.is_locked:
self._context.lock_counter += 1
return AsyncAcquireReturnProxy(lock=self)
# A canceled provisional acquire must finish rollback before another caller can claim its descriptor.
canonical = await self._off_loop(functools.partial(_canonical, self.lock_file))
self._context.lock_counter += 1
self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking)
self._pin_lock_path(canonical)
try:
await self._async_poll_until_acquired(
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
poll_interval=poll_interval,
start_time=start_time,
)
except BaseException:
self._reconcile_failed_acquire(canonical)
raise
self._commit_acquire(canonical)
return AsyncAcquireReturnProxy(lock=self)
async def _off_loop(self, call: Callable[[], _T]) -> _T:
# A slow mount stalls a filesystem call, and the loop with it; a coroutine backend runs on the loop anyway.
if not self.run_in_executor or iscoroutinefunction(self._acquire):
return call()
future = _run_in_executor(asyncio.get_running_loop(), self.executor, call)
await _wait_until_done(future)
return _future_result(future)
async def _async_poll_until_acquired(
self,
*,
blocking: bool,
cancel_check: Callable[[], bool] | None,
timeout: float,
poll_interval: float,
start_time: float,
) -> None:
lock_id = id(self)
lock_filename = self.lock_file
await self._off_loop(functools.partial(ensure_directory_exists, self._lock_path))
while True:
self._raise_if_inherited()
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
await self._run_acquire_attempt()
self._raise_if_inherited()
if self.is_locked:
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
return
if self._check_give_up(
blocking=blocking,
cancel_check=cancel_check,
timeout=timeout,
start_time=start_time,
):
raise Timeout(lock_filename, await self._off_loop(self._unprobeable_holder))
delay = poll_interval
if timeout >= 0:
delay = min(delay, max(start_time + timeout - time.perf_counter(), 0.0))
_LOGGER.debug("Lock %s not acquired on %s, waiting %s seconds ...", lock_id, lock_filename, delay)
await asyncio.sleep(delay)
async def _run_acquire_attempt(self) -> None:
acquire_future = self._start_internal_method(
self._acquire_with_fork_tracking_async
if iscoroutinefunction(self._acquire)
else self._break_expired_and_acquire
)
try:
await _wait_until_done(acquire_future)
except asyncio.CancelledError as cancellation:
acquire_error: BaseException | None = None
try:
await _drain_future(acquire_future)
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
acquire_error = error
rollback_error: BaseException | None = None
if self.is_locked:
try:
await _drain_future(self._start_tracked_release())
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
rollback_error = error
if acquire_error is not None:
if rollback_error is not None: # pragma: needs fcntl
self._raise_cancelled_errors(
"lock acquisition cancellation, backend attempt, and rollback failed",
cancellation,
acquire_error,
rollback_error,
)
self._raise_cancelled_errors(
"lock acquisition cancellation and backend attempt both failed", cancellation, acquire_error
)
if rollback_error is not None:
self._raise_cancelled_errors(
"lock acquisition cancellation and rollback both failed", cancellation, rollback_error
)
raise
try:
_future_result(acquire_future)
except asyncio.CancelledError as acquire_error:
await self._rollback_backend_cancelled_acquire(acquire_error)
async def _rollback_backend_cancelled_acquire(self, acquire_error: asyncio.CancelledError) -> NoReturn:
if self.is_locked:
try:
await _drain_future(self._start_tracked_release())
except BaseException as rollback_error: # ruff:ignore[blind-except] # both backend errors must surface
self._raise_acquire_rollback_errors(acquire_error, rollback_error)
raise acquire_error
def _raise_acquire_rollback_errors(self, acquire_error: BaseException, rollback_error: BaseException) -> NoReturn:
if self._context_error_policy == "group":
_raise_grouped_errors("lock acquisition backend and rollback both failed", acquire_error, rollback_error)
_raise_chained_errors(acquire_error, rollback_error)
def _raise_cancelled_errors(
self,
message: str,
cancellation: asyncio.CancelledError,
first_error: BaseException,
second_error: BaseException | None = None,
) -> NoReturn:
if self._context_error_policy == "group":
marker = (
(_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER)
if message == _ASYNC_RELEASE_CANCELLATION_ERRORS
else None
)
if second_error is None:
_raise_grouped_errors(message, cancellation, first_error, marker=marker)
_raise_grouped_errors(message, cancellation, first_error, second_error, marker=marker)
if (context := first_error.__context__) is not None and context is not cancellation:
if (cancellation_context := cancellation.__context__) is not None:
_append_exception_context(context, cancellation_context)
cancellation.__context__ = context
first_error.__context__ = cancellation
_raise_chained_errors(first_error, second_error)
async def release(self, force: bool = False) -> None: # ty: ignore[invalid-method-override] # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility
"""
Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file
itself may be deleted automatically, the behavior is platform-specific.
:param force: If true, the lock counter is ignored and the lock is released in every case.
"""
if self._creator_pid != os.getpid() or not self.is_locked:
return
async with self._async_context.transition_gate.hold():
await self._release_serialized(force=force)
async def _release_serialized(self, *, force: bool) -> None:
counter_before: Final = self._context.lock_counter
if not self.is_locked:
return
if not force and self._context.lock_counter > 1:
self._context.lock_counter -= 1
return
try:
await self._release_through_backend()
except BaseException as release_error:
if self._release_interrupted_before_commit(counter_before, release_error):
with contextlib.suppress(Exception):
self._release_with_fork_tracking()
self._commit_release_if_released()
raise
def _release_interrupted_before_commit(self, counter_before: int, release_error: BaseException) -> bool:
# Same rule as the sync release: a handler that raised before anything committed leaves the counter as it was,
# and only then does a second attempt release the hold. A worker thread never runs a handler, so a lock that
# runs its backend in the executor has nothing to retry.
return (
counter_before == self._context.lock_counter
and self.is_locked
and not self.run_in_executor
and not iscoroutinefunction(self._release)
and not isinstance(release_error, asyncio.CancelledError)
and not _is_system_call_failure(release_error)
)
async def _release_through_backend(self) -> None:
lock_id, lock_filename = id(self), self.lock_file
_LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename)
release_future = self._start_tracked_release()
try:
await _wait_until_done(release_future)
except asyncio.CancelledError as cancellation:
try:
await _drain_future(release_future)
except BaseException as release_error: # ruff:ignore[blind-except] # cancellation and backend failure must both surface
self._commit_release_if_released()
self._raise_cancelled_errors(_ASYNC_RELEASE_CANCELLATION_ERRORS, cancellation, release_error)
self._commit_release()
raise
try:
_future_result(release_future)
except BaseException: # state follows the backend for control-flow exceptions too
self._commit_release_if_released()
raise
self._commit_release()
_LOGGER.debug("Lock %s released on %s", lock_id, lock_filename)
def _commit_release_if_released(self) -> None:
# Commit only when the backend actually unlocked (close or unlink failed after the OS unlock). If the lock is
# still held, keep the counter so a later release can retry.
if not self.is_locked:
self._commit_release()
def _start_internal_method(
self, method: Callable[[], None] | Callable[[], Coroutine[None, None, None]]
) -> asyncio.Future[_BackendOutcome[None]]:
if iscoroutinefunction(method):
return asyncio.create_task(_capture_awaitable(cast("Callable[[], Coroutine[None, None, None]]", method)()))
loop = asyncio.get_running_loop()
sync_method = cast("Callable[[], None]", method)
if self.run_in_executor:
return _run_in_executor(loop, self.executor, sync_method)
future: asyncio.Future[_BackendOutcome[None]] = loop.create_future()
future.set_result(_capture_call(sync_method))
return future
def _start_tracked_release(self) -> asyncio.Future[_BackendOutcome[None]]:
return self._start_internal_method(
self._release_with_fork_tracking_async
if iscoroutinefunction(self._release)
else self._release_with_fork_tracking
)
def _break_expired_and_acquire(self) -> None:
# Runs as one backend call so the lifetime check's lstat and unlink stay off the event loop with the acquire.
self._try_break_expired_lock()
self._acquire_with_fork_tracking()
async def _acquire_with_fork_tracking_async(self) -> None:
self._try_break_expired_lock()
with _fork_transition(self):
try:
await cast("Callable[[], Awaitable[None]]", self._acquire)()
except BaseException as acquisition_error:
await self._rollback_failed_acquire_async(acquisition_error)
raise
try:
self._register_context_descriptor()
except BaseException as registration_error: # cancellation must roll back the descriptor
await self._rollback_failed_registration_async(registration_error)
raise
if self.is_locked:
await self._invoke_on_acquired_async()
async def _rollback_failed_acquire_async(self, acquisition_error: BaseException) -> None:
if not self.is_locked:
return
registration_error: BaseException | None = None
tracking_error: BaseException | None = None
try:
self._register_context_descriptor()
except BaseException as error: # ruff:ignore[blind-except] # preserve registration and acquisition failures
registration_error = error
try:
# Rollback may fail too; retain the fd so a child can close it without another identity probe.
self._register_unverified_context_descriptor()
except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback
tracking_error = error
try:
await _drain_future(self._start_tracked_release())
except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and acquisition failures
if registration_error is None and tracking_error is None:
self._raise_acquire_rollback_errors(acquisition_error, rollback_error)
_raise_cleanup_errors(
"lock acquisition cleanup failed",
acquisition_error,
registration_error,
tracking_error,
rollback_error,
)
if registration_error is not None:
_raise_cleanup_errors(
"lock acquisition cleanup failed", acquisition_error, registration_error, tracking_error
)
async def _rollback_failed_registration_async(self, registration_error: BaseException) -> None:
tracking_error: BaseException | None = None
try:
# Rollback may fail too; retain the fd so a child can close it without another identity probe.
self._register_unverified_context_descriptor()
except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback
tracking_error = error
try:
await _drain_future(self._start_tracked_release())
except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and registration failures
_raise_cleanup_errors(
"descriptor registration cleanup failed", registration_error, tracking_error, rollback_error
)
if tracking_error is not None: # pragma: no cover - requires failed in-memory fallback
_raise_cleanup_errors("descriptor registration cleanup failed", registration_error, tracking_error)
async def _invoke_on_acquired_async(self) -> None:
if self._on_acquired is None or self._context.lock_counter != 1:
return
try:
self._on_acquired(cast("int", self._context.lock_file_fd))
except BaseException as callback_error: # caller control-flow errors must release the lock
callback_context = callback_error.__context__
try:
await _drain_future(self._start_tracked_release())
except BaseException as release_error: # ruff:ignore[blind-except] # both errors surface via the group below
_raise_body_and_release(callback_error, release_error)
callback_error.__context__ = callback_context
raise
async def _release_with_fork_tracking_async(self) -> None:
with _fork_transition(self):
try:
await cast("Callable[[], Awaitable[None]]", self._release)()
finally:
self._unregister_released_descriptor()
def __enter__(self) -> NoReturn:
"""Sync context manager entry is not supported because lock acquisition is a coroutine."""
msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager."
raise NotImplementedError(msg)
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
"""Sync context manager exit is not supported because lock release is a coroutine."""
msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager."
raise NotImplementedError(msg)
async def __aenter__(self) -> Self:
"""
Acquire the lock.
:returns: the lock object
"""
await self.acquire()
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
"""
Release the lock, reconciling a release failure with any body failure per :attr:`context_error_policy`.
:param exc_type: the exception type if raised
:param exc_value: the exception value if raised
:param traceback: the exception traceback if raised
"""
await self._release_in_context(exc_value)
async def _release_in_context( # ty: ignore[invalid-method-override]
self, body_error: BaseException | None
) -> None:
# The async counterpart of BaseFileLock._release_in_context: await release, then apply the same policy.
try:
await self.release()
except BaseException as release_error:
if body_error is None:
raise
if self._context_error_policy == "chain":
_append_exception_context(release_error, body_error)
raise
if (
errors := _grouped_errors(
release_error,
_ASYNC_RELEASE_CANCELLATION_ERRORS,
(_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER),
)
) is not None:
match errors:
# The marker is only ever set on a (CancelledError, backend_error) pair, so this always matches.
case (asyncio.CancelledError() as cancellation, backend_error): # pragma: no branch
_raise_grouped_errors(_ASYNC_CONTEXT_RELEASE_ERRORS, body_error, cancellation, backend_error)
_raise_body_and_release(body_error, release_error)
def _release_abandoned_holds(self) -> None:
# Runs from garbage collection, where no event loop may be running.
with contextlib.suppress(Exception):
# A thread-local lock never uses an executor, so an in-place release matches the loop's and also reaches
# threads whose loops it cannot schedule on.
if self._is_thread_local and not iscoroutinefunction(self._release):
self._release_without_loop()
return
try:
loop = asyncio.get_running_loop()
except RuntimeError:
# At interpreter exit daemon threads no longer run, so a loop running in one never runs the release.
if (loop := self._async_context.loop) is None or loop.is_closed() or sys.is_finalizing():
self._release_without_loop()
return
if not loop.is_running(): # pragma: no cover
loop.run_until_complete(self.release(force=True))
return
# Off the deadlock registry, a new lock in this task waits for the scheduled release instead of failing.
self._drop_registry_entry()
# The stored loop runs in another thread, and only its own thread may create tasks on it.
pending = asyncio.run_coroutine_threadsafe(self.release(force=True), loop)
else:
self._drop_registry_entry()
pending = loop.create_task(self.release(force=True))
# The loop keeps only a weak reference to a task, so hold one until the release finishes.
_FINALIZER_RELEASES.add(pending)
pending.add_done_callback(self._settle_finalizer_release)
def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future[None]) -> None:
_FINALIZER_RELEASES.discard(pending)
# Nothing awaits this release, so a failure or cancellation falls back to the sync release here.
with contextlib.suppress(asyncio.CancelledError, futures.CancelledError):
if pending.exception() is None:
return
with contextlib.suppress(Exception):
self._release_without_loop()
def _release_without_loop(self) -> None:
# No loop can run a release here, so call a sync backend directly as BaseFileLock.__del__ does; a coroutine
# backend cannot run and keeps its lock.
if iscoroutinefunction(self._release):
return
if self.is_locked:
try:
self._release_with_fork_tracking()
finally:
self._commit_release_if_released()
# A thread-local lock also holds for the threads that ran its loops, and nothing else releases those now.
self._release_thread_holds(exited_only=False)
def _new_transition_gate() -> _AsyncTransitionGate:
# Called per context rather than passed as the factory, so the lazy import resolves on first use, not at import.
return _AsyncTransitionGate()
@dataclass
class AsyncFileLockContext(FileLockContext):
"""A dataclass which holds the context for a ``BaseAsyncFileLock`` object."""
#: Whether run in executor
run_in_executor: bool = True
#: The executor
executor: futures.Executor | None = None
#: The loop
loop: asyncio.AbstractEventLoop | None = None
#: Serializes the tasks that transition this context. An acquire keeps it while it polls, so a thread-local lock
#: needs one per thread. A thread waiting for the lock must not keep the thread that holds it from releasing.
transition_gate: _AsyncTransitionGate = field(default_factory=_new_transition_gate)
class AsyncThreadLocalFileContext(AsyncFileLockContext, ThreadLocalFileContext):
"""A thread local version of the ``FileLockContext`` class."""
# threading.local runs __init__ again with these arguments the first time each thread touches the context.
def __init__( # ruff:ignore[too-many-arguments] # mirrors the AsyncFileLockContext fields set at construction
self,
*,
lock_file: str,
timeout: float,
mode: int,
blocking: bool,
poll_interval: float,
lifetime: float | None = None,
run_in_executor: bool = True,
executor: futures.Executor | None = None,
loop: asyncio.AbstractEventLoop | None = None,
) -> None:
self.current = AsyncFileLockContext(
lock_file=lock_file,
timeout=timeout,
mode=mode,
blocking=blocking,
poll_interval=poll_interval,
lifetime=lifetime,
run_in_executor=run_in_executor,
executor=executor,
loop=loop,
)
for _field in fields(AsyncFileLockContext):
setattr(AsyncThreadLocalFileContext, _field.name, _delegate_to_current(_field.name))
class AsyncAcquireReturnProxy:
"""A context-aware object that will release the lock file when exiting."""
def __init__(self, lock: BaseAsyncFileLock) -> None: # ruff:ignore[undocumented-public-init] # trivial release-on-exit proxy
self.lock = lock
async def __aenter__(self) -> BaseAsyncFileLock: # ruff:ignore[undocumented-magic-method] # returns the wrapped lock
return self.lock
async def __aexit__( # ruff:ignore[undocumented-magic-method] # releases the wrapped lock on exit
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
await self.lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # releases the wrapped lock's context
class AsyncSoftFileLock(SoftFileLock, BaseAsyncFileLock):
"""Simply watches the existence of the lock file."""
_lifetime_replacements: tuple[str, str] | None = ("AsyncStrictSoftFileLock", "AsyncSoftFileLease")
class AsyncStrictSoftFileLock(StrictSoftFileLock, BaseAsyncFileLock):
"""Run strict owner-claim locking without blocking the event loop."""
class AsyncSoftFileLease(SoftFileLease, BaseAsyncFileLock):
"""Existence lock whose claim expires, so a peer may take it while the previous holder still runs."""
class AsyncUnixFileLock(UnixFileLock, BaseAsyncFileLock):
"""Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems."""
class AsyncWindowsFileLock(WindowsFileLock, BaseAsyncFileLock):
"""Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems."""
__all__ = [
"AsyncAcquireReturnProxy",
"AsyncSoftFileLease",
"AsyncSoftFileLock",
"AsyncStrictSoftFileLock",
"AsyncUnixFileLock",
"AsyncWindowsFileLock",
"BaseAsyncFileLock",
]
def __dir__() -> list[str]:
# Hide the sync bases and helpers imported at top level.
return list(__all__)
Whitespace-only changes.
@@ -0,0 +1,24 @@
# file generated by vcs-versioning
# don't change, don't track in version control
from __future__ import annotations
__all__ = [
"__version__",
"__version_tuple__",
"version",
"version_tuple",
"__commit_id__",
"commit_id",
]
version: str
__version__: str
__version_tuple__: tuple[int | str, ...]
version_tuple: tuple[int | str, ...]
commit_id: str | None
__commit_id__: str | None
__version__ = version = '4.2.4'
__version_tuple__ = version_tuple = (4, 2, 4)
__commit_id__ = commit_id = None