Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,14 @@
|
||||
"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["filelock._soft_rw._async"]
|
||||
|
||||
from filelock._soft_rw._async import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock
|
||||
from filelock._soft_rw._sync import SoftReadWriteLock
|
||||
|
||||
__all__ = [
|
||||
"AsyncAcquireSoftReadWriteReturnProxy",
|
||||
"AsyncSoftReadWriteLock",
|
||||
"SoftReadWriteLock",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,373 @@
|
||||
"""Async wrapper around :class:`SoftReadWriteLock` for use with ``asyncio``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["asyncio", "filelock._async", "filelock._error", "functools", "threading"]
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import os
|
||||
import threading
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar
|
||||
|
||||
from filelock._async import (
|
||||
_TASKS_THREAD_ID,
|
||||
_BackendOutcome,
|
||||
_capture_call,
|
||||
_drain_future,
|
||||
_future_result,
|
||||
_raise_cancelled_error,
|
||||
_task_owners_for,
|
||||
_wait_until_done,
|
||||
)
|
||||
from filelock._error import Timeout
|
||||
from filelock._soft_rw._sync import SoftReadWriteLock, _lost_hold_error
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from concurrent import futures
|
||||
from types import TracebackType
|
||||
|
||||
from filelock._api import AcquireReturnProxy
|
||||
from filelock._lease import LeaseCompromise
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_R = TypeVar("_R")
|
||||
|
||||
|
||||
class AsyncSoftReadWriteLock:
|
||||
"""
|
||||
Async wrapper around :class:`SoftReadWriteLock` for ``asyncio`` applications.
|
||||
|
||||
The sync class's blocking filesystem operations run on a thread pool via ``loop.run_in_executor()``. Each
|
||||
``asyncio`` task owns its own hold. Tasks share a read lock, and a task asking for the write lock waits until the
|
||||
other holders release, with waiting writers ahead of new readers. Upgrading or downgrading a held lock raises
|
||||
:class:`RuntimeError`. The underlying :class:`SoftReadWriteLock` handles forks, the heartbeat, and stale eviction.
|
||||
Singleton wrappers share one sync lock and its task holds, and closing one ends only the holds taken through it.
|
||||
|
||||
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
|
||||
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
|
||||
:param is_singleton: if ``True``, reuse existing :class:`SoftReadWriteLock` instances per resolved path
|
||||
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
|
||||
:param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to
|
||||
``3 * heartbeat_interval``
|
||||
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
|
||||
:param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is
|
||||
lost
|
||||
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
|
||||
:param executor: executor for ``run_in_executor``; ``None`` uses the default executor
|
||||
|
||||
.. versionadded:: 3.27.0
|
||||
|
||||
"""
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
|
||||
self,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = True,
|
||||
heartbeat_interval: float = 30.0,
|
||||
stale_threshold: float | None = None,
|
||||
poll_interval: float = 0.25,
|
||||
on_compromise: Callable[[LeaseCompromise], None] | None = None,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
executor: futures.Executor | None = None,
|
||||
) -> None:
|
||||
self._creator_pid = os.getpid()
|
||||
self._closed = False
|
||||
self._owns_backend = not is_singleton
|
||||
#: Marks the task holds taken through this wrapper, which its close() ends.
|
||||
self._via: Final = object()
|
||||
self._lock = SoftReadWriteLock(
|
||||
lock_file,
|
||||
timeout,
|
||||
blocking=blocking,
|
||||
is_singleton=is_singleton,
|
||||
heartbeat_interval=heartbeat_interval,
|
||||
stale_threshold=stale_threshold,
|
||||
poll_interval=poll_interval,
|
||||
on_compromise=on_compromise,
|
||||
)
|
||||
self._owners = _task_owners_for(self._lock)
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
|
||||
@property
|
||||
def lock_file(self) -> str:
|
||||
"""The path to the lock file passed to the constructor."""
|
||||
return self._lock.lock_file
|
||||
|
||||
@property
|
||||
def timeout(self) -> float:
|
||||
"""The default timeout applied when ``acquire_read`` / ``acquire_write`` is called without one."""
|
||||
return self._lock.timeout
|
||||
|
||||
@property
|
||||
def blocking(self) -> bool:
|
||||
"""Whether ``acquire_*`` defaults to blocking; ``False`` makes contention raise immediately."""
|
||||
return self._lock.blocking
|
||||
|
||||
@property
|
||||
def generation(self) -> int | None:
|
||||
"""The generation the current hold was granted at, a fencing token; ``None`` when no lock is held."""
|
||||
return self._lock.generation
|
||||
|
||||
@property
|
||||
def compromise(self) -> LeaseCompromise | None:
|
||||
"""How the current hold was lost, or ``None`` while it stands; acquiring raises until the hold is released."""
|
||||
return self._lock.compromise
|
||||
|
||||
@property
|
||||
def loop(self) -> asyncio.AbstractEventLoop | None:
|
||||
"""The event loop used for ``run_in_executor``, or ``None`` for the running loop."""
|
||||
return self._loop
|
||||
|
||||
@property
|
||||
def executor(self) -> futures.Executor | None:
|
||||
"""The executor used for ``run_in_executor``, or ``None`` for the default executor."""
|
||||
return self._executor
|
||||
|
||||
@asynccontextmanager
|
||||
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
|
||||
"""
|
||||
Async context manager that acquires and releases a shared read lock.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the write lock
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
await self.acquire_read(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await self.release()
|
||||
|
||||
@asynccontextmanager
|
||||
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
|
||||
"""
|
||||
Async context manager that acquires and releases an exclusive write lock.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the read lock
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
await self.acquire_write(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await self.release()
|
||||
|
||||
async def acquire_read(
|
||||
self, timeout: float | None = None, *, blocking: bool | None = None
|
||||
) -> AsyncAcquireSoftReadWriteReturnProxy:
|
||||
"""
|
||||
Acquire a shared read lock.
|
||||
|
||||
See :meth:`SoftReadWriteLock.acquire_read` for reentrancy / upgrade / fork semantics. The blocking work runs
|
||||
inside ``run_in_executor`` so other coroutines on the same loop keep progressing while this call waits.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:returns: a proxy usable as an async context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the write lock, if this instance was invalidated by
|
||||
:func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self._raise_if_unusable()
|
||||
await self._acquire("read", timeout, blocking=blocking)
|
||||
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
|
||||
|
||||
async def acquire_write(
|
||||
self, timeout: float | None = None, *, blocking: bool | None = None
|
||||
) -> AsyncAcquireSoftReadWriteReturnProxy:
|
||||
"""
|
||||
Acquire an exclusive write lock.
|
||||
|
||||
See :meth:`SoftReadWriteLock.acquire_write` for the writer-preferring semantics. The blocking work runs inside
|
||||
``run_in_executor``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:returns: a proxy usable as an async context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the read lock, if this instance was invalidated by
|
||||
:func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self._raise_if_unusable()
|
||||
await self._acquire("write", timeout, blocking=blocking)
|
||||
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
|
||||
|
||||
async def _acquire(self, mode: Literal["read", "write"], timeout: float | None, *, blocking: bool | None) -> None:
|
||||
blocking = self._lock.blocking if blocking is None else blocking
|
||||
|
||||
async def enter(remaining: float) -> None:
|
||||
# A close() may have landed while this task waited for the table.
|
||||
self._raise_if_unusable()
|
||||
await self._run_acquire(
|
||||
functools.partial(
|
||||
self._lock._acquire_level_of, # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
_TASKS_THREAD_ID,
|
||||
mode,
|
||||
remaining,
|
||||
blocking=blocking,
|
||||
)
|
||||
)
|
||||
|
||||
def joinable() -> bool:
|
||||
self._raise_if_unusable()
|
||||
if (compromise := self._lock.compromise) is not None:
|
||||
raise _lost_hold_error(self.lock_file, mode, compromise)
|
||||
return self._backend_holds()
|
||||
|
||||
await self._owners.acquire(
|
||||
mode,
|
||||
timeout=self._lock.timeout if timeout is None else timeout,
|
||||
blocking=blocking,
|
||||
lock_file=self.lock_file,
|
||||
via=self._via,
|
||||
enter=enter,
|
||||
joinable=joinable,
|
||||
)
|
||||
|
||||
async def release(self, *, force: bool = False) -> None:
|
||||
"""
|
||||
Release one level of the calling task's hold.
|
||||
|
||||
The wrapper releases the lock once no task holds it.
|
||||
|
||||
:param force: if ``True``, end every level the calling task took through this wrapper. Levels taken through
|
||||
other wrappers on the same path stay. A wrapper that holds no level for the task releases nothing.
|
||||
|
||||
:raises RuntimeError: if the calling task holds no level through this wrapper and *force* is ``False``
|
||||
|
||||
"""
|
||||
if self._creator_pid == os.getpid():
|
||||
await self._owners.release(
|
||||
force=force,
|
||||
lock_file=self.lock_file,
|
||||
via=self._via,
|
||||
leave=functools.partial(self._submit, self._leave_backend),
|
||||
still_held=self._backend_holds,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""
|
||||
End the holds of the tasks that acquired through this instance and release the lock once no task holds it.
|
||||
|
||||
Idempotent. After calling this method the instance can no longer acquire locks. With ``is_singleton=False`` it
|
||||
also closes the underlying :class:`SoftReadWriteLock`. A singleton wrapper leaves the shared one open, so tasks
|
||||
that acquired through another wrapper and synchronous users of the same path keep their holds. The next acquire
|
||||
notices a synchronous ``close()`` or forced release of that shared lock. One that lands in the few bytecodes
|
||||
between that check and the admission still admits the task into the ended hold.
|
||||
|
||||
"""
|
||||
if self._creator_pid != os.getpid() or self._closed:
|
||||
return
|
||||
# Refuse acquires from here on, so none is admitted into a hold the backend is letting go of.
|
||||
self._closed = True
|
||||
try:
|
||||
await self._owners.retire(
|
||||
None if self._owns_backend else self._via,
|
||||
leave=functools.partial(self._submit, self._lock.close if self._owns_backend else self._leave_backend),
|
||||
still_held=self._backend_holds,
|
||||
)
|
||||
except BaseException as error:
|
||||
# Only a failed backend call can leave the level held, and then a retried close() has to find it.
|
||||
if not isinstance(error, asyncio.CancelledError) and self._backend_holds():
|
||||
self._closed = False
|
||||
raise
|
||||
|
||||
def _raise_if_unusable(self) -> None:
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
msg = f"AsyncSoftReadWriteLock on {self.lock_file} was inherited across fork; construct a new instance"
|
||||
raise RuntimeError(msg)
|
||||
if self._closed:
|
||||
msg = f"AsyncSoftReadWriteLock on {self.lock_file} has been closed"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
def _backend_holds(self) -> bool:
|
||||
# A sync close() or forced release on a shared singleton ends the tasks' level without the table taking part.
|
||||
return self._lock._holds_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
|
||||
def _leave_backend(self) -> None:
|
||||
# Only the tasks' level: a sync user of the same singleton keeps its own.
|
||||
self._lock._release_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
|
||||
async def _run_acquire(self, acquire: Callable[..., AcquireReturnProxy]) -> None:
|
||||
# run_in_executor cannot recall work the pool already started. A sync acquire that won the lock before it saw
|
||||
# the cancellation has published its claim and started the heartbeat, which keeps the record fresh so no peer
|
||||
# evicts it. Wait the call out and hand the claim back, as AsyncReadWriteLock does.
|
||||
abandoned = threading.Event()
|
||||
acquire_future = self._submit(acquire, cancel_check=abandoned.is_set)
|
||||
try:
|
||||
await _wait_until_done(acquire_future)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
# Stops a backend wait on another process's hold, which would otherwise outlast the caller by as long.
|
||||
abandoned.set()
|
||||
try:
|
||||
await _drain_future(acquire_future)
|
||||
except Timeout:
|
||||
# Nothing was taken, so the caller's cancellation is the whole outcome.
|
||||
raise cancellation from None
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
_raise_cancelled_error(cancellation, error)
|
||||
try:
|
||||
await _drain_future(self._submit(self._leave_backend))
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
_raise_cancelled_error(cancellation, error)
|
||||
raise
|
||||
_future_result(acquire_future)
|
||||
|
||||
def _submit(
|
||||
self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> asyncio.Future[_BackendOutcome[_R]]:
|
||||
loop = self._loop or asyncio.get_running_loop()
|
||||
return loop.run_in_executor(self._executor, _capture_call, functools.partial(func, *args, **kwargs))
|
||||
|
||||
|
||||
class AsyncAcquireSoftReadWriteReturnProxy:
|
||||
"""Async context-aware object that releases an :class:`AsyncSoftReadWriteLock` on exit."""
|
||||
|
||||
def __init__(self, lock: AsyncSoftReadWriteLock) -> None:
|
||||
self.lock = lock
|
||||
|
||||
async def __aenter__(self) -> AsyncSoftReadWriteLock:
|
||||
return self.lock
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
await self.lock.release()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AsyncAcquireSoftReadWriteReturnProxy",
|
||||
"AsyncSoftReadWriteLock",
|
||||
]
|
||||
@@ -0,0 +1,638 @@
|
||||
"""
|
||||
The generation-log protocol behind :class:`~filelock.SoftReadWriteLock`.
|
||||
|
||||
The lock's whole state is a sequence of immutable snapshots, ``gen/<N>``, each naming the writer and the readers that
|
||||
hold the lock at generation ``N``. Every transition is one atomic publication of the next snapshot: a participant reads
|
||||
the latest generation, derives the successor it wants, and links a fully written temporary file to ``gen/<N+1>``. The
|
||||
link fails if a peer got there first, so the participant re-reads and tries again. No step is ever half done: a crash
|
||||
leaves at most an unlinked temporary file, never a state that another participant must repair before it can proceed.
|
||||
|
||||
Liveness rests on a nonce, not a clock. Each participant keeps ``holders/<token>`` and rewrites its nonce every
|
||||
heartbeat. A contender records the bytes it read there and the moment it read them on its own monotonic clock; a
|
||||
member whose bytes have not changed for ``stale_threshold`` is evicted by committing a snapshot without it. Two hosts
|
||||
never compare clocks, so a skewed file server or a suspended virtual machine cannot make a live holder look dead.
|
||||
|
||||
The generation number is monotonic and unique to one transition within one incarnation of the log, so the generation at
|
||||
which a hold was granted fences it: a resource that rejects anything carrying a lower generation than the highest it has
|
||||
accepted refuses a holder that paused past its expiry and resumed. A random epoch file names the incarnation, so a
|
||||
participant that outlives a removal of the log reads the new one instead of continuing the old numbering into it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["contextlib", "filelock._error", "filelock._identity", "math", "pathlib", "secrets"]
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol
|
||||
|
||||
from filelock._error import SoftFileLockProtocolError
|
||||
from filelock._identity import host_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
Mode = Literal["read", "write"]
|
||||
HeartbeatOutcome = Literal["ok", "lost", "transient"]
|
||||
|
||||
_PROTOCOL: Final[str] = "filelock-rw/1"
|
||||
_GENERATION_DIGITS: Final[int] = 20
|
||||
_TOKEN_HEX_LENGTH: Final[int] = 32
|
||||
#: A snapshot line is ``reader=`` plus a token, so this bounds a hostile record without capping real reader counts.
|
||||
_MAX_RECORD_SIZE: Final[int] = 1 << 20
|
||||
#: Snapshots older than this many generations behind the latest are removed by whoever commits. The window is also how
|
||||
#: far ``latest`` probes past a gap before it trusts that nothing newer exists; ``gen/HEAD``, not the window, is what
|
||||
#: keeps a listing served further behind from passing an old generation off as the head.
|
||||
_RETAINED_GENERATIONS: Final[int] = 64
|
||||
_GENERATIONS_DIRECTORY: Final[str] = "gen"
|
||||
#: A random token created once per log: removing ``gen/`` removes it, so a participant can tell a log that was wiped and
|
||||
#: restarted from the one it remembers.
|
||||
_EPOCH_NAME: Final[str] = "epoch"
|
||||
#: The epoch plus a copy of the newest snapshot, renamed into place after each commit. NFS revalidates a file opened by
|
||||
#: name but may serve a listing from cache, so this is the head a stale client cannot miss; it may lag, never lead.
|
||||
_HEAD_NAME: Final[str] = "HEAD"
|
||||
_HOLDERS_DIRECTORY: Final[str] = "holders"
|
||||
_TEMPORARY_PREFIX: Final[str] = ".commit-"
|
||||
|
||||
|
||||
class TreeReplacedError(RuntimeError):
|
||||
"""
|
||||
Raised by a ``Files`` implementation when a protocol directory is not a real directory (a symlink, a junction, a
|
||||
file) or belongs to another account.
|
||||
"""
|
||||
|
||||
|
||||
class Files(Protocol):
|
||||
"""The filesystem operations the protocol runs on; :mod:`._storage` implements them over :mod:`os`."""
|
||||
|
||||
def read(self, path: str) -> bytes | None:
|
||||
"""The file's bytes, ``None`` when it does not exist, or ``b""`` when it is not a regular file."""
|
||||
|
||||
def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]:
|
||||
"""
|
||||
Create the file exclusively with the whole record written; raise ``FileExistsError`` when it exists.
|
||||
|
||||
The returned device and inode (the kernel's identity for a file, independent of its name) let ``overwrite``
|
||||
check that the name still leads to this file.
|
||||
|
||||
*durable* flushes it to stable storage first, for a record that outlives the host that wrote it: a generation a
|
||||
crash left empty would wedge every other participant. A holder record dies with its writer, so it skips that.
|
||||
"""
|
||||
|
||||
def link(self, source: str, target: str) -> bool:
|
||||
"""Hard-link *source* to *target* without replacing; ``True`` when *target* names *source*'s file afterwards."""
|
||||
|
||||
def replace(self, source: str, target: str) -> bool:
|
||||
"""Rename *source* over *target* atomically; ``False`` when the platform refused because *target* is in use."""
|
||||
|
||||
def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool:
|
||||
"""
|
||||
Rewrite from offset zero the file ``create`` returned *identity* for.
|
||||
|
||||
``False`` when *path* no longer leads to that file.
|
||||
"""
|
||||
|
||||
def unlink(self, path: str) -> None:
|
||||
"""Remove the file, tolerating one that is already gone."""
|
||||
|
||||
def listdir(self, path: str) -> list[str]:
|
||||
"""The names in the directory, or an empty list when it does not exist."""
|
||||
|
||||
def prepare(self, root: str, *, force: bool = False) -> None:
|
||||
"""
|
||||
Create the protocol directories under *root*, refusing anything at those paths that is not a directory.
|
||||
|
||||
Done once per lock; *force* repeats it after a write found a directory missing.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Snapshot:
|
||||
"""The lock's state at one generation: which token writes, which tokens read."""
|
||||
|
||||
generation: int
|
||||
writer: str | None
|
||||
readers: frozenset[str]
|
||||
|
||||
@property
|
||||
def members(self) -> frozenset[str]:
|
||||
"""Every token this generation names."""
|
||||
return self.readers if self.writer is None else self.readers | {self.writer}
|
||||
|
||||
def encode(self) -> bytes:
|
||||
lines = [_PROTOCOL, f"generation={self.generation}"]
|
||||
if self.writer is not None:
|
||||
lines.append(f"writer={self.writer}")
|
||||
lines.extend(f"reader={token}" for token in sorted(self.readers))
|
||||
return "".join(f"{line}\n" for line in lines).encode("ascii")
|
||||
|
||||
|
||||
def parse_snapshot(data: bytes) -> Snapshot | None:
|
||||
"""The snapshot a ``gen/<N>`` record encodes, or ``None`` when the record is malformed."""
|
||||
if not data or len(data) > _MAX_RECORD_SIZE:
|
||||
return None
|
||||
try:
|
||||
lines = data.decode("ascii").split("\n")
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
if lines[0] != _PROTOCOL or lines[-1]:
|
||||
return None
|
||||
generation: int | None = None
|
||||
writer: str | None = None
|
||||
readers: set[str] = set()
|
||||
for line in lines[1:-1]:
|
||||
key, _, value = line.partition("=")
|
||||
if (
|
||||
key == "generation"
|
||||
and generation is None
|
||||
and len(value) <= _GENERATION_DIGITS
|
||||
and value.isdigit()
|
||||
and str(int(value)) == value
|
||||
):
|
||||
generation = int(value)
|
||||
elif key == "writer" and writer is None and is_token(value):
|
||||
writer = value
|
||||
elif key == "reader" and is_token(value) and value not in readers:
|
||||
readers.add(value)
|
||||
else:
|
||||
return None
|
||||
return None if generation is None else Snapshot(generation=generation, writer=writer, readers=frozenset(readers))
|
||||
|
||||
|
||||
def is_token(value: str) -> bool:
|
||||
"""Whether *value* is a claim token: exactly 32 lowercase hex digits."""
|
||||
return len(value) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in value)
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
"""A fresh claim token."""
|
||||
return secrets.token_hex(_TOKEN_HEX_LENGTH // 2)
|
||||
|
||||
|
||||
def encode_holder(token: str, nonce: str, lease: float) -> bytes:
|
||||
"""
|
||||
The ``holders/<token>`` record; only the nonce changes between heartbeats, so the length stays constant.
|
||||
|
||||
*lease* is the writer's own ``stale_threshold``: a peer configured tighter must not read this holder's longer
|
||||
heartbeat interval as death, so evictors wait the longer of their threshold and this one.
|
||||
"""
|
||||
return (
|
||||
f"{_PROTOCOL}\ntoken={token}\npid={os.getpid()}\nhost={host_name()}\nlease={float(lease)!r}\nnonce={nonce}\n"
|
||||
).encode("ascii")
|
||||
|
||||
|
||||
def holder_lease(record: bytes | None) -> float | None:
|
||||
"""The ``lease`` a holder record carries, or ``None`` when it has none (an older release) or it is malformed."""
|
||||
try:
|
||||
text = next(
|
||||
line for line in (record or b"").decode("ascii").split("\n") if line.startswith("lease=")
|
||||
).removeprefix("lease=")
|
||||
lease = float(text)
|
||||
except (StopIteration, ValueError): # UnicodeDecodeError is a ValueError
|
||||
return None
|
||||
# Only the exact spelling repr() writes: float() alone also takes "1_000", " 1", and non-ASCII digits.
|
||||
return lease if isfinite(lease) and lease > 0 and repr(lease) == text else None
|
||||
|
||||
|
||||
class Ledger:
|
||||
"""
|
||||
How long each observed value has stayed the same, measured on this observer's monotonic clock.
|
||||
|
||||
A value that has not changed for ``stale_threshold`` belongs to a participant that stopped refreshing it. The clock
|
||||
is the observer's own and only ever measures the gap between two of its own observations, which is what lets two
|
||||
hosts agree on liveness without agreeing on the time.
|
||||
"""
|
||||
|
||||
def __init__(self, clock: Callable[[], float]) -> None:
|
||||
self._clock = clock
|
||||
self._seen: dict[str, tuple[bytes | None, float]] = {}
|
||||
|
||||
def observe(self, key: str, value: bytes | None, *, asked: float | None = None) -> float:
|
||||
"""
|
||||
Record *value* for *key* and return for how long it has stayed the same.
|
||||
|
||||
*asked* is the clock reading taken before the caller fetched *value*. An unchanged value counts as unchanged up
|
||||
to *asked*, so a pause between the fetch and this call adds no time.
|
||||
"""
|
||||
now = self._clock()
|
||||
if (seen := self._seen.get(key)) is None or seen[0] != value:
|
||||
self._seen[key] = (value, now)
|
||||
return 0.0
|
||||
return (now if asked is None else asked) - seen[1]
|
||||
|
||||
def forget(self, key: str) -> None:
|
||||
self._seen.pop(key, None)
|
||||
|
||||
def retain(self, keys: set[str]) -> None:
|
||||
"""Drop every key not in *keys*: peers that left would otherwise stay in the ledger for the lock's lifetime."""
|
||||
self._seen = {key: seen for key, seen in self._seen.items() if key in keys}
|
||||
|
||||
|
||||
class GenerationLog:
|
||||
"""The ordered snapshots under ``gen/``, found by listing and probing, advanced by linking the next one."""
|
||||
|
||||
def __init__(self, files: Files, lock_file: str, root: str) -> None:
|
||||
self._files = files
|
||||
self._lock_file = lock_file
|
||||
self._directory = Path(root, _GENERATIONS_DIRECTORY)
|
||||
self._epoch_path = str(self._directory / _EPOCH_NAME)
|
||||
self._head_path = str(self._directory / _HEAD_NAME)
|
||||
self._epoch: bytes | None = None
|
||||
self._latest: Snapshot | None = None
|
||||
|
||||
def latest(self) -> Snapshot:
|
||||
"""
|
||||
The newest snapshot.
|
||||
|
||||
Starts from the newest readable generation among those ``gen/HEAD``, the listing, or memory name, then probes
|
||||
forward by name across the retained window. ``HEAD`` only names a generation, and that generation must still
|
||||
read back: a committer stalled before its rename can leave ``HEAD`` on a generation compacted since. A
|
||||
generation compacted between the listing and the read, and a hole left by a committer that died before it
|
||||
compacted, are covered by the probe. Memory and ``HEAD`` are bound to the log's epoch, so a log wiped and
|
||||
restarted is never continued from a generation it no longer holds. Only a log that names generations, now or as
|
||||
remembered, of which none can be read is refused, because that is a client so far behind the log that nothing
|
||||
it reads can be trusted.
|
||||
"""
|
||||
# When everything the first pass named is gone, peers compacted past it while this client looked, or the log was
|
||||
# replaced; one more pass settles whether this client cannot see the head at all.
|
||||
if (latest := self._find()) is None and (latest := self._find()) is None:
|
||||
raise SoftFileLockProtocolError(self._lock_file, None, "generation log names no readable snapshot")
|
||||
self._latest = latest
|
||||
return latest
|
||||
|
||||
def _find(self) -> Snapshot | None:
|
||||
if (epoch := self._incarnation()) != self._epoch:
|
||||
self._epoch, self._latest = epoch, None
|
||||
named = set(self._list())
|
||||
# Generation 0 is the empty log, which has no record to read back.
|
||||
if self._latest is not None and self._latest.generation:
|
||||
named.add(self._latest.generation)
|
||||
if head := self._head_generation():
|
||||
named.add(head)
|
||||
for generation in sorted(named, reverse=True):
|
||||
if (start := self._read(generation)) is not None:
|
||||
# Compaction keeps the surviving generations contiguous, so the first missing name past a readable HEAD
|
||||
# generation is the end; probing the whole window on every poll cost 64 failed opens.
|
||||
return self._probe(start, patience=1 if generation == head else _RETAINED_GENERATIONS)
|
||||
newest = max(named, default=0)
|
||||
found = self._probe(Snapshot(generation=newest, writer=None, readers=frozenset()))
|
||||
return found if found.generation > newest or not named else None
|
||||
|
||||
def _head_generation(self) -> int | None:
|
||||
# HEAD is never flushed, so a crash can leave it unreadable; readers verify what it names anyway, so a torn one
|
||||
# only sends them to the window probe.
|
||||
if (record := self._head_record()) is None or (snapshot := parse_snapshot(record)) is None:
|
||||
return None
|
||||
return snapshot.generation
|
||||
|
||||
def _head_record(self) -> bytes | None:
|
||||
if (data := self._files.read(self._head_path)) is None:
|
||||
return None
|
||||
# A HEAD from another epoch landed after its log was removed; a log whose epoch was never written runs without.
|
||||
epoch, newline, record = data.partition(b"\n")
|
||||
return record if self._valid_epoch() == epoch + newline else None
|
||||
|
||||
def _valid_epoch(self) -> bytes | None:
|
||||
epoch = self._epoch
|
||||
return epoch if epoch is not None and is_token(epoch.decode("ascii", "replace").removesuffix("\n")) else None
|
||||
|
||||
def _incarnation(self) -> bytes | None:
|
||||
if (epoch := self._files.read(self._epoch_path)) is None:
|
||||
# Exclusive creation, not a link: reading the log must keep working where os.link does not exist. A peer
|
||||
# that reads the file before its token is written takes the empty value for the epoch, and its next commit
|
||||
# sees the change and re-reads, so a partial read costs one retry.
|
||||
try:
|
||||
self._files.create(self._epoch_path, f"{new_token()}\n".encode("ascii"), durable=True)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
# No gen/ yet: there is no log to tell apart, and the first participant's prepare() creates it.
|
||||
return None
|
||||
epoch = self._files.read(self._epoch_path)
|
||||
return epoch
|
||||
|
||||
def _list(self) -> list[int]:
|
||||
listed = self._listed()
|
||||
for generation in listed[:-_RETAINED_GENERATIONS]:
|
||||
self._files.unlink(self._path(generation))
|
||||
return listed[-_RETAINED_GENERATIONS:]
|
||||
|
||||
def _listed(self) -> list[int]:
|
||||
# isdigit() alone admits Unicode digits: superscripts int() rejects, fullwidth ones it parses to another number.
|
||||
return sorted(
|
||||
int(name)
|
||||
for name in self._files.listdir(str(self._directory))
|
||||
if len(name) == _GENERATION_DIGITS and name.isascii() and name.isdigit()
|
||||
)
|
||||
|
||||
def _probe(self, start: Snapshot, patience: int = _RETAINED_GENERATIONS) -> Snapshot:
|
||||
latest = start
|
||||
generation = start.generation
|
||||
gap = 0
|
||||
while gap < patience:
|
||||
generation += 1
|
||||
if (snapshot := self._read(generation)) is None:
|
||||
gap += 1
|
||||
continue
|
||||
latest = snapshot
|
||||
gap = 0
|
||||
return latest
|
||||
|
||||
def _read(self, generation: int) -> Snapshot | None:
|
||||
if (data := self._files.read(self._path(generation))) is None:
|
||||
return None
|
||||
if (snapshot := parse_snapshot(data)) is None or snapshot.generation != generation:
|
||||
name = self._name(generation)
|
||||
raise SoftFileLockProtocolError(self._lock_file, name, "malformed generation record")
|
||||
return snapshot
|
||||
|
||||
def commit(self, successor: Snapshot) -> bool:
|
||||
"""
|
||||
Publish *successor* as the next generation; ``False`` when a peer published that generation first.
|
||||
|
||||
The record is complete before it gets its public name, and a no-replace hard link is the compare-and-swap: only
|
||||
one link to ``gen/<N+1>`` can ever succeed. The link's return code is not trusted, because an NFS retry can
|
||||
report a link that did land as failed; the file's identity after the call is what decides. A log whose epoch
|
||||
changed since the snapshot was read is a different log, so the commit is refused as lost and the caller
|
||||
re-reads; that narrows a wipe racing this commit to the moment between the epoch check and the link. A slot
|
||||
compaction freed is refused the same way, before the link and again after it, since a committer can stall in
|
||||
between. After the link a surviving predecessor proves little: it may be a record another stalled committer
|
||||
linked into its own freed slot and has yet to remove. A slot comes free once the log is a whole window past it,
|
||||
and the head stays, so a record the listing or ``HEAD`` leads by a window is refused as well. A record refused
|
||||
after it landed is removed when nothing was built on it. One that peers lead by less than a window stays, or its
|
||||
slot would come free early.
|
||||
"""
|
||||
if self._files.read(self._epoch_path) != self._epoch or not self._extends_the_log(successor.generation):
|
||||
return False
|
||||
if not self._publish(
|
||||
self._path(successor.generation), data := successor.encode(), self._files.link, durable=True
|
||||
):
|
||||
return False
|
||||
lead = max([*self._listed(), self._head_generation() or 0]) - successor.generation
|
||||
if lead >= _RETAINED_GENERATIONS or not self._extends_the_log(successor.generation):
|
||||
if not 0 < lead < _RETAINED_GENERATIONS:
|
||||
self._files.unlink(self._path(successor.generation))
|
||||
return False
|
||||
self._latest = successor
|
||||
# Readers verify the generation HEAD names, so a failed rename costs a probe, not the commit. A late one would
|
||||
# move HEAD back past later commits, so HEAD only ever moves forward.
|
||||
with suppress(OSError):
|
||||
if (
|
||||
(epoch := self._valid_epoch()) is not None
|
||||
and self._files.read(self._epoch_path) == epoch
|
||||
and (self._head_generation() or 0) < successor.generation
|
||||
):
|
||||
self._publish(self._head_path, epoch + data, self._files.replace, durable=False)
|
||||
if (expired := successor.generation - _RETAINED_GENERATIONS - 1) > 0:
|
||||
self._compact_through(expired)
|
||||
return True
|
||||
|
||||
def _extends_the_log(self, generation: int) -> bool:
|
||||
# Compaction removes oldest first, so a missing predecessor means compaction freed this slot. The first
|
||||
# generation has none; a HEAD of this epoch means the log already moved past it.
|
||||
if generation > 1:
|
||||
return self._files.read(self._path(generation - 1)) is not None
|
||||
return self._head_record() is None
|
||||
|
||||
def _publish(self, target: str, data: bytes, place: Callable[[str, str], bool], *, durable: bool) -> bool:
|
||||
temporary = str(self._directory / f"{_TEMPORARY_PREFIX}{new_token()}")
|
||||
try:
|
||||
self._files.create(temporary, data, durable=durable)
|
||||
except FileNotFoundError:
|
||||
# No gen/: the log was removed, or nobody has prepared it yet. Either way there is nothing to publish into
|
||||
# until a participant's prepare() creates it, and the caller re-reads whatever log exists by then.
|
||||
return False
|
||||
try:
|
||||
return place(temporary, target)
|
||||
finally:
|
||||
self._files.unlink(temporary)
|
||||
|
||||
def _compact_through(self, expired: int) -> None:
|
||||
# Oldest first, so survivors stay contiguous: a stalled committer's leftover would otherwise sit below a hole.
|
||||
oldest = expired
|
||||
while oldest > 1 and self._files.read(self._path(oldest - 1)) is not None:
|
||||
oldest -= 1
|
||||
for generation in range(oldest, expired + 1):
|
||||
self._files.unlink(self._path(generation))
|
||||
|
||||
def _path(self, generation: int) -> str:
|
||||
return str(self._directory / self._name(generation))
|
||||
|
||||
@staticmethod
|
||||
def _name(generation: int) -> str:
|
||||
return f"{generation:0{_GENERATION_DIGITS}d}"
|
||||
|
||||
|
||||
class Participant:
|
||||
"""
|
||||
One acquisition, from publishing a claim through holding the lock to leaving.
|
||||
|
||||
``advance`` is polled until it reports the lock granted, ``heartbeat`` runs on the holder's schedule, and ``leave``
|
||||
commits the participant out. None of them sleep: the caller owns the deadline and the poll interval. The ledger and
|
||||
the log outlive one acquisition: how long a peer's record has stayed unchanged is knowledge every attempt against
|
||||
the same lock shares, or a caller whose attempts are each shorter than the stale threshold could never evict.
|
||||
"""
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # one value per protocol input; the lock builds this once per acquisition
|
||||
self,
|
||||
files: Files,
|
||||
root: str,
|
||||
mode: Mode,
|
||||
*,
|
||||
stale_threshold: float,
|
||||
clock: Callable[[], float],
|
||||
ledger: Ledger,
|
||||
log: GenerationLog,
|
||||
) -> None:
|
||||
self.token: Final[str] = new_token()
|
||||
self.mode: Final[Mode] = mode
|
||||
self.generation: int | None = None
|
||||
self._files = files
|
||||
self._root = root
|
||||
self._stale_threshold = stale_threshold
|
||||
self._clock = clock
|
||||
self._ledger = ledger
|
||||
self._log = log
|
||||
self._holder = self._holder_path(self.token)
|
||||
self._holder_identity: tuple[int, int] | None = None
|
||||
self._entered = False
|
||||
self._next_sweep = clock()
|
||||
|
||||
def publish(self) -> None:
|
||||
"""Create the holder record peers watch for liveness; it exists before the token appears in any snapshot."""
|
||||
self._files.prepare(self._root)
|
||||
self._create_holder()
|
||||
|
||||
def _create_holder(self) -> None:
|
||||
try:
|
||||
self._holder_identity = self._files.create(self._holder, self._record())
|
||||
except FileNotFoundError:
|
||||
# Removing the whole log takes the directories with it.
|
||||
self._files.prepare(self._root, force=True)
|
||||
self._holder_identity = self._files.create(self._holder, self._record())
|
||||
|
||||
def _record(self) -> bytes:
|
||||
return encode_holder(self.token, new_token(), self._stale_threshold)
|
||||
|
||||
def advance(self) -> bool:
|
||||
"""
|
||||
Take the next step toward holding the lock; ``True`` once the lock is granted.
|
||||
|
||||
A reader enters as soon as no live writer is named. A writer enters as soon as no live writer is named, which
|
||||
blocks every new reader, then waits for the named readers to leave. Stale members are evicted in the same commit
|
||||
that admits this participant, and a writer's final admission is itself a commit, so no admission can rest on a
|
||||
snapshot a peer is about to supersede.
|
||||
"""
|
||||
# A lost race is not a reason to wait: re-read and try again before handing the poll interval back to the
|
||||
# caller.
|
||||
for _ in range(4):
|
||||
latest = self._log.latest()
|
||||
self._keep_claim_fresh(latest)
|
||||
self._sweep(latest)
|
||||
stale = frozenset(token for token in latest.members - {self.token} if self._is_stale(token))
|
||||
writer = None if latest.writer in stale else latest.writer
|
||||
readers = latest.readers - stale
|
||||
if not self._entered:
|
||||
if writer is not None:
|
||||
# A live writer blocks entry in both modes; only an eviction is worth committing.
|
||||
if stale:
|
||||
self._commit(latest, writer, readers, stale)
|
||||
return False
|
||||
writer, readers = (self.token, readers) if self.mode == "write" else (None, readers | {self.token})
|
||||
granted = self.mode == "read" or not readers
|
||||
if self._entered and not stale and not granted:
|
||||
return False
|
||||
if (generation := self._commit(latest, writer, readers, stale)) is None:
|
||||
continue
|
||||
self._entered = True
|
||||
if granted:
|
||||
self.generation = generation
|
||||
return granted
|
||||
return False
|
||||
|
||||
def _keep_claim_fresh(self, latest: Snapshot) -> None:
|
||||
# The snapshot is the truth about membership: a commit the filesystem reported lost may have landed, and a peer
|
||||
# may have evicted this participant while it waited to drain, so both directions are taken from it. A wait can
|
||||
# outlast the stale threshold, so the nonce changes on every poll; a peer then never reads a patient contender
|
||||
# as a corpse. A record that has gone missing was taken by an evictor or a sweeper that did, or by a removal of
|
||||
# the whole log.
|
||||
self._entered = self.token in latest.members
|
||||
if not self._refresh_nonce():
|
||||
with suppress(FileExistsError):
|
||||
self._create_holder()
|
||||
|
||||
def _sweep(self, latest: Snapshot) -> None:
|
||||
# A participant that crashed before its first commit leaves a record no snapshot names, and a commit that
|
||||
# crashed between creating and unlinking its temporary file leaves that too. Neither blocks anyone, so they are
|
||||
# collected on the same unchanged-for-a-threshold rule as a member, and only every half threshold at most.
|
||||
if (now := self._clock()) < self._next_sweep:
|
||||
return
|
||||
self._next_sweep = now + self._stale_threshold / 2
|
||||
watched = set(latest.members)
|
||||
holders = Path(self._root, _HOLDERS_DIRECTORY)
|
||||
for name in self._files.listdir(str(holders)):
|
||||
if name != self.token and name not in latest.members and is_token(name):
|
||||
watched.add(key := f"holder:{name}")
|
||||
self._collect(key, str(holders / name))
|
||||
generations = Path(self._root, _GENERATIONS_DIRECTORY)
|
||||
for name in self._files.listdir(str(generations)):
|
||||
if name.startswith(_TEMPORARY_PREFIX):
|
||||
watched.add(key := f"temporary:{name}")
|
||||
self._collect(key, str(generations / name))
|
||||
self._ledger.retain(watched)
|
||||
|
||||
def _collect(self, key: str, path: str) -> None:
|
||||
if self._unchanged_for_its_lease(key, path):
|
||||
self._files.unlink(path)
|
||||
self._ledger.forget(key)
|
||||
|
||||
def _commit(
|
||||
self, latest: Snapshot, writer: str | None, readers: frozenset[str], stale: frozenset[str]
|
||||
) -> int | None:
|
||||
successor = Snapshot(generation=latest.generation + 1, writer=writer, readers=readers)
|
||||
if not self._log.commit(successor):
|
||||
return None
|
||||
for token in stale:
|
||||
self._files.unlink(self._holder_path(token))
|
||||
self._ledger.forget(token)
|
||||
return successor.generation
|
||||
|
||||
def _is_stale(self, token: str) -> bool:
|
||||
return self._unchanged_for_its_lease(token, self._holder_path(token))
|
||||
|
||||
def _unchanged_for_its_lease(self, key: str, path: str) -> bool:
|
||||
asked = self._clock()
|
||||
record = self._files.read(path)
|
||||
return self._ledger.observe(key, record, asked=asked) >= max(self._stale_threshold, holder_lease(record) or 0.0)
|
||||
|
||||
def _holder_path(self, token: str) -> str:
|
||||
return str(Path(self._root, _HOLDERS_DIRECTORY, token))
|
||||
|
||||
def _refresh_nonce(self) -> bool:
|
||||
if (identity := self._holder_identity) is None:
|
||||
return False
|
||||
return self._files.overwrite(self._holder, self._record(), identity)
|
||||
|
||||
def heartbeat(self) -> tuple[HeartbeatOutcome, OSError | None]:
|
||||
"""
|
||||
Refresh the nonce and confirm the latest snapshot still names this participant.
|
||||
|
||||
``lost`` means a peer evicted this participant or removed its holder record: the lock is no longer held, and the
|
||||
holder must stop using what it protects. ``transient`` is a filesystem error refreshing the nonce, worth
|
||||
retrying on the next tick. A failure to read the log or to sweep decides nothing about this holder's liveness,
|
||||
so it is not counted against it.
|
||||
"""
|
||||
try:
|
||||
if not self._refresh_nonce():
|
||||
return "lost", None
|
||||
except TreeReplacedError:
|
||||
return "lost", None
|
||||
except OSError as error:
|
||||
return "transient", error
|
||||
try:
|
||||
latest = self._log.latest()
|
||||
self._sweep(latest)
|
||||
except (OSError, TreeReplacedError):
|
||||
return "ok", None
|
||||
return ("ok" if self.token in latest.members else "lost"), None
|
||||
|
||||
def leave(self) -> None:
|
||||
"""Commit this participant out of the latest snapshot, then remove its holder record."""
|
||||
# The record goes even when the log cannot be read: a contender that never entered then leaves nothing behind,
|
||||
# and a member still named ages out the same way with or without one.
|
||||
# A tree that was swapped out leaves no record this participant can reach, so there is nothing to commit out of.
|
||||
with suppress(TreeReplacedError):
|
||||
try:
|
||||
while self.token in (latest := self._log.latest()).members:
|
||||
successor = Snapshot(
|
||||
generation=latest.generation + 1,
|
||||
writer=None if latest.writer == self.token else latest.writer,
|
||||
readers=latest.readers - {self.token},
|
||||
)
|
||||
if self._log.commit(successor):
|
||||
break
|
||||
finally:
|
||||
self._entered = False
|
||||
self.generation = None
|
||||
self._files.unlink(self._holder)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Files",
|
||||
"GenerationLog",
|
||||
"HeartbeatOutcome",
|
||||
"Ledger",
|
||||
"Mode",
|
||||
"Participant",
|
||||
"Snapshot",
|
||||
"TreeReplacedError",
|
||||
"encode_holder",
|
||||
"holder_lease",
|
||||
"is_token",
|
||||
"new_token",
|
||||
"parse_snapshot",
|
||||
]
|
||||
@@ -0,0 +1,278 @@
|
||||
"""The :class:`~._protocol.Files` implementation over :mod:`os`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["filelock._soft_rw._protocol", "filelock._strict", "filelock._util", "pathlib"]
|
||||
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager, suppress
|
||||
from errno import ELOOP, ENOENT, ENOTDIR, ENXIO, ESTALE
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from filelock._soft_rw._protocol import (
|
||||
_GENERATIONS_DIRECTORY,
|
||||
_HOLDERS_DIRECTORY,
|
||||
_MAX_RECORD_SIZE,
|
||||
TreeReplacedError,
|
||||
)
|
||||
from filelock._strict import _link_no_follow, _pin_directories, _raise_if_hard_links_unsupported
|
||||
from filelock._util import ensure_directory_exists, foreign_directory_owner, write_all
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
_O_NOFOLLOW: Final[int] = getattr(os, "O_NOFOLLOW", 0)
|
||||
#: Probed once at import, as :mod:`filelock._strict` does, because a per-call check flips when a test mocks the
|
||||
#: function. The set lists os.rename, whose implementation os.replace shares.
|
||||
_SUPPORTS_DIR_FD: Final[bool] = (
|
||||
hasattr(os, "link")
|
||||
and {os.link, os.open, os.stat, os.mkdir, os.rename, os.unlink} <= os.supports_dir_fd
|
||||
and os.scandir in os.supports_fd
|
||||
)
|
||||
#: Windows opens descriptors in text mode by default, which rewrites newlines; every record here is exact bytes.
|
||||
_O_BINARY: Final[int] = getattr(os, "O_BINARY", 0)
|
||||
#: ESTALE is what an NFS client reports for a file a peer unlinked out from under its cached handle: gone, not broken.
|
||||
_MISSING_ERRNOS: Final[frozenset[int]] = frozenset({ENOENT, ESTALE})
|
||||
#: Windows refuses an open with EACCES while another process is deleting the file or holds it open without sharing;
|
||||
#: both clear within moments, so an open is retried this long before the refusal is taken as real.
|
||||
_WINDOWS_OPEN_GRACE: Final[float] = 0.5
|
||||
|
||||
|
||||
class OsFiles:
|
||||
"""Record files under a lock's protocol root, opened without following symlinks and owned by this UID alone."""
|
||||
|
||||
def __init__(self, lock_file: str) -> None:
|
||||
self._lock_file = lock_file
|
||||
self._prepared = False
|
||||
self._identities: dict[str, tuple[int, int]] = {}
|
||||
|
||||
def read(self, path: str) -> bytes | None:
|
||||
# O_NONBLOCK keeps an open of a FIFO planted at the path from blocking; the regular-file check below rejects it.
|
||||
if (fd := self._open(path, os.O_RDONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY)) is None:
|
||||
return None
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
return b""
|
||||
chunks: list[bytes] = []
|
||||
remaining = _MAX_RECORD_SIZE + 1
|
||||
while remaining > 0 and (chunk := os.read(fd, remaining)):
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
return b"".join(chunks)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]:
|
||||
with self._relative(path) as (name, dir_fd):
|
||||
fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_NOFOLLOW | _O_BINARY, 0o600, dir_fd=dir_fd)
|
||||
try:
|
||||
write_all(fd, data)
|
||||
if durable:
|
||||
os.fsync(fd)
|
||||
created = os.fstat(fd)
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
# The name is unique to this call, so removing it cannot take a record from anyone else.
|
||||
self.unlink(path)
|
||||
raise
|
||||
os.close(fd)
|
||||
return created.st_dev, created.st_ino
|
||||
|
||||
def link(self, source: str, target: str) -> bool:
|
||||
failure: NotImplementedError | OSError | None = None
|
||||
with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd):
|
||||
try:
|
||||
_link_no_follow(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except (NotImplementedError, OSError) as error:
|
||||
_raise_if_hard_links_unsupported(self._lock_file, error)
|
||||
failure = error
|
||||
# The file's identity decides, not the call's result: an NFS retransmit can report a link that landed as
|
||||
# failed, and an EEXIST can be this very link answered twice. Any other error on a link that did not land is
|
||||
# a fault, not a peer winning the race, and a retry would spin.
|
||||
if (identity := _identity(source_name, source_fd)) is not None and identity == _identity(
|
||||
target_name, target_fd
|
||||
):
|
||||
return True
|
||||
if failure is not None:
|
||||
raise failure
|
||||
return False
|
||||
|
||||
def replace(self, source: str, target: str) -> bool:
|
||||
try:
|
||||
with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd):
|
||||
os.replace(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd)
|
||||
except PermissionError:
|
||||
# Windows refuses to rename over a file a reader holds open; callers tolerate a target one rename behind.
|
||||
return False
|
||||
return True
|
||||
|
||||
def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool:
|
||||
# O_NONBLOCK keeps an open of a FIFO (a named pipe) at the path from blocking, as in read. With no reader the
|
||||
# open fails with ENXIO.
|
||||
try:
|
||||
fd = self._open(path, os.O_WRONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY)
|
||||
except OSError as error:
|
||||
if error.errno != ENXIO:
|
||||
raise
|
||||
return False # pragma: needs fifo
|
||||
if fd is None:
|
||||
return False
|
||||
try:
|
||||
# O_NOFOLLOW refuses a symlink but not a hard link (a second name for the same file) or another file renamed
|
||||
# over the name, and a write through either lands in a file that is not the record. The protocol does not
|
||||
# link holder records, so a second name marks a file that inherited a removed record's inode number.
|
||||
opened = os.fstat(fd)
|
||||
if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != identity or opened.st_nlink != 1:
|
||||
return False
|
||||
write_all(fd, data)
|
||||
finally:
|
||||
os.close(fd)
|
||||
return True
|
||||
|
||||
def unlink(self, path: str) -> None:
|
||||
try:
|
||||
with self._relative(path) as (name, dir_fd):
|
||||
os.unlink(name, dir_fd=dir_fd)
|
||||
except OSError as error:
|
||||
# Windows refuses to remove a file another process holds open; the record then outlives its owner until
|
||||
# that process exits, which only delays the cleanup a later participant repeats.
|
||||
if error.errno not in _MISSING_ERRNOS and not isinstance(error, PermissionError):
|
||||
raise
|
||||
|
||||
def listdir(self, path: str) -> list[str]:
|
||||
try:
|
||||
with self._directory(path) as dir_fd, os.scandir(path if dir_fd is None else dir_fd) as entries:
|
||||
return [entry.name for entry in entries]
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
def prepare(self, root: str, *, force: bool = False) -> None:
|
||||
if self._prepared and not force:
|
||||
return
|
||||
if force:
|
||||
self._identities.clear()
|
||||
ensure_directory_exists(root)
|
||||
for directory, depth in (
|
||||
(Path(root), 0),
|
||||
(Path(root, _GENERATIONS_DIRECTORY), 1),
|
||||
(Path(root, _HOLDERS_DIRECTORY), 1),
|
||||
):
|
||||
with self._relative(str(directory), depth) as (name, dir_fd):
|
||||
with suppress(FileExistsError):
|
||||
os.mkdir(name, 0o700, dir_fd=dir_fd)
|
||||
# mkdir has no O_NOFOLLOW, so check what the name resolves to before the lock creates anything in it.
|
||||
found = os.stat(name, dir_fd=dir_fd, follow_symlinks=False)
|
||||
parent = directory.parent.stat() if dir_fd is None else os.fstat(dir_fd)
|
||||
if (
|
||||
not stat.S_ISDIR(found.st_mode)
|
||||
or getattr(found, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT
|
||||
):
|
||||
msg = f"{directory} exists but is not a directory or is a symlink; refusing to use it"
|
||||
raise TreeReplacedError(msg)
|
||||
owner = foreign_directory_owner(found, parent)
|
||||
if owner is not None: # pragma: needs foreign-sticky-tmp
|
||||
msg = f"{directory} belongs to uid {owner}, not to this process's account or the parent's owner"
|
||||
raise TreeReplacedError(msg)
|
||||
self._identities[str(directory)] = found.st_dev, found.st_ino
|
||||
self._prepared = True
|
||||
|
||||
def _open(self, path: str, flags: int) -> int | None:
|
||||
# Missing is the one answer a caller acts on without raising. A refused open is real on POSIX, where nothing
|
||||
# this lock does makes its own records unreadable; on Windows it is usually another process mid-delete or
|
||||
# holding the file without sharing, so it is retried through a short grace before it counts, and a file still
|
||||
# refusing after that has been in delete-pending for longer than any deletion takes, so it is on its way out:
|
||||
# missing.
|
||||
deadline: float | None = None
|
||||
while True:
|
||||
if isinstance(opened := self._attempt_open(path, flags), int):
|
||||
return opened
|
||||
if opened.errno in _MISSING_ERRNOS:
|
||||
return None
|
||||
if not isinstance(opened, PermissionError) or sys.platform != "win32":
|
||||
raise opened
|
||||
if deadline is None: # pragma: win32 cover
|
||||
deadline = time.monotonic() + _WINDOWS_OPEN_GRACE
|
||||
elif time.monotonic() >= deadline: # pragma: win32 cover
|
||||
return None
|
||||
time.sleep(0.002) # pragma: win32 cover
|
||||
|
||||
def _attempt_open(self, path: str, flags: int) -> int | OSError:
|
||||
try:
|
||||
with self._relative(path) as (name, dir_fd):
|
||||
return os.open(name, flags, dir_fd=dir_fd)
|
||||
except OSError as error:
|
||||
return error
|
||||
|
||||
@contextmanager
|
||||
def _relative(self, path: str, depth: int = 2) -> Generator[tuple[str, int | None]]:
|
||||
directory, name = os.path.split(path)
|
||||
with self._directory(directory, depth) as dir_fd:
|
||||
yield (path if dir_fd is None else name), dir_fd
|
||||
|
||||
@contextmanager
|
||||
def _directory(self, path: str, depth: int = 2) -> Generator[int | None]:
|
||||
# O_NOFOLLOW applies to the last path component. The check prepare() made on <root> and <root>/gen holds until a
|
||||
# process that can write the lock's directory renames the tree aside and puts one with a symlinked gen in its
|
||||
# place. Each call therefore opens the root and then the record's directory without following symlinks and
|
||||
# works inside what it opened. No descriptor outlives the call, so a later call finds a log removed and started
|
||||
# again by name, as before. None means descriptors cannot anchor a call and the call uses the whole path.
|
||||
if depth and _SUPPORTS_DIR_FD: # pragma: needs link-dir-fd
|
||||
with self._relative(path, depth - 1) as (name, parent_fd):
|
||||
try:
|
||||
fd = os.open(name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | _O_NOFOLLOW, dir_fd=parent_fd)
|
||||
except OSError as error:
|
||||
# NetBSD reports EFTYPE rather than ELOOP when O_NOFOLLOW refuses a symlink.
|
||||
if error.errno not in {ELOOP, ENOTDIR, getattr(errno, "EFTYPE", ELOOP)}:
|
||||
raise
|
||||
msg = f"{path} is not a directory or is a symlink; refusing to use it"
|
||||
raise TreeReplacedError(msg) from error
|
||||
try:
|
||||
self._admit(fd, path, parent_fd)
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
raise
|
||||
try:
|
||||
yield fd
|
||||
finally:
|
||||
os.close(fd)
|
||||
else:
|
||||
# Without dir_fd, hold each of those directories open instead, so no process can rename or replace one.
|
||||
try:
|
||||
pins = _pin_directories(*[Path(path), *Path(path).parents][:depth][::-1])
|
||||
except NotADirectoryError as error: # pragma: win32 cover
|
||||
msg = f"{error.filename} is not a directory or is a symlink; refusing to use it"
|
||||
raise TreeReplacedError(msg) from error
|
||||
with pins:
|
||||
yield None
|
||||
|
||||
def _admit(self, fd: int, path: str, parent_fd: int | None) -> None: # pragma: needs link-dir-fd
|
||||
# A directory other than the one prepare() checked may belong to another account: the tree can be renamed aside
|
||||
# and recreated while this object lives, and the ownership check has to hold for what is opened now.
|
||||
found = os.fstat(fd)
|
||||
if (found.st_dev, found.st_ino) == self._identities.get(path):
|
||||
return
|
||||
parent = os.fstat(parent_fd) if parent_fd is not None else Path(path).parent.stat()
|
||||
if (owner := foreign_directory_owner(found, parent)) is not None: # pragma: needs foreign-sticky-tmp
|
||||
msg = f"{path} belongs to uid {owner}, not to this process's account or the parent's owner"
|
||||
raise TreeReplacedError(msg)
|
||||
self._identities[path] = found.st_dev, found.st_ino
|
||||
|
||||
|
||||
def _identity(path: str, dir_fd: int | None) -> tuple[int, int] | None:
|
||||
try:
|
||||
st = os.stat(path, dir_fd=dir_fd, follow_symlinks=False)
|
||||
except OSError:
|
||||
return None
|
||||
return st.st_dev, st.st_ino
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OsFiles",
|
||||
]
|
||||
@@ -0,0 +1,793 @@
|
||||
"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = [
|
||||
"filelock._error",
|
||||
"filelock._lease",
|
||||
"filelock._soft_rw._protocol",
|
||||
"filelock._soft_rw._storage",
|
||||
"math",
|
||||
]
|
||||
|
||||
import atexit
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from typing import TYPE_CHECKING, ClassVar, Final
|
||||
from weakref import WeakValueDictionary
|
||||
|
||||
from filelock._api import (
|
||||
AcquireReturnProxy,
|
||||
_acquire_thread_lock,
|
||||
_canonical,
|
||||
_check_timeout_max,
|
||||
_ensure_current_process,
|
||||
_register_fork_class,
|
||||
_register_fork_object,
|
||||
_resolve_read_write_timeout,
|
||||
_seconds,
|
||||
_timeout_behavior,
|
||||
)
|
||||
from filelock._error import Timeout
|
||||
from filelock._lease import LeaseCompromise
|
||||
from filelock._soft_rw._protocol import GenerationLog, Ledger, Mode, Participant
|
||||
from filelock._soft_rw._storage import OsFiles
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Generator
|
||||
|
||||
from filelock._lease import CompromiseReason
|
||||
|
||||
_ALL_INSTANCES: Final[WeakValueDictionary[int, SoftReadWriteLock]] = WeakValueDictionary()
|
||||
_ALL_INSTANCES_LOCK: threading.Lock = threading.Lock()
|
||||
|
||||
|
||||
class _SoftRWMeta(type):
|
||||
_instances: WeakValueDictionary[str, SoftReadWriteLock]
|
||||
_instances_lock: threading.RLock
|
||||
_instances_under_construction: set[str]
|
||||
|
||||
def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters
|
||||
cls,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = True,
|
||||
heartbeat_interval: float = 30.0,
|
||||
stale_threshold: float | None = None,
|
||||
poll_interval: float = 0.25,
|
||||
on_compromise: Callable[[LeaseCompromise], None] | None = None,
|
||||
) -> SoftReadWriteLock:
|
||||
_ensure_current_process()
|
||||
timeout = _seconds("timeout", timeout)
|
||||
# Passed through only when set, so a subclass that declares its own constructor without it keeps working.
|
||||
extra = {} if on_compromise is None else {"on_compromise": on_compromise}
|
||||
if not is_singleton:
|
||||
return super().__call__(
|
||||
lock_file,
|
||||
timeout,
|
||||
blocking=blocking,
|
||||
is_singleton=is_singleton,
|
||||
heartbeat_interval=heartbeat_interval,
|
||||
stale_threshold=stale_threshold,
|
||||
poll_interval=poll_interval,
|
||||
**extra,
|
||||
)
|
||||
|
||||
# Keyed like the log root, so one instance never spans two logs.
|
||||
normalized = _canonical(lock_file)
|
||||
with cls._instances_lock:
|
||||
instance = cls._instances.get(normalized)
|
||||
if instance is None:
|
||||
if normalized in cls._instances_under_construction:
|
||||
msg = f"Singleton lock construction is already active for {lock_file!s}"
|
||||
raise RuntimeError(msg)
|
||||
construction_pid = os.getpid()
|
||||
construction_registry = cls._instances_under_construction
|
||||
construction_registry.add(normalized)
|
||||
try:
|
||||
instance = super().__call__(
|
||||
lock_file,
|
||||
timeout,
|
||||
blocking=blocking,
|
||||
is_singleton=is_singleton,
|
||||
heartbeat_interval=heartbeat_interval,
|
||||
stale_threshold=stale_threshold,
|
||||
poll_interval=poll_interval,
|
||||
**extra,
|
||||
)
|
||||
finally:
|
||||
construction_registry.discard(normalized)
|
||||
if os.getpid() != construction_pid: # pragma: needs fork
|
||||
msg = "Lock construction cannot continue after fork; construct a new lock in the child"
|
||||
raise RuntimeError(msg)
|
||||
cls._instances[normalized] = instance
|
||||
elif (
|
||||
_timeout_behavior(instance.timeout, any_negative=False)
|
||||
!= _timeout_behavior(timeout, any_negative=False)
|
||||
or instance.blocking != blocking
|
||||
):
|
||||
msg = (
|
||||
f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking},"
|
||||
f" cannot be changed to timeout={timeout}, blocking={blocking}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
elif on_compromise is not None and instance._on_compromise != on_compromise: # ruff: ignore[private-member-access] # the metaclass owns the singleton it compares
|
||||
# A tuning difference keeps the first caller's values; a different loss callback is a different safety
|
||||
# contract, and returning the cached instance would silently drop it.
|
||||
msg = f"Singleton lock created with a different on_compromise callback for {lock_file!s}"
|
||||
raise ValueError(msg)
|
||||
else:
|
||||
_validate_intervals(heartbeat_interval, stale_threshold, poll_interval)
|
||||
return instance
|
||||
|
||||
|
||||
class SoftReadWriteLock(metaclass=_SoftRWMeta):
|
||||
"""
|
||||
Cross-process and cross-host reader/writer lock for shared filesystems.
|
||||
|
||||
Use this class instead of :class:`~filelock.ReadWriteLock` when the lock file lives on a network filesystem (NFS,
|
||||
Lustre, HPC cluster shared storage). ``ReadWriteLock`` is backed by SQLite and cannot run on NFS because SQLite's
|
||||
``fcntl`` locking is unreliable there.
|
||||
|
||||
The lock's state is a log of immutable snapshots under ``foo.lock.rw/gen/<N>``, each naming the writer and the
|
||||
readers holding the lock at generation ``N``. Every acquire, release, and eviction publishes the next snapshot with
|
||||
one atomic no-replace hard link, so no transition can be left half done by a crash on any host. Each participant
|
||||
also keeps ``foo.lock.rw/holders/<token>``, a record whose nonce a daemon heartbeat thread rewrites every
|
||||
``heartbeat_interval`` seconds. A contender evicts a member whose record has not changed for ``stale_threshold``
|
||||
seconds of the contender's own monotonic clock; no clock is ever compared across hosts, so skew between hosts or
|
||||
against the file server cannot make a live holder look dead.
|
||||
|
||||
Writer acquire is writer-preferring: a writer enters the snapshot as soon as no live writer is named, which blocks
|
||||
any new reader, then waits for the named readers to leave. Writer starvation is impossible.
|
||||
|
||||
An expired holder is not fenced by the lock: a process paused past ``stale_threshold`` resumes believing it holds
|
||||
the lock until its next heartbeat reports the loss through ``on_compromise``. :attr:`generation` is a monotonic
|
||||
fencing token for the protected resource: a store that rejects writes carrying a lower generation than the highest
|
||||
it has accepted refuses such a holder.
|
||||
|
||||
Reentrancy, upgrade/downgrade rules, thread pinning, and singleton caching by canonical path match
|
||||
:class:`~filelock.ReadWriteLock`.
|
||||
|
||||
Forking invalidates the inherited instance in the child so the child cannot double-own the lock with its parent;
|
||||
``release()`` on that instance is a no-op, and the child must construct a new instance if it needs a lock.
|
||||
|
||||
Trust boundary: protects against same-UID non-cooperating processes (one host or cross-host) and same-host
|
||||
different-UID users via ``0o600`` / ``0o700`` permissions. Does not protect against root compromise or multi-tenant
|
||||
mounts where hostile co-tenants share the UID.
|
||||
|
||||
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
|
||||
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
|
||||
:param is_singleton: if ``True``, reuse existing instances for the same canonical path
|
||||
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
|
||||
:param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to
|
||||
``3 * heartbeat_interval``, matching etcd's ``LeaseKeepAlive`` convention
|
||||
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
|
||||
:param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is
|
||||
lost: a peer evicted it, or refreshes failed for long enough that a peer could have
|
||||
|
||||
.. versionadded:: 3.27.0
|
||||
|
||||
.. versionchanged:: 3.33.0
|
||||
|
||||
The on-disk protocol is a generation log; earlier releases cannot share a lock path with this one.
|
||||
|
||||
"""
|
||||
|
||||
_instances: WeakValueDictionary[str, SoftReadWriteLock] = WeakValueDictionary()
|
||||
_instances_lock = threading.RLock()
|
||||
_instances_under_construction: ClassVar[set[str]] = set()
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None:
|
||||
"""Give each lock subclass its own singleton registry and lock."""
|
||||
super().__init_subclass__(**kwargs)
|
||||
cls._instances = WeakValueDictionary()
|
||||
cls._instances_lock = threading.RLock()
|
||||
cls._instances_under_construction = set()
|
||||
_register_fork_class(cls)
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
|
||||
self,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = True, # ruff:ignore[unused-method-argument] # consumed by _SoftRWMeta.__call__
|
||||
heartbeat_interval: float = 30.0,
|
||||
stale_threshold: float | None = None,
|
||||
poll_interval: float = 0.25,
|
||||
on_compromise: Callable[[LeaseCompromise], None] | None = None,
|
||||
) -> None:
|
||||
self._creator_pid = os.getpid()
|
||||
heartbeat_interval, stale_threshold, poll_interval = _validate_intervals(
|
||||
heartbeat_interval, stale_threshold, poll_interval
|
||||
)
|
||||
|
||||
self.lock_file: str = os.fspath(lock_file)
|
||||
self.timeout: float = timeout
|
||||
self.blocking: bool = blocking
|
||||
self.heartbeat_interval: float = heartbeat_interval
|
||||
self.stale_threshold: float = stale_threshold
|
||||
self.poll_interval: float = poll_interval
|
||||
|
||||
# Resolved once: a relative lock path must keep naming the same log after the process changes directory. The
|
||||
# final component stays unresolved, as in 4.0.12, so every release shares one log on a symlinked lock path.
|
||||
self._root = f"{_canonical(self.lock_file)}.rw"
|
||||
self._files = OsFiles(self.lock_file)
|
||||
self._ledger = Ledger(time.monotonic)
|
||||
self._log = GenerationLog(self._files, self.lock_file, self._root)
|
||||
self._on_compromise = on_compromise
|
||||
self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock())
|
||||
self._hold: _Hold | None = None
|
||||
self._compromise: LeaseCompromise | None = None
|
||||
self._closed: bool = False
|
||||
|
||||
with _ALL_INSTANCES_LOCK:
|
||||
_ALL_INSTANCES[id(self)] = self
|
||||
_register_fork_object(self)
|
||||
|
||||
@classmethod
|
||||
def _reset_class_after_fork(cls) -> None: # pragma: forked child
|
||||
global _ALL_INSTANCES_LOCK # ruff:ignore[global-statement] # rebinds the module lock to a fresh one in the fork child
|
||||
_ALL_INSTANCES_LOCK = threading.Lock()
|
||||
cls._instances = WeakValueDictionary()
|
||||
cls._instances_lock = threading.RLock()
|
||||
cls._instances_under_construction = set()
|
||||
|
||||
@property
|
||||
def generation(self) -> int | None:
|
||||
"""
|
||||
The generation at which the current hold was granted, or ``None`` when no lock is held.
|
||||
|
||||
Generations are monotonic across all participants and hosts, so this is a fencing token: pass it to the
|
||||
protected resource and have the resource reject any operation carrying a lower generation than the highest it
|
||||
has accepted. That refuses a holder that paused past ``stale_threshold``, was evicted, and resumed. Removing the
|
||||
``<lock_file>.rw`` directory starts a new log whose generations begin again at 1.
|
||||
|
||||
.. versionadded:: 3.33.0
|
||||
|
||||
"""
|
||||
with self._locks.internal:
|
||||
return None if self._hold is None else self._hold.participant.generation
|
||||
|
||||
@property
|
||||
def compromise(self) -> LeaseCompromise | None:
|
||||
"""
|
||||
How the current hold was lost, or ``None`` while it stands.
|
||||
|
||||
Set by the heartbeat thread once a peer has evicted this holder or refreshes have failed for long enough that a
|
||||
peer could have. The holder should stop using the protected resource when it is set. Until the lost hold is
|
||||
released, acquiring on this instance raises :class:`RuntimeError` instead of joining it.
|
||||
|
||||
.. versionadded:: 3.33.0
|
||||
|
||||
"""
|
||||
with self._locks.internal:
|
||||
return self._compromise
|
||||
|
||||
@contextmanager
|
||||
def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
|
||||
"""
|
||||
Context manager that acquires and releases a shared read lock.
|
||||
|
||||
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if a write lock is already held on this instance
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self.acquire_read(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
@contextmanager
|
||||
def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
|
||||
"""
|
||||
Context manager that acquires and releases an exclusive write lock.
|
||||
|
||||
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self.acquire_write(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
|
||||
"""
|
||||
Acquire a shared read lock.
|
||||
|
||||
If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a
|
||||
read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). On the 0→1
|
||||
transition the reader publishes its holder record, enters the next snapshot once no live writer is named, and
|
||||
starts a daemon heartbeat thread that rewrites the record's nonce every ``heartbeat_interval`` seconds so peers
|
||||
on other hosts do not evict it.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
|
||||
indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
|
||||
``None`` uses the instance default
|
||||
|
||||
:returns: a proxy that can be used as a context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if a write lock is already held on this instance, if this instance was invalidated by
|
||||
:func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
:raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem
|
||||
refuses the no-replace hard links the protocol commits with
|
||||
|
||||
"""
|
||||
return self._acquire("read", timeout, blocking=blocking)
|
||||
|
||||
def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
|
||||
"""
|
||||
Acquire an exclusive write lock.
|
||||
|
||||
If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant).
|
||||
Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not
|
||||
allowed). Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises
|
||||
:class:`RuntimeError`.
|
||||
|
||||
A writer enters the next snapshot as its writer as soon as no live writer is named, which blocks every new
|
||||
reader on every host, then waits for the readers that snapshot names to leave. Writer starvation is impossible:
|
||||
new readers see the named writer and wait behind it.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
|
||||
indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
|
||||
``None`` uses the instance default
|
||||
|
||||
:returns: a proxy that can be used as a context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if this
|
||||
instance was invalidated by :func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
:raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem
|
||||
refuses the no-replace hard links the protocol commits with
|
||||
|
||||
"""
|
||||
return self._acquire("write", timeout, blocking=blocking)
|
||||
|
||||
@classmethod
|
||||
def get_lock(
|
||||
cls,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
) -> SoftReadWriteLock:
|
||||
"""
|
||||
Return the singleton :class:`SoftReadWriteLock` for *lock_file*.
|
||||
|
||||
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
|
||||
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
|
||||
|
||||
:returns: the singleton lock instance
|
||||
|
||||
:raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values
|
||||
|
||||
"""
|
||||
return cls(lock_file, timeout, blocking=blocking)
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Release any held lock and mark the instance closed.
|
||||
|
||||
Idempotent. After calling this method the instance can no longer acquire locks — subsequent acquires raise
|
||||
:class:`RuntimeError`, and so does an acquire another thread is still waiting in, which gives up its claim
|
||||
instead of taking the lock. A fork-invalidated instance is closed without raising.
|
||||
"""
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
return
|
||||
# Closed first, under the lock an acquire publishes its hold under: an acquire still waiting then sees the flag
|
||||
# and leaves instead of publishing, and one that published before it is released below.
|
||||
with self._locks.internal:
|
||||
self._closed = True
|
||||
self.release(force=True)
|
||||
|
||||
def release(self, *, force: bool = False) -> None:
|
||||
"""
|
||||
Release one level of the current lock.
|
||||
|
||||
When the lock level reaches zero the heartbeat thread is stopped, the participant is committed out of the next
|
||||
snapshot, and its holder record is removed. On a fork-invalidated instance (that is, the child of a
|
||||
:func:`os.fork` call made while the parent held a lock) this method is a no-op so inherited ``with`` blocks can
|
||||
unwind cleanly in the child.
|
||||
|
||||
:param force: if ``True``, release the lock completely regardless of the current lock level
|
||||
|
||||
:raises RuntimeError: if no lock is currently held and *force* is ``False``, or if the calling thread holds no
|
||||
level of the current lock and *force* is ``False``
|
||||
|
||||
"""
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
return
|
||||
self._release(threading.get_ident(), force=force)
|
||||
|
||||
def _acquire_level_of(
|
||||
self, holder: int, mode: Mode, timeout: float, *, blocking: bool, cancel_check: Callable[[], bool]
|
||||
) -> AcquireReturnProxy:
|
||||
# Records the level under *holder* instead of the calling thread: an async wrapper's executor thread does not
|
||||
# identify the tasks it holds for. Gives up with Timeout once *cancel_check* is true.
|
||||
return self._acquire(mode, timeout, blocking=blocking, holder=holder, cancel_check=cancel_check)
|
||||
|
||||
def _release_level_of(self, holder: int) -> None:
|
||||
self._release(holder, force=False)
|
||||
|
||||
def _holds_level_of(self, holder: int) -> bool:
|
||||
# Read without the internal lock: an async wrapper asks from its event loop, which must not wait out a release
|
||||
# running on another thread.
|
||||
return (hold := self._hold) is not None and (holder == hold.write_thread_id or holder in hold.read_depths)
|
||||
|
||||
def _release(self, thread_id: int, *, force: bool) -> None:
|
||||
with self._locks.internal:
|
||||
hold = self._hold
|
||||
if hold is None:
|
||||
if force:
|
||||
return
|
||||
msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held"
|
||||
raise RuntimeError(msg)
|
||||
if force:
|
||||
hold.level = 0
|
||||
else:
|
||||
self._raise_if_not_holder(hold, thread_id)
|
||||
hold.leave_one_level(thread_id)
|
||||
if hold.level > 0:
|
||||
return
|
||||
self._hold = None
|
||||
|
||||
# Stop the heartbeat before leaving, so a late tick cannot re-read a snapshot this participant just left and
|
||||
# report an eviction that was its own release. on_compromise runs on that thread and may be what called here,
|
||||
# in which case there is nothing to join.
|
||||
hold.heartbeat_stop.set()
|
||||
try:
|
||||
if hold.heartbeat_thread is not threading.current_thread():
|
||||
hold.heartbeat_thread.join(timeout=min(self.heartbeat_interval + 1.0, threading.TIMEOUT_MAX))
|
||||
finally:
|
||||
# The hold is already gone from this instance, so a join that raises must not strand its record too.
|
||||
hold.participant.leave()
|
||||
|
||||
def _raise_if_not_holder(self, hold: _Hold, thread_id: int) -> None:
|
||||
# The hold's own heartbeat thread may release it, since on_compromise runs there.
|
||||
if thread_id == hold.heartbeat_thread.ident:
|
||||
return
|
||||
# Another thread's release would leave the generation log under the holder and let a peer in.
|
||||
if hold.mode == "write" and thread_id != hold.write_thread_id:
|
||||
msg = (
|
||||
f"Cannot release write lock on {self.lock_file} (lock id: {id(self)}) "
|
||||
f"from thread {thread_id} while it is held by thread {hold.write_thread_id}"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
# Readers share one participant, so a thread that holds no read level would end another's hold.
|
||||
if hold.mode == "read" and thread_id not in hold.read_depths:
|
||||
msg = (
|
||||
f"Cannot release read lock on {self.lock_file} (lock id: {id(self)}) "
|
||||
f"from thread {thread_id}, which does not hold it"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
|
||||
def _acquire(
|
||||
self,
|
||||
mode: Mode,
|
||||
timeout: float | None,
|
||||
*,
|
||||
blocking: bool | None,
|
||||
holder: int | None = None,
|
||||
cancel_check: Callable[[], bool] | None = None,
|
||||
) -> AcquireReturnProxy:
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
msg = f"SoftReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance"
|
||||
raise RuntimeError(msg)
|
||||
timeout = self.timeout if timeout is None else timeout
|
||||
blocking = self.blocking if blocking is None else blocking
|
||||
holder = threading.get_ident() if holder is None else holder
|
||||
|
||||
with self._locks.internal:
|
||||
if self._closed:
|
||||
raise self._closed_error()
|
||||
timeout = _resolve_read_write_timeout(timeout, blocking=blocking)
|
||||
if self._hold is not None:
|
||||
return self._validate_reentrant(mode, holder)
|
||||
|
||||
deadline = None if timeout == -1 else time.perf_counter() + timeout
|
||||
if not _acquire_thread_lock(
|
||||
self._locks.transaction,
|
||||
blocking=blocking,
|
||||
timeout=timeout,
|
||||
poll_interval=self.poll_interval,
|
||||
cancel_check=cancel_check,
|
||||
):
|
||||
raise Timeout(self.lock_file) from None
|
||||
try:
|
||||
return self._do_acquire_inner(mode, deadline, holder, blocking=blocking, cancel_check=cancel_check)
|
||||
finally:
|
||||
self._locks.transaction.release()
|
||||
|
||||
def _closed_error(self) -> RuntimeError:
|
||||
return RuntimeError(f"SoftReadWriteLock on {self.lock_file} has been closed")
|
||||
|
||||
def _do_acquire_inner(
|
||||
self,
|
||||
mode: Mode,
|
||||
deadline: float | None,
|
||||
holder: int,
|
||||
*,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
) -> AcquireReturnProxy:
|
||||
with self._locks.internal:
|
||||
if self._hold is not None:
|
||||
return self._validate_reentrant(mode, holder)
|
||||
participant = Participant(
|
||||
self._files,
|
||||
self._root,
|
||||
mode,
|
||||
stale_threshold=self.stale_threshold,
|
||||
clock=time.monotonic,
|
||||
ledger=self._ledger,
|
||||
log=self._log,
|
||||
)
|
||||
participant.publish()
|
||||
try:
|
||||
self._wait_for(participant.advance, deadline=deadline, blocking=blocking, cancel_check=cancel_check)
|
||||
except BaseException:
|
||||
# A contender that gives up must not stay named in the snapshot: a writer left there would block every
|
||||
# reader until a peer waits out the stale threshold. Its own token is all leave() touches, so it cannot
|
||||
# undo a peer's claim.
|
||||
with suppress(OSError):
|
||||
participant.leave()
|
||||
raise
|
||||
stop_event = threading.Event()
|
||||
hold = _Hold(
|
||||
level=1,
|
||||
mode=mode,
|
||||
write_thread_id=holder if mode == "write" else None,
|
||||
read_depths={holder: 1} if mode == "read" else {},
|
||||
participant=participant,
|
||||
heartbeat_thread=_HeartbeatThread(
|
||||
refresh=lambda: self._refresh(participant, hold),
|
||||
interval=self.heartbeat_interval,
|
||||
stop_event=stop_event,
|
||||
name=f"filelock-heartbeat-{id(self):x}",
|
||||
),
|
||||
heartbeat_stop=stop_event,
|
||||
last_refresh=time.monotonic(),
|
||||
)
|
||||
# Publish the hold and start its heartbeat under one internal-lock section, so a concurrent release() never
|
||||
# observes a hold whose thread has not started and joins it. If the OS refuses the thread, clear the hold and
|
||||
# leave: left in place, a peer evicts the unrefreshed record and acquires while this instance still believes it
|
||||
# holds the lock. A close() that landed while this acquire waited gets the same treatment: published, the hold
|
||||
# would keep its heartbeat and its slot on an instance nobody can release through.
|
||||
start_error: BaseException | None = None
|
||||
with self._locks.internal:
|
||||
if self._closed:
|
||||
start_error = self._closed_error()
|
||||
else:
|
||||
self._hold = hold
|
||||
self._compromise = None
|
||||
try:
|
||||
hold.heartbeat_thread.start()
|
||||
except BaseException as error: # ruff:ignore[blind-except] # clear the slot below and re-raise
|
||||
self._hold = None
|
||||
start_error = error
|
||||
if start_error is not None:
|
||||
participant.leave()
|
||||
raise start_error
|
||||
return AcquireReturnProxy(lock=self)
|
||||
|
||||
def _validate_reentrant(self, mode: Mode, holder: int) -> AcquireReturnProxy:
|
||||
hold = self._hold
|
||||
assert hold is not None # ruff:ignore[assert] # callers dispatch here only inside the self._hold is not None branch
|
||||
if hold.mode != mode:
|
||||
opposite = "write" if mode == "read" else "read"
|
||||
direction = "downgrade" if mode == "read" else "upgrade"
|
||||
msg = (
|
||||
f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): "
|
||||
f"already holding a {opposite} lock ({direction} not allowed)"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
if mode == "write" and holder != hold.write_thread_id:
|
||||
msg = (
|
||||
f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) "
|
||||
f"from thread {holder} while it is held by thread {hold.write_thread_id}"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
# Publishing a hold clears the loss recorded for the one before, so a loss seen here is this hold's. Its
|
||||
# heartbeat has stopped and a peer may hold the lock, so a new level must not bypass the log.
|
||||
if (compromise := self._compromise) is not None:
|
||||
raise _lost_hold_error(self.lock_file, mode, compromise)
|
||||
if mode == "read":
|
||||
hold.read_depths[holder] = hold.read_depths.get(holder, 0) + 1
|
||||
hold.level += 1
|
||||
return AcquireReturnProxy(lock=self)
|
||||
|
||||
def _wait_for(
|
||||
self,
|
||||
predicate: Callable[[], bool],
|
||||
*,
|
||||
deadline: float | None,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
) -> None:
|
||||
while not predicate():
|
||||
# The flag is read without the lock here only to stop waiting early; publishing the hold re-checks it.
|
||||
if self._closed:
|
||||
raise self._closed_error()
|
||||
now = time.perf_counter()
|
||||
if not blocking or (deadline is not None and now >= deadline):
|
||||
raise Timeout(self.lock_file)
|
||||
# An abandoned acquire gives up like a timed-out one, so the caller's rollback withdraws its claim.
|
||||
if cancel_check is not None and cancel_check():
|
||||
raise Timeout(self.lock_file)
|
||||
sleep_for = self.poll_interval
|
||||
if deadline is not None:
|
||||
sleep_for = min(sleep_for, max(deadline - now, 0.0))
|
||||
time.sleep(sleep_for)
|
||||
|
||||
def _refresh(self, participant: Participant, hold: _Hold) -> bool:
|
||||
# The loop ends at the first loss, so the holder hears about it once. A transient filesystem error (ESTALE /
|
||||
# EIO on the NFS-style filesystems this lock targets) is not a loss: retry rather than report a false
|
||||
# compromise. Report the record unrefreshable only once failures have run long enough that a peer could evict
|
||||
# it before the next success would land, a margin before the record actually ages out.
|
||||
try:
|
||||
outcome, error = participant.heartbeat()
|
||||
except Exception as failure: # ruff:ignore[blind-except] # the thread would end with the hold still published
|
||||
self._report_compromise(hold, "refresh-failed", failure)
|
||||
return False
|
||||
if outcome == "ok":
|
||||
hold.last_refresh = time.monotonic()
|
||||
return True
|
||||
if outcome == "lost":
|
||||
self._report_compromise(hold, "evicted", None)
|
||||
return False
|
||||
if time.monotonic() - hold.last_refresh >= self.stale_threshold - self.heartbeat_interval:
|
||||
self._report_compromise(hold, "refresh-failed", error)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _report_compromise(self, hold: _Hold, reason: CompromiseReason, error: Exception | None) -> None:
|
||||
# A tick that outlived its release's join must not stamp the old hold's loss onto whatever was acquired since.
|
||||
with self._locks.internal:
|
||||
if self._hold is not hold:
|
||||
return
|
||||
compromise = LeaseCompromise(
|
||||
lock_file=self.lock_file, token=hold.participant.token, reason=reason, error=error
|
||||
)
|
||||
self._compromise = compromise
|
||||
if self._on_compromise is not None:
|
||||
self._on_compromise(compromise)
|
||||
|
||||
def _reset_after_fork_in_child(self) -> None: # pragma: forked child
|
||||
self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock())
|
||||
self._hold = None
|
||||
|
||||
|
||||
class _HeartbeatThread(threading.Thread):
|
||||
def __init__(
|
||||
self,
|
||||
refresh: Callable[[], bool],
|
||||
interval: float,
|
||||
stop_event: threading.Event,
|
||||
name: str,
|
||||
) -> None:
|
||||
super().__init__(name=name, daemon=True)
|
||||
self._refresh = refresh
|
||||
self._interval = interval
|
||||
self._stop_event = stop_event
|
||||
|
||||
def run(self) -> None:
|
||||
# Set on every exit, an on_compromise that raises included, so nothing waits on a thread that is gone.
|
||||
try:
|
||||
while not self._stop_event.wait(self._interval):
|
||||
if not self._refresh():
|
||||
return
|
||||
finally:
|
||||
self._stop_event.set()
|
||||
|
||||
|
||||
def _lost_hold_error(lock_file: str, mode: Mode, compromise: LeaseCompromise) -> RuntimeError:
|
||||
return RuntimeError(
|
||||
f"Cannot acquire {mode} lock on {lock_file}: the current hold was lost ({compromise.reason}), "
|
||||
"release it before acquiring again"
|
||||
)
|
||||
|
||||
|
||||
def _validate_intervals(
|
||||
heartbeat_interval: float, stale_threshold: float | None, poll_interval: float
|
||||
) -> tuple[float, float, float]:
|
||||
heartbeat_interval = _seconds("heartbeat_interval", heartbeat_interval)
|
||||
if not isfinite(heartbeat_interval) or heartbeat_interval <= 0:
|
||||
msg = f"heartbeat_interval must be positive and finite, got {heartbeat_interval}"
|
||||
raise ValueError(msg)
|
||||
# The heartbeat thread waits this long between refreshes, and Event.wait overflows past TIMEOUT_MAX.
|
||||
_check_timeout_max("heartbeat_interval", heartbeat_interval)
|
||||
stale_threshold = (
|
||||
heartbeat_interval * 3 if stale_threshold is None else _seconds("stale_threshold", stale_threshold)
|
||||
)
|
||||
if not isfinite(stale_threshold) or stale_threshold <= heartbeat_interval:
|
||||
msg = (
|
||||
f"stale_threshold must exceed heartbeat_interval ({heartbeat_interval}) "
|
||||
f"and be finite, got {stale_threshold}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
if not isfinite(poll_interval := _seconds("poll_interval", poll_interval)) or poll_interval <= 0:
|
||||
msg = f"poll_interval must be positive and finite, got {poll_interval}"
|
||||
raise ValueError(msg)
|
||||
_check_timeout_max("poll_interval", poll_interval)
|
||||
# A waiting writer refreshes its record once per poll, so a poll slower than the threshold reads as dead.
|
||||
if poll_interval >= stale_threshold:
|
||||
msg = f"poll_interval must be below stale_threshold ({stale_threshold}), got {poll_interval}"
|
||||
raise ValueError(msg)
|
||||
return heartbeat_interval, stale_threshold, poll_interval
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Locks:
|
||||
internal: threading.Lock
|
||||
transaction: threading.Lock
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Hold:
|
||||
"""Everything that exists only while a lock is held; ``None`` when the instance has no lock."""
|
||||
|
||||
level: int
|
||||
mode: Mode
|
||||
write_thread_id: int | None
|
||||
#: Read levels per thread, so a thread can give back only the levels it took.
|
||||
read_depths: dict[int, int]
|
||||
participant: Participant
|
||||
heartbeat_thread: _HeartbeatThread
|
||||
heartbeat_stop: threading.Event
|
||||
last_refresh: float
|
||||
|
||||
def leave_one_level(self, thread_id: int) -> None:
|
||||
self.level -= 1
|
||||
if self.mode == "read":
|
||||
# The heartbeat thread holds no level of its own, so its release gives back one of a reader's.
|
||||
owner = thread_id if thread_id in self.read_depths else next(iter(self.read_depths))
|
||||
if depth := self.read_depths.pop(owner) - 1:
|
||||
self.read_depths[owner] = depth
|
||||
|
||||
|
||||
def _cleanup_all_instances() -> None: # pragma: no cover - runs from atexit at interpreter shutdown
|
||||
for instance in list(_ALL_INSTANCES.values()):
|
||||
with suppress(Exception):
|
||||
instance.release(force=True)
|
||||
|
||||
|
||||
atexit.register(_cleanup_all_instances)
|
||||
_register_fork_class(SoftReadWriteLock)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SoftReadWriteLock",
|
||||
"_lost_hold_error",
|
||||
]
|
||||
Reference in new issue
Block a user