Initial commit
This commit is contained in:
commit
57fc3525eb
7311 files changed
+1291398
No files matched your search
@@ -0,0 +1,174 @@
|
||||
"""
|
||||
A platform independent file lock that supports the with-statement.
|
||||
|
||||
.. autodata:: filelock.__version__
|
||||
:no-value:
|
||||
|
||||
.. autodata:: filelock.OwnerMode
|
||||
|
||||
.. autodata:: filelock.CompromiseReason
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["filelock._descriptor", "filelock._error", "importlib"]
|
||||
|
||||
import sys
|
||||
import warnings
|
||||
from importlib import import_module
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from filelock._api import AcquireReturnProxy, BaseFileLock, CloseErrorPolicy, ContextErrorPolicy, LockOptions
|
||||
from filelock._descriptor import lock_descriptor, unlock_descriptor
|
||||
from filelock._error import LeaseSettingsMismatch, SoftFileLockLifetimeWarning, SoftFileLockProtocolError, Timeout
|
||||
from filelock._unix import UnixFileLock
|
||||
from filelock._unix import has_fcntl as _has_fcntl
|
||||
from filelock._windows import WindowsFileLock
|
||||
|
||||
# Keeps ``filelock.version`` the string 4.0.12 exposed rather than the submodule.
|
||||
from filelock.version import version
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from filelock import asyncio
|
||||
from filelock._async_read_write import AsyncAcquireReadWriteReturnProxy, AsyncReadWriteLock
|
||||
from filelock._lease import CompromiseReason, LeaseCompromise, SoftFileLease
|
||||
from filelock._marker import MarkerSoftFileLock, OwnerMode, OwnerRecord
|
||||
from filelock._read_write import ReadWriteLock
|
||||
from filelock._soft import SoftFileLock
|
||||
from filelock._soft_rw import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock, SoftReadWriteLock
|
||||
from filelock._strict import StrictSoftFileClaim, StrictSoftFileClaimState, StrictSoftFileLock
|
||||
from filelock.asyncio import (
|
||||
AsyncAcquireReturnProxy,
|
||||
AsyncSoftFileLease,
|
||||
AsyncSoftFileLock,
|
||||
AsyncStrictSoftFileLock,
|
||||
AsyncUnixFileLock,
|
||||
AsyncWindowsFileLock,
|
||||
BaseAsyncFileLock,
|
||||
)
|
||||
|
||||
#: version of the project as a string
|
||||
__version__: Final[str] = version
|
||||
|
||||
#: ``True`` when :func:`fcntl.flock` is available. Off Windows, :data:`FileLock` then resolves to
|
||||
#: :class:`UnixFileLock`, otherwise to :class:`SoftFileLock`. Always ``False`` on Windows.
|
||||
#:
|
||||
#: :meta hide-value:
|
||||
has_fcntl: Final[bool] = _has_fcntl
|
||||
|
||||
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
_FileLock: type[BaseFileLock] = WindowsFileLock
|
||||
else: # pragma: win32 no cover # ruff:ignore[collapsible-else-if] # the else carries the win32 no-cover pragma
|
||||
if has_fcntl:
|
||||
_FileLock: type[BaseFileLock] = UnixFileLock
|
||||
else:
|
||||
from filelock._soft import SoftFileLock
|
||||
|
||||
_FileLock = SoftFileLock
|
||||
warnings.warn("only soft file lock is available", stacklevel=2)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
FileLock = SoftFileLock
|
||||
AsyncFileLock = AsyncSoftFileLock
|
||||
else:
|
||||
#: Alias for the lock, which should be used for the current platform.
|
||||
FileLock = _FileLock
|
||||
|
||||
# Importing asyncio and sqlite3 costs more than the rest of the package, so names that need them load on first access.
|
||||
_LAZY_ATTRIBUTES: Final[dict[str, str]] = {
|
||||
"AsyncAcquireReadWriteReturnProxy": "._async_read_write",
|
||||
"AsyncAcquireReturnProxy": ".asyncio",
|
||||
"AsyncAcquireSoftReadWriteReturnProxy": "._soft_rw",
|
||||
"AsyncFileLock": ".asyncio",
|
||||
"AsyncReadWriteLock": "._async_read_write",
|
||||
"AsyncSoftFileLease": ".asyncio",
|
||||
"AsyncSoftFileLock": ".asyncio",
|
||||
"AsyncSoftReadWriteLock": "._soft_rw",
|
||||
"AsyncStrictSoftFileLock": ".asyncio",
|
||||
"AsyncUnixFileLock": ".asyncio",
|
||||
"AsyncWindowsFileLock": ".asyncio",
|
||||
"BaseAsyncFileLock": ".asyncio",
|
||||
"CompromiseReason": "._lease",
|
||||
"LeaseCompromise": "._lease",
|
||||
"MarkerSoftFileLock": "._marker",
|
||||
"OwnerMode": "._marker",
|
||||
"OwnerRecord": "._marker",
|
||||
"ReadWriteLock": "._read_write",
|
||||
"SoftFileLease": "._lease",
|
||||
"SoftFileLock": "._soft",
|
||||
"SoftReadWriteLock": "._soft_rw",
|
||||
"StrictSoftFileClaim": "._strict",
|
||||
"StrictSoftFileClaimState": "._strict",
|
||||
"StrictSoftFileLock": "._strict",
|
||||
"asyncio": ".asyncio",
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str) -> object:
|
||||
if (module_name := _LAZY_ATTRIBUTES.get(name)) is None:
|
||||
msg = f"module {__name__!r} has no attribute {name!r}"
|
||||
raise AttributeError(msg)
|
||||
try:
|
||||
module = import_module(module_name, __name__)
|
||||
except ImportError: # pragma: lacks sqlite3
|
||||
if module_name not in {"._read_write", "._async_read_write"}:
|
||||
raise
|
||||
value = None
|
||||
else:
|
||||
if name == "asyncio":
|
||||
value = module
|
||||
else:
|
||||
value = getattr(module, f"Async{_FileLock.__name__}" if name == "AsyncFileLock" else name)
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
return sorted({*globals(), *_LAZY_ATTRIBUTES})
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AcquireReturnProxy",
|
||||
"AsyncAcquireReadWriteReturnProxy",
|
||||
"AsyncAcquireReturnProxy",
|
||||
"AsyncAcquireSoftReadWriteReturnProxy",
|
||||
"AsyncFileLock",
|
||||
"AsyncReadWriteLock",
|
||||
"AsyncSoftFileLease",
|
||||
"AsyncSoftFileLock",
|
||||
"AsyncSoftReadWriteLock",
|
||||
"AsyncStrictSoftFileLock",
|
||||
"AsyncUnixFileLock",
|
||||
"AsyncWindowsFileLock",
|
||||
"BaseAsyncFileLock",
|
||||
"BaseFileLock",
|
||||
"CloseErrorPolicy",
|
||||
"CompromiseReason",
|
||||
"ContextErrorPolicy",
|
||||
"FileLock",
|
||||
"LeaseCompromise",
|
||||
"LeaseSettingsMismatch",
|
||||
"LockOptions",
|
||||
"MarkerSoftFileLock",
|
||||
"OwnerMode",
|
||||
"OwnerRecord",
|
||||
"ReadWriteLock",
|
||||
"SoftFileLease",
|
||||
"SoftFileLock",
|
||||
"SoftFileLockLifetimeWarning",
|
||||
"SoftFileLockProtocolError",
|
||||
"SoftReadWriteLock",
|
||||
"StrictSoftFileClaim",
|
||||
"StrictSoftFileClaimState",
|
||||
"StrictSoftFileLock",
|
||||
"Timeout",
|
||||
"UnixFileLock",
|
||||
"WindowsFileLock",
|
||||
"__version__",
|
||||
"asyncio",
|
||||
"has_fcntl",
|
||||
"lock_descriptor",
|
||||
"unlock_descriptor",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,507 @@
|
||||
"""Separate caller cancellation from backend task and executor-future results."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["asyncio", "collections", "concurrent.futures", "filelock._error"]
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import time
|
||||
from collections import deque
|
||||
from concurrent.futures import Future as ConcurrentFuture
|
||||
from dataclasses import dataclass
|
||||
from threading import Lock
|
||||
from typing import TYPE_CHECKING, Final, Generic, Literal, NoReturn, TypeVar, cast
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from filelock._api import (
|
||||
_append_exception_context,
|
||||
_fork_transition,
|
||||
_raise_chained_errors,
|
||||
_resolve_read_write_timeout,
|
||||
)
|
||||
from filelock._error import Timeout
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable
|
||||
from concurrent.futures import Executor
|
||||
|
||||
from filelock._read_write import ReadWriteLock
|
||||
from filelock._soft_rw import SoftReadWriteLock
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
#: The holder an async wrapper records its backend level under. No thread has this identifier, so sync code on the
|
||||
#: executor thread that took the level cannot re-enter or release it.
|
||||
_TASKS_THREAD_ID: Final[int] = 0
|
||||
|
||||
|
||||
class _AsyncTransitionUnavailableError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _BackendOutcome(Generic[_T]):
|
||||
value: _T | None = None
|
||||
error: BaseException | None = None
|
||||
|
||||
|
||||
class _AsyncTransitionGate:
|
||||
"""
|
||||
First-in, first-out admission for the calls that transition one lock context.
|
||||
|
||||
Each call queues a ticket, and the head of the queue holds the gate. A call that gives up removes its own ticket,
|
||||
so abandoned calls do not pile up behind a long holder.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._queue_lock: Final[Lock] = Lock()
|
||||
self._queue: Final[deque[ConcurrentFuture[None]]] = deque()
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def hold(self) -> AsyncGenerator[None]:
|
||||
ticket: ConcurrentFuture[None] = ConcurrentFuture()
|
||||
try:
|
||||
self._join(ticket)
|
||||
if not ticket.done():
|
||||
await self._wait_for_turn(ticket, blocking=True, cancel_check=None, deadline=None, poll_interval=0.05)
|
||||
yield
|
||||
finally:
|
||||
self._leave(ticket)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def hold_for_acquire(
|
||||
self,
|
||||
*,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
deadline: float | None,
|
||||
poll_interval: float,
|
||||
) -> AsyncGenerator[None]:
|
||||
ticket: ConcurrentFuture[None] = ConcurrentFuture()
|
||||
try:
|
||||
self._join(ticket)
|
||||
if not ticket.done():
|
||||
await self._wait_for_turn(
|
||||
ticket,
|
||||
blocking=blocking,
|
||||
cancel_check=cancel_check,
|
||||
deadline=deadline,
|
||||
poll_interval=poll_interval,
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
self._leave(ticket)
|
||||
|
||||
async def _wait_for_turn(
|
||||
self,
|
||||
ticket: ConcurrentFuture[None],
|
||||
*,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
deadline: float | None,
|
||||
poll_interval: float,
|
||||
) -> None:
|
||||
if not blocking:
|
||||
raise _AsyncTransitionUnavailableError
|
||||
waiter = asyncio.wrap_future(ticket)
|
||||
while not ticket.done():
|
||||
if cancel_check is not None and cancel_check():
|
||||
raise _AsyncTransitionUnavailableError
|
||||
wait_interval = poll_interval
|
||||
if deadline is not None:
|
||||
if (remaining := deadline - time.perf_counter()) <= 0:
|
||||
raise _AsyncTransitionUnavailableError
|
||||
wait_interval = min(wait_interval, remaining)
|
||||
await asyncio.wait((waiter,), timeout=wait_interval)
|
||||
# A hand-off that a signal handler's exception cut short leaves the head undone, so the head settles itself.
|
||||
self._settle_head()
|
||||
|
||||
def _join(self, ticket: ConcurrentFuture[None]) -> None:
|
||||
with self._queue_lock:
|
||||
self._queue.append(ticket)
|
||||
if len(self._queue) == 1:
|
||||
ticket.set_result(None)
|
||||
|
||||
def _leave(self, ticket: ConcurrentFuture[None]) -> None:
|
||||
with self._queue_lock:
|
||||
try:
|
||||
self._queue.remove(ticket)
|
||||
except ValueError:
|
||||
# The join never completed, or this ticket already left.
|
||||
return
|
||||
self._settle_head_locked()
|
||||
|
||||
def _settle_head(self) -> None:
|
||||
with self._queue_lock:
|
||||
self._settle_head_locked()
|
||||
|
||||
def _settle_head_locked(self) -> None:
|
||||
if self._queue and not (head := self._queue[0]).done():
|
||||
head.set_result(None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TaskHold:
|
||||
"""
|
||||
One task's hold, as the nesting count per wrapper it entered through.
|
||||
|
||||
A task's hold ends when its last wrapper level is released or closed. Closing one wrapper ends only the levels
|
||||
taken through it.
|
||||
"""
|
||||
|
||||
depths: dict[object, int]
|
||||
|
||||
|
||||
class _TaskOwners:
|
||||
"""
|
||||
Per-task ownership in front of a sync reader/writer lock that tracks threads.
|
||||
|
||||
An async wrapper runs each call on an executor thread, and that thread does not identify the calling task. Waiting
|
||||
writers block new readers so a stream of readers cannot starve them. A thread lock guards the table because tasks on
|
||||
different event loops share it. The table stands for one backend level, taken once for all tasks. Sync code can
|
||||
end that level directly, with ``close()`` or a forced release on a shared singleton. The next admission then
|
||||
forgets the holds instead of letting more tasks join them.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock: Final[Lock] = Lock()
|
||||
self._mode: Literal["read", "write"] | None = None
|
||||
self._holds: dict[asyncio.Task[object], _TaskHold] = {}
|
||||
self._transitioning = False
|
||||
self._waiting_writers = 0
|
||||
#: One ticket per waiting call, settled on the next state change. A call that stops waiting takes its own out,
|
||||
#: so one that times out or is canceled leaves nothing behind.
|
||||
self._waiters: dict[ConcurrentFuture[None], None] = {}
|
||||
|
||||
async def acquire( # ruff:ignore[too-many-arguments] # one keyword per piece of wrapper state the table consults
|
||||
self,
|
||||
mode: Literal["read", "write"],
|
||||
*,
|
||||
timeout: float,
|
||||
blocking: bool,
|
||||
lock_file: str,
|
||||
via: object,
|
||||
enter: Callable[[float], Awaitable[None]],
|
||||
joinable: Callable[[], bool],
|
||||
) -> None:
|
||||
task = _current_task()
|
||||
timeout = _resolve_read_write_timeout(timeout, blocking=blocking)
|
||||
deadline = None if not blocking or timeout < 0 else time.perf_counter() + timeout
|
||||
waiting_writer = False
|
||||
try:
|
||||
while isinstance(
|
||||
admission := self._admit(
|
||||
task, mode, lock_file, via=via, joinable=joinable, waiting_writer=waiting_writer
|
||||
),
|
||||
ConcurrentFuture,
|
||||
):
|
||||
try:
|
||||
if not blocking or (deadline is not None and time.perf_counter() >= deadline):
|
||||
raise Timeout(lock_file)
|
||||
if mode == "write" and not waiting_writer:
|
||||
waiting_writer = True
|
||||
with self._lock:
|
||||
self._waiting_writers += 1
|
||||
wait = None if deadline is None else deadline - time.perf_counter()
|
||||
await asyncio.wait((asyncio.wrap_future(admission),), timeout=wait)
|
||||
finally:
|
||||
self._forget(admission)
|
||||
finally:
|
||||
if waiting_writer:
|
||||
with self._lock:
|
||||
self._waiting_writers -= 1
|
||||
self._notify()
|
||||
if admission == "held":
|
||||
return
|
||||
try:
|
||||
await enter(-1 if deadline is None else max(0.0, deadline - time.perf_counter()))
|
||||
except BaseException:
|
||||
self._finish_transition(mode=None)
|
||||
raise
|
||||
self._finish_transition(mode=mode, holds={task: _TaskHold(depths={via: 1})})
|
||||
|
||||
def _admit( # ruff:ignore[too-many-arguments] # the acquire's state, passed through under the table lock
|
||||
self,
|
||||
task: asyncio.Task[object],
|
||||
mode: Literal["read", "write"],
|
||||
lock_file: str,
|
||||
*,
|
||||
via: object,
|
||||
joinable: Callable[[], bool],
|
||||
waiting_writer: bool,
|
||||
) -> Literal["held", "enter"] | ConcurrentFuture[None]:
|
||||
with self._lock:
|
||||
# A "held" answer trusts the backend level. Once the backend lost it, the holds are gone and the next task
|
||||
# has to take the lock again.
|
||||
if self._mode is not None and not self._transitioning and not joinable():
|
||||
self._holds.clear()
|
||||
self._mode = None
|
||||
if (hold := self._holds.get(task)) is not None:
|
||||
if self._mode != mode:
|
||||
msg = (
|
||||
f"Cannot acquire {mode} lock on {lock_file}: already holding a "
|
||||
f"{'write' if mode == 'read' else 'read'} lock "
|
||||
f"({'downgrade' if mode == 'read' else 'upgrade'} not allowed)"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
hold.depths[via] = hold.depths.get(via, 0) + 1
|
||||
return "held"
|
||||
writers_ahead = self._waiting_writers - waiting_writer
|
||||
if not self._transitioning:
|
||||
if mode == "read" and self._mode == "read" and not writers_ahead:
|
||||
self._holds[task] = _TaskHold(depths={via: 1})
|
||||
return "held"
|
||||
if not self._holds and (mode == "write" or not writers_ahead):
|
||||
self._transitioning, self._mode = True, mode
|
||||
return "enter"
|
||||
return self._ticket()
|
||||
|
||||
async def release(
|
||||
self,
|
||||
*,
|
||||
force: bool,
|
||||
lock_file: str,
|
||||
via: object,
|
||||
leave: Callable[[], asyncio.Future[_BackendOutcome[None]]],
|
||||
still_held: Callable[[], bool],
|
||||
) -> None:
|
||||
task = _current_task()
|
||||
with self._lock:
|
||||
if (hold := self._holds.get(task)) is None or via not in hold.depths:
|
||||
if force:
|
||||
return
|
||||
msg = f"Cannot release a lock on {lock_file} that is not held by this task through this wrapper"
|
||||
raise RuntimeError(msg)
|
||||
if not force and hold.depths[via] > 1:
|
||||
hold.depths[via] -= 1
|
||||
return
|
||||
removed = hold.depths.pop(via)
|
||||
if hold.depths:
|
||||
return
|
||||
del self._holds[task]
|
||||
if self._holds:
|
||||
return
|
||||
self._transitioning = True
|
||||
mode = self._mode
|
||||
if not still_held():
|
||||
self._finish_transition(mode=None)
|
||||
msg = f"Cannot release a lock on {lock_file} that is not held by this task"
|
||||
raise RuntimeError(msg)
|
||||
try:
|
||||
cancellation = await _drain_cancellation(leave())
|
||||
except BaseException:
|
||||
hold.depths[via] = removed
|
||||
# Keep the hold for a retried release only while the sync lock still has the level. Restored after the sync
|
||||
# lock let go, it would stand for a lock nobody holds.
|
||||
self._finish_transition(mode=mode if still_held() else None, holds={task: hold})
|
||||
raise
|
||||
self._finish_transition(mode=None)
|
||||
if cancellation is not None:
|
||||
# The release succeeded, so a restored hold would be a phantom no backend call can clear.
|
||||
raise cancellation
|
||||
|
||||
async def retire(
|
||||
self,
|
||||
via: object | None,
|
||||
*,
|
||||
leave: Callable[[], asyncio.Future[_BackendOutcome[None]]],
|
||||
still_held: Callable[[], bool],
|
||||
) -> None:
|
||||
"""
|
||||
End the holds taken through *via*, or every hold when it is ``None``, as one transition.
|
||||
|
||||
Leaves the backend through *leave* when that ends the last hold, and always when *via* is ``None``. Arrivals
|
||||
wait while it runs, so none joins a hold the backend is letting go of. A transition already under way is waited
|
||||
out first: a task entering through *via* must not get a hold after the wrapper is gone. A cancellation is raised
|
||||
once the holds are settled.
|
||||
"""
|
||||
cancellation: asyncio.CancelledError | None = None
|
||||
while isinstance(settled := self._drop_holds_via(via), ConcurrentFuture):
|
||||
try:
|
||||
await _wait_until_done(asyncio.wrap_future(settled))
|
||||
except asyncio.CancelledError as error:
|
||||
cancellation = error
|
||||
finally:
|
||||
self._forget(settled)
|
||||
mode, dropped = settled
|
||||
if via is None or (mode is not None and still_held()):
|
||||
try:
|
||||
cancellation = await _drain_cancellation(leave()) or cancellation
|
||||
except BaseException:
|
||||
# A close that failed with the backend still holding stays retryable, as a failed release does.
|
||||
self._finish_transition(mode=mode if still_held() else None, holds=dropped)
|
||||
raise
|
||||
self._finish_transition(mode=None)
|
||||
elif mode is not None:
|
||||
# Sync code already ended the level taken for tasks, so there is nothing to give back.
|
||||
self._finish_transition(mode=None)
|
||||
if cancellation is not None:
|
||||
raise cancellation
|
||||
|
||||
def _drop_holds_via(
|
||||
self, via: object | None
|
||||
) -> tuple[Literal["read", "write"] | None, dict[asyncio.Task[object], _TaskHold]] | ConcurrentFuture[None]:
|
||||
# Returns the future a running transition settles, or else the mode to leave (None when other holds keep the
|
||||
# backend level) and the dropped holds. The table stays transitioning while that mode is left.
|
||||
with self._lock:
|
||||
if self._transitioning:
|
||||
return self._ticket()
|
||||
if via is None:
|
||||
dropped = dict(self._holds)
|
||||
self._holds.clear()
|
||||
else:
|
||||
dropped = {}
|
||||
for task, hold in list(self._holds.items()):
|
||||
if (count := hold.depths.pop(via, None)) is not None:
|
||||
dropped[task] = _TaskHold(depths={via: count})
|
||||
if not hold.depths:
|
||||
del self._holds[task]
|
||||
if self._holds and via is not None:
|
||||
return None, dropped
|
||||
mode = self._mode
|
||||
self._transitioning = mode is not None or via is None
|
||||
return mode, dropped
|
||||
|
||||
def _finish_transition(
|
||||
self, *, mode: Literal["read", "write"] | None, holds: dict[asyncio.Task[object], _TaskHold] | None = None
|
||||
) -> None:
|
||||
with self._lock:
|
||||
self._transitioning, self._mode = False, mode
|
||||
if mode is not None and holds:
|
||||
for task, hold in holds.items():
|
||||
self._holds.setdefault(task, _TaskHold(depths={})).depths.update(hold.depths)
|
||||
self._notify()
|
||||
|
||||
def _ticket(self) -> ConcurrentFuture[None]:
|
||||
# Called under the table lock, in the section that decided to wait, so no state change slips in between.
|
||||
ticket: ConcurrentFuture[None] = ConcurrentFuture()
|
||||
self._waiters[ticket] = None
|
||||
return ticket
|
||||
|
||||
def _forget(self, ticket: ConcurrentFuture[None]) -> None:
|
||||
with self._lock:
|
||||
self._waiters.pop(ticket, None)
|
||||
|
||||
def _notify(self) -> None:
|
||||
waiters, self._waiters = self._waiters, {}
|
||||
for ticket in waiters:
|
||||
ticket.set_result(None)
|
||||
|
||||
|
||||
_TASK_OWNERS: Final[WeakKeyDictionary[ReadWriteLock | SoftReadWriteLock, _TaskOwners]] = WeakKeyDictionary()
|
||||
_TASK_OWNERS_LOCK: Final[Lock] = Lock()
|
||||
|
||||
|
||||
def _task_owners_for(sync_lock: ReadWriteLock | SoftReadWriteLock) -> _TaskOwners:
|
||||
"""Singleton wrappers share one sync lock, so they must share its task holds too."""
|
||||
with _fork_transition(), _TASK_OWNERS_LOCK:
|
||||
if (owners := _TASK_OWNERS.get(sync_lock)) is None:
|
||||
owners = _TASK_OWNERS[sync_lock] = _TaskOwners()
|
||||
return owners
|
||||
|
||||
|
||||
def _current_task() -> asyncio.Task[object]:
|
||||
if (task := asyncio.current_task()) is None:
|
||||
msg = "an async reader/writer lock must be used from inside an asyncio task"
|
||||
raise RuntimeError(msg)
|
||||
return task
|
||||
|
||||
|
||||
async def _drain_cancellation(future: asyncio.Future[_BackendOutcome[None]]) -> asyncio.CancelledError | None:
|
||||
# Returned, not raised, so the caller can tell a call that completed under cancellation from one that failed.
|
||||
try:
|
||||
await _wait_until_done(future)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
try:
|
||||
await _drain_future(future)
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
_raise_cancelled_error(cancellation, error)
|
||||
return cancellation
|
||||
_future_result(future)
|
||||
return None
|
||||
|
||||
|
||||
async def _drain_future(future: asyncio.Future[_BackendOutcome[_T]]) -> _T:
|
||||
while not future.done():
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await _wait_until_done(future)
|
||||
return _future_result(future)
|
||||
|
||||
|
||||
async def _wait_until_done(future: asyncio.Future[_T]) -> None:
|
||||
if not future.done():
|
||||
await asyncio.wait((future,))
|
||||
|
||||
|
||||
def _future_result(future: asyncio.Future[_BackendOutcome[_T]]) -> _T:
|
||||
outcome = future.result()
|
||||
if (error := outcome.error) is None:
|
||||
return cast("_T", outcome.value)
|
||||
context = error.__context__
|
||||
try:
|
||||
raise error # ruff:ignore[raise-within-try] # the handler restores context changed across the async boundary
|
||||
except BaseException:
|
||||
error.__context__ = context
|
||||
raise
|
||||
|
||||
|
||||
def _run_in_executor(
|
||||
loop: asyncio.AbstractEventLoop, executor: Executor | None, func: Callable[[], _T]
|
||||
) -> asyncio.Future[_BackendOutcome[_T]]:
|
||||
outcome: asyncio.Future[_BackendOutcome[_T]] = loop.create_future()
|
||||
|
||||
def settle(call: asyncio.Future[_BackendOutcome[_T]]) -> None:
|
||||
try:
|
||||
result = call.result()
|
||||
except asyncio.CancelledError:
|
||||
# shutdown(cancel_futures=True) dropped the call unrun: a CancelledError would cancel a task nobody canceled
|
||||
# and hide that the lock state never changed.
|
||||
msg = "the executor shut down before running the lock backend call"
|
||||
result = _BackendOutcome(error=RuntimeError(msg))
|
||||
outcome.set_result(result)
|
||||
|
||||
loop.run_in_executor(executor, _capture_call, func).add_done_callback(settle)
|
||||
return outcome
|
||||
|
||||
|
||||
def _capture_call(func: Callable[[], _T]) -> _BackendOutcome[_T]:
|
||||
try:
|
||||
return _BackendOutcome(value=func())
|
||||
except BaseException as error: # ruff:ignore[blind-except] # backend control-flow exceptions are operation results
|
||||
return _BackendOutcome(error=error)
|
||||
|
||||
|
||||
def _raise_cancelled_error(cancellation: asyncio.CancelledError, error: BaseException) -> NoReturn:
|
||||
# A reconciliation step failed while unwinding a cancellation, so keep both exception chains. Splice the error's
|
||||
# existing context onto the cancellation, then make the cancellation the error's context, so both the failure and
|
||||
# the cancellation that triggered it survive. Shared by the async wrappers so cancellations report the same way.
|
||||
if (context := error.__context__) is not None and context is not cancellation:
|
||||
if (cancellation_context := cancellation.__context__) is not None:
|
||||
_append_exception_context(context, cancellation_context)
|
||||
cancellation.__context__ = context
|
||||
error.__context__ = cancellation
|
||||
_raise_chained_errors(error)
|
||||
|
||||
|
||||
async def _capture_awaitable(awaitable: Awaitable[_T]) -> _BackendOutcome[_T]:
|
||||
try:
|
||||
return _BackendOutcome(value=await awaitable)
|
||||
except BaseException as error: # ruff:ignore[blind-except] # backend cancellation must remain distinct from caller cancellation
|
||||
return _BackendOutcome(error=error)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"_TASKS_THREAD_ID",
|
||||
"_AsyncTransitionGate",
|
||||
"_AsyncTransitionUnavailableError",
|
||||
"_BackendOutcome",
|
||||
"_capture_awaitable",
|
||||
"_capture_call",
|
||||
"_drain_future",
|
||||
"_future_result",
|
||||
"_raise_cancelled_error",
|
||||
"_run_in_executor",
|
||||
"_task_owners_for",
|
||||
"_wait_until_done",
|
||||
]
|
||||
@@ -0,0 +1,413 @@
|
||||
"""Async wrapper around :class:`ReadWriteLock` for use with ``asyncio``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["asyncio", "concurrent.futures", "filelock._async", "filelock._error", "functools", "threading"]
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import thread as executor_threads
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar, cast
|
||||
|
||||
from filelock._api import (
|
||||
_append_exception_context,
|
||||
_ensure_current_process,
|
||||
_fork_transition,
|
||||
_register_fork_object,
|
||||
)
|
||||
from filelock._async import (
|
||||
_TASKS_THREAD_ID,
|
||||
_BackendOutcome,
|
||||
_capture_call,
|
||||
_drain_future,
|
||||
_future_result,
|
||||
_raise_cancelled_error,
|
||||
_task_owners_for,
|
||||
_wait_until_done,
|
||||
)
|
||||
from filelock._error import Timeout
|
||||
from filelock._read_write import ReadWriteLock
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator, Callable, MutableMapping
|
||||
from concurrent import futures
|
||||
from types import TracebackType
|
||||
|
||||
from filelock._api import AcquireReturnProxy
|
||||
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_R = TypeVar("_R")
|
||||
|
||||
|
||||
class AsyncReadWriteLock:
|
||||
"""
|
||||
Async wrapper around :class:`ReadWriteLock` for use in ``asyncio`` applications.
|
||||
|
||||
This wrapper dispatches every blocking SQLite operation to a thread pool via ``loop.run_in_executor()`` because
|
||||
Python's :mod:`sqlite3` module has no async API. Each ``asyncio`` task owns its own hold. Tasks share a read lock,
|
||||
and a task asking for the write lock waits until the other holders release, with waiting writers ahead of new
|
||||
readers. As with :class:`ReadWriteLock`, upgrading or downgrading a held lock raises :class:`RuntimeError`.
|
||||
Singleton wrappers over one path share their task holds, and closing one ends only the holds taken through it.
|
||||
|
||||
:param lock_file: path to the SQLite database file used as the lock
|
||||
:param timeout: maximum wait time in seconds; ``-1`` waits up to SQLite's busy-timeout cap of about 23 days
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
|
||||
:param is_singleton: if ``True``, reuse existing :class:`ReadWriteLock` instances for the same resolved path
|
||||
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
|
||||
:param executor: executor for ``run_in_executor``. When ``None`` this lock creates and owns a dedicated
|
||||
single-thread executor so every operation runs on the same thread (SQLite affinity requires this) and shuts it
|
||||
down in :meth:`close`. This lock uses a caller-supplied executor as-is and never shuts it down, so after passing
|
||||
no executor call :meth:`close` to release the owned one.
|
||||
|
||||
.. versionadded:: 3.21.0
|
||||
|
||||
"""
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
|
||||
self,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = True,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
executor: futures.Executor | None = None,
|
||||
) -> None:
|
||||
creator_pid = os.getpid()
|
||||
self._creator_pid = creator_pid
|
||||
self._fork_invalidated = False
|
||||
self._closed = False
|
||||
self._owns_backend = not is_singleton
|
||||
#: Marks the task holds taken through this wrapper, which its close() ends.
|
||||
self._via: Final = object()
|
||||
_register_fork_object(self)
|
||||
with _fork_transition():
|
||||
self._lock = ReadWriteLock(lock_file, timeout, blocking=blocking, is_singleton=is_singleton)
|
||||
self._owners = _task_owners_for(self._lock)
|
||||
self._loop = loop
|
||||
self._owns_executor = executor is None
|
||||
self._executor = executor or ThreadPoolExecutor(max_workers=1)
|
||||
if os.getpid() != creator_pid: # pragma: forked child
|
||||
msg = "AsyncReadWriteLock construction cannot continue after fork"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
@property
|
||||
def lock_file(self) -> str:
|
||||
"""The path to the lock file."""
|
||||
return self._lock.lock_file
|
||||
|
||||
@property
|
||||
def timeout(self) -> float:
|
||||
"""The default timeout."""
|
||||
return self._lock.timeout
|
||||
|
||||
@property
|
||||
def blocking(self) -> bool:
|
||||
"""Whether blocking is enabled by default."""
|
||||
return self._lock.blocking
|
||||
|
||||
@property
|
||||
def loop(self) -> asyncio.AbstractEventLoop | None:
|
||||
"""The event loop (or ``None`` for the running loop)."""
|
||||
return self._loop
|
||||
|
||||
@property
|
||||
def executor(self) -> futures.Executor:
|
||||
"""The executor used for ``run_in_executor`` (a dedicated single-thread one if none was supplied)."""
|
||||
return self._executor
|
||||
|
||||
@asynccontextmanager
|
||||
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
|
||||
"""
|
||||
Async context manager that acquires and releases a shared read lock.
|
||||
|
||||
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
"""
|
||||
await self.acquire_read(timeout, blocking=blocking)
|
||||
body_error: BaseException | None = None
|
||||
try:
|
||||
yield
|
||||
except BaseException as error:
|
||||
body_error = error
|
||||
raise
|
||||
finally:
|
||||
await self._release_in_context(body_error)
|
||||
|
||||
@asynccontextmanager
|
||||
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
|
||||
"""
|
||||
Async context manager that acquires and releases an exclusive write lock.
|
||||
|
||||
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
"""
|
||||
await self.acquire_write(timeout, blocking=blocking)
|
||||
body_error: BaseException | None = None
|
||||
try:
|
||||
yield
|
||||
except BaseException as error:
|
||||
body_error = error
|
||||
raise
|
||||
finally:
|
||||
await self._release_in_context(body_error)
|
||||
|
||||
async def _release_in_context(self, body_error: BaseException | None) -> None:
|
||||
try:
|
||||
await self.release()
|
||||
except BaseException as release_error:
|
||||
if body_error is not None:
|
||||
_append_exception_context(release_error, body_error)
|
||||
raise
|
||||
|
||||
async def acquire_read(
|
||||
self, timeout: float | None = None, *, blocking: bool | None = None
|
||||
) -> AsyncAcquireReadWriteReturnProxy:
|
||||
"""
|
||||
Acquire a shared read lock.
|
||||
|
||||
See :meth:`ReadWriteLock.acquire_read` for full semantics.
|
||||
|
||||
:param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's
|
||||
busy-timeout cap of about 23 days
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention;
|
||||
``None`` uses the instance setting
|
||||
|
||||
:returns: a proxy that can be used as an async context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the write lock
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self._raise_if_unusable()
|
||||
await self._acquire("read", timeout, blocking=blocking)
|
||||
return AsyncAcquireReadWriteReturnProxy(lock=self)
|
||||
|
||||
async def acquire_write(
|
||||
self, timeout: float | None = None, *, blocking: bool | None = None
|
||||
) -> AsyncAcquireReadWriteReturnProxy:
|
||||
"""
|
||||
Acquire an exclusive write lock.
|
||||
|
||||
See :meth:`ReadWriteLock.acquire_write` for full semantics.
|
||||
|
||||
:param timeout: seconds to wait; ``None`` uses the instance setting; ``-1`` waits up to SQLite's
|
||||
busy-timeout cap of about 23 days
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` on contention;
|
||||
``None`` uses the instance setting
|
||||
|
||||
:returns: a proxy that can be used as an async context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the read lock
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self._raise_if_unusable()
|
||||
await self._acquire("write", timeout, blocking=blocking)
|
||||
return AsyncAcquireReadWriteReturnProxy(lock=self)
|
||||
|
||||
async def _acquire(self, mode: Literal["read", "write"], timeout: float | None, *, blocking: bool | None) -> None:
|
||||
timeout = self.timeout if timeout is None else timeout
|
||||
blocking = self.blocking if blocking is None else blocking
|
||||
|
||||
async def enter(remaining: float) -> None:
|
||||
# A close() may have landed while this task waited for the table.
|
||||
self._raise_if_unusable()
|
||||
await self._run_acquire(
|
||||
functools.partial(
|
||||
self._lock._acquire_level_of, # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
_TASKS_THREAD_ID,
|
||||
mode,
|
||||
remaining,
|
||||
blocking=blocking,
|
||||
)
|
||||
)
|
||||
|
||||
def joinable() -> bool:
|
||||
self._raise_if_unusable()
|
||||
return self._backend_holds()
|
||||
|
||||
await self._owners.acquire(
|
||||
mode,
|
||||
timeout=timeout,
|
||||
blocking=blocking,
|
||||
lock_file=self.lock_file,
|
||||
via=self._via,
|
||||
enter=enter,
|
||||
joinable=joinable,
|
||||
)
|
||||
|
||||
async def release(self, *, force: bool = False) -> None:
|
||||
"""
|
||||
Release one level of the calling task's hold.
|
||||
|
||||
The wrapper releases the database lock once no task holds it.
|
||||
|
||||
:param force: if ``True``, end every level the calling task took through this wrapper. Levels taken through
|
||||
other wrappers on the same path stay. A wrapper that holds no level for the task releases nothing.
|
||||
|
||||
:raises RuntimeError: if the calling task holds no level through this wrapper and *force* is ``False``
|
||||
|
||||
"""
|
||||
_ensure_current_process()
|
||||
if self._inherited: # pragma: needs fork
|
||||
return
|
||||
await self._owners.release(
|
||||
force=force,
|
||||
lock_file=self.lock_file,
|
||||
via=self._via,
|
||||
leave=functools.partial(self._submit, self._leave_backend),
|
||||
still_held=self._backend_holds,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""
|
||||
End the holds of the tasks that acquired through this instance and release the lock once no task holds it.
|
||||
|
||||
After calling this method, the lock instance is no longer usable. With ``is_singleton=False`` it also closes the
|
||||
underlying SQLite connection. A singleton wrapper leaves the shared :class:`ReadWriteLock` open, so tasks that
|
||||
acquired through another wrapper and synchronous users of the same path keep their holds. The next acquire
|
||||
notices a synchronous ``close()`` or forced release of that shared lock. One that lands in the few bytecodes
|
||||
between that check and the admission still admits the task into the ended hold.
|
||||
|
||||
"""
|
||||
_ensure_current_process()
|
||||
if self._inherited: # pragma: needs fork
|
||||
return
|
||||
if self._closed:
|
||||
return
|
||||
# Refuse acquires from here on, so none is admitted into a hold the backend is letting go of.
|
||||
self._closed = True
|
||||
try:
|
||||
await self._owners.retire(
|
||||
None if self._owns_backend else self._via,
|
||||
leave=functools.partial(self._submit, self._lock.close if self._owns_backend else self._leave_backend),
|
||||
still_held=self._backend_holds,
|
||||
)
|
||||
except BaseException as error:
|
||||
# Only a failed backend call can leave the level held, and then a retried close() has to find it.
|
||||
if not isinstance(error, asyncio.CancelledError) and self._backend_holds():
|
||||
self._closed = False
|
||||
else:
|
||||
self._shutdown_owned_executor()
|
||||
raise
|
||||
# Wait for the worker to exit rather than letting it drain in the background: a caller that forks right
|
||||
# after closing deserves a single-threaded process, and os.fork warns about any surviving thread.
|
||||
if self._owns_executor:
|
||||
await asyncio.to_thread(functools.partial(self._executor.shutdown, wait=True))
|
||||
|
||||
async def _run_acquire(self, acquire: Callable[..., AcquireReturnProxy]) -> None:
|
||||
abandoned = threading.Event()
|
||||
acquire_future = self._submit(acquire, cancel_check=abandoned.is_set)
|
||||
try:
|
||||
await _wait_until_done(acquire_future)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
# Stops a backend wait on another process's hold, which would otherwise outlast the caller by as long.
|
||||
abandoned.set()
|
||||
try:
|
||||
await _drain_future(acquire_future)
|
||||
except Timeout:
|
||||
# Nothing was taken, so the caller's cancellation is the whole outcome.
|
||||
raise cancellation from None
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
_raise_cancelled_error(cancellation, error)
|
||||
try:
|
||||
await _drain_future(self._submit(self._leave_backend))
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
_raise_cancelled_error(cancellation, error)
|
||||
raise
|
||||
_future_result(acquire_future)
|
||||
|
||||
def _backend_holds(self) -> bool:
|
||||
# A sync close() or forced release on a shared singleton ends the tasks' level without the table taking part.
|
||||
return self._lock._holds_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
|
||||
def _leave_backend(self) -> None:
|
||||
# Only the tasks' level: a sync user of the same singleton keeps its own.
|
||||
self._lock._release_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
|
||||
def _submit(
|
||||
self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> asyncio.Future[_BackendOutcome[_R]]:
|
||||
return (self._loop or asyncio.get_running_loop()).run_in_executor(
|
||||
self._executor,
|
||||
_capture_call,
|
||||
functools.partial(func, *args, **kwargs),
|
||||
)
|
||||
|
||||
def _shutdown_owned_executor(self) -> None:
|
||||
if self._owns_executor:
|
||||
self._executor.shutdown(wait=False)
|
||||
|
||||
@property
|
||||
def _inherited(self) -> bool:
|
||||
return self._fork_invalidated or os.getpid() != self._creator_pid
|
||||
|
||||
def _raise_if_unusable(self) -> None:
|
||||
_ensure_current_process()
|
||||
if self._inherited: # pragma: needs fork
|
||||
msg = f"AsyncReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance"
|
||||
raise RuntimeError(msg)
|
||||
if self._closed:
|
||||
msg = "Cannot operate on a closed database."
|
||||
raise sqlite3.ProgrammingError(msg)
|
||||
|
||||
def _reset_after_fork_in_child(self) -> None: # pragma: forked child
|
||||
self._fork_invalidated = True
|
||||
if self._owns_executor:
|
||||
# The worker did not survive the fork, yet before 3.12 the child's exit still wakes it through its queue. On
|
||||
# PyPy that queue is pure Python, and its lock may have been held by the worker at fork time, so exit hangs.
|
||||
# CPython 3.12+ forgets every inherited worker in the child; do the same for the one this lock owns.
|
||||
# typeshed declares the registry read-only; it is a WeakKeyDictionary the exit hook iterates.
|
||||
registry = cast(
|
||||
"MutableMapping[threading.Thread, object]",
|
||||
executor_threads._threads_queues, # ruff:ignore[private-member-access] # the exit hook's registry
|
||||
)
|
||||
for worker in cast("ThreadPoolExecutor", self._executor)._threads: # ruff:ignore[private-member-access] # only the executor lists its workers
|
||||
registry.pop(worker, None)
|
||||
|
||||
def __del__(self) -> None:
|
||||
# Safety net when close() was never called: shut down the executor we own so its worker thread does not
|
||||
# outlive the lock. shutdown(wait=False) never blocks.
|
||||
if os.getpid() == getattr(self, "_creator_pid", None) and getattr(self, "_owns_executor", False):
|
||||
self._executor.shutdown(wait=False)
|
||||
|
||||
|
||||
class AsyncAcquireReadWriteReturnProxy:
|
||||
"""Context-aware object that releases the async read/write lock on exit."""
|
||||
|
||||
def __init__(self, lock: AsyncReadWriteLock) -> None:
|
||||
self.lock = lock
|
||||
|
||||
async def __aenter__(self) -> AsyncReadWriteLock:
|
||||
return self.lock
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
await self.lock.release()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AsyncAcquireReadWriteReturnProxy",
|
||||
"AsyncReadWriteLock",
|
||||
]
|
||||
@@ -0,0 +1,76 @@
|
||||
"""A minimal native lock over a caller-owned file descriptor, contending with :class:`FileLock` on the same file."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["filelock._unix", "threading", "typing"]
|
||||
|
||||
import sys
|
||||
import time
|
||||
from threading import TIMEOUT_MAX
|
||||
from typing import Final
|
||||
|
||||
from filelock._api import _seconds
|
||||
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
from filelock._windows import _lock_fd_nonblocking, _unlock_fd
|
||||
else: # pragma: win32 no cover
|
||||
from filelock._unix import _lock_fd_nonblocking, _unlock_fd
|
||||
|
||||
|
||||
def lock_descriptor(fd: int, *, blocking: bool = True, poll_interval: float = 0.05) -> bool:
|
||||
"""
|
||||
Take the native OS lock on *fd*, a file descriptor the caller opened and owns.
|
||||
|
||||
This is the same one-byte exclusive lock :class:`FileLock` uses, so a descriptor lock and a path lock on the same
|
||||
file contend with each other. Unlike :class:`FileLock` it adds no path handling: it never opens, truncates, closes,
|
||||
unlinks, chmods, canonicalizes, or falls back. The caller owns *fd* before, during, and after the call, and must
|
||||
close it. On Windows *fd* must be a synchronous descriptor (its handle not opened with ``FILE_FLAG_OVERLAPPED``).
|
||||
|
||||
For timeout, reentrancy, singleton, lifetime, or stale-break behavior, use :class:`FileLock`. There is no async
|
||||
wrapper. Run this in an executor, or drive ``blocking=False`` from your own polling loop.
|
||||
|
||||
:param fd: an open file descriptor the caller owns.
|
||||
:param blocking: when ``True`` (default), retry the nonblocking attempt every *poll_interval* seconds until it
|
||||
succeeds; when ``False``, make one attempt.
|
||||
:param poll_interval: positive seconds between attempts while blocking, at most :data:`threading.TIMEOUT_MAX`;
|
||||
ignored when *blocking* is ``False``.
|
||||
|
||||
:returns: ``True`` once the lock is held, or ``False`` on contention when ``blocking`` is ``False``.
|
||||
|
||||
:raises OSError: for a permanent native failure, such as an invalid descriptor, or with ``errno.ENOSYS`` when the
|
||||
Python build lacks the native locking primitive. The descriptor is left open.
|
||||
:raises ValueError: if a blocking call receives a *poll_interval* outside that range, including ``nan``.
|
||||
|
||||
.. versionadded:: 3.30.0
|
||||
|
||||
"""
|
||||
if not blocking:
|
||||
return _lock_fd_nonblocking(fd)
|
||||
# time.sleep raises OverflowError past TIMEOUT_MAX; nan fails both comparisons.
|
||||
if not 0 < (poll_interval := _seconds("poll_interval", poll_interval)) <= TIMEOUT_MAX:
|
||||
msg: Final[str] = f"poll_interval must be greater than 0 and at most {TIMEOUT_MAX}, got {poll_interval}"
|
||||
raise ValueError(msg)
|
||||
while not _lock_fd_nonblocking(fd):
|
||||
time.sleep(poll_interval)
|
||||
return True
|
||||
|
||||
|
||||
def unlock_descriptor(fd: int) -> None:
|
||||
"""
|
||||
Release the native OS lock on *fd* without touching the descriptor.
|
||||
|
||||
:param fd: the descriptor a prior :func:`lock_descriptor` locked; the caller still owns and must close it.
|
||||
|
||||
:raises OSError: if the native unlock fails, including ``errno.ENOSYS`` when the Python build lacks the native
|
||||
locking primitive; the caller may retry on the same descriptor.
|
||||
|
||||
.. versionadded:: 3.30.0
|
||||
|
||||
"""
|
||||
_unlock_fd(fd)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"lock_descriptor",
|
||||
"unlock_descriptor",
|
||||
]
|
||||
@@ -0,0 +1,90 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["typing"]
|
||||
|
||||
from typing import Final
|
||||
|
||||
|
||||
class Timeout(TimeoutError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API
|
||||
"""Raised when the lock could not be acquired in *timeout* seconds."""
|
||||
|
||||
def __init__(self, lock_file: str, holder: str | None = None) -> None:
|
||||
super().__init__()
|
||||
self._lock_file = lock_file
|
||||
self._holder = holder
|
||||
|
||||
def __reduce__(self) -> tuple[type[Timeout], tuple[str], dict[str, object]]:
|
||||
# holder rides in the state so a subclass taking only lock_file, and a 4.0.12 reader, can still unpickle it.
|
||||
return self.__class__, (self._lock_file,), self.__dict__
|
||||
|
||||
def __str__(self) -> str: # pragma: needs hard-link
|
||||
message: Final = f"The file lock '{self._lock_file}' could not be acquired."
|
||||
if self._holder is None:
|
||||
return message
|
||||
return f"{message} It is held by {self._holder}, which this host cannot probe."
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}({self.lock_file!r})"
|
||||
|
||||
@property
|
||||
def lock_file(self) -> str:
|
||||
"""The path of the file lock."""
|
||||
return self._lock_file
|
||||
|
||||
|
||||
class SoftFileLockLifetimeWarning(FutureWarning):
|
||||
"""
|
||||
The configured soft-lock lifetime permits overlapping live holders after expiry.
|
||||
|
||||
A :class:`FutureWarning` shows by default wherever the lock is configured; a :class:`DeprecationWarning` shows only
|
||||
in ``__main__``.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class LeaseSettingsMismatch(ValueError): # ruff:ignore[error-suffix-on-exception-name] # public exception name; renaming breaks the API
|
||||
"""A lease contender disagrees with the published claim about how long the lease lasts."""
|
||||
|
||||
|
||||
class SoftFileLockProtocolError(OSError):
|
||||
"""Raised when strict soft-lock state cannot be interpreted without risking overlap."""
|
||||
|
||||
def __init__(self, lock_file: str, claim_name: str | None, reason: str) -> None:
|
||||
self._lock_file = lock_file
|
||||
self._claim_name = claim_name
|
||||
self._reason = reason
|
||||
super().__init__(self.__str__())
|
||||
|
||||
def __reduce__(
|
||||
self,
|
||||
) -> tuple[
|
||||
type[SoftFileLockProtocolError], tuple[str, str | None, str], dict[str, object]
|
||||
]: # pragma: needs hard-link
|
||||
return self.__class__, (self._lock_file, self._claim_name, self._reason), self.__dict__
|
||||
|
||||
def __str__(self) -> str:
|
||||
location = self._lock_file if self._claim_name is None else f"{self._lock_file}: claim {self._claim_name!r}"
|
||||
return f"Invalid strict soft-lock state at {location}: {self._reason}"
|
||||
|
||||
@property
|
||||
def lock_file(self) -> str: # pragma: needs hard-link
|
||||
"""The requested lock path."""
|
||||
return self._lock_file
|
||||
|
||||
@property
|
||||
def claim_name(self) -> str | None: # pragma: needs hard-link
|
||||
"""The claim that caused the error, if scanning identified one."""
|
||||
return self._claim_name
|
||||
|
||||
@property
|
||||
def reason(self) -> str: # pragma: needs hard-link
|
||||
"""The protocol validation failure."""
|
||||
return self._reason
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LeaseSettingsMismatch",
|
||||
"SoftFileLockLifetimeWarning",
|
||||
"SoftFileLockProtocolError",
|
||||
"Timeout",
|
||||
]
|
||||
@@ -0,0 +1,328 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["hashlib", "pathlib", "socket", "typing"]
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
from errno import EPERM, ESRCH
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
#: Every marker format in this package caps the hostname at the RFC 1123 limit and reads a longer one as malformed.
|
||||
_HOST_NAME_LIMIT: Final[int] = 253
|
||||
#: The bytes those formats carry verbatim: printable non-space ASCII, less the ``?`` reserved for the escape.
|
||||
_HOST_NAME_VERBATIM: Final[frozenset[int]] = frozenset(range(0x21, 0x7F)) - {ord("?")}
|
||||
#: The characters at the end of an over-long name that carry a digest of the part cut off.
|
||||
_HOST_NAME_DIGEST_LEN: Final[int] = 8
|
||||
#: The kernel ABI fixes the initial PID namespace's inode (``PID_NS_INIT_INO`` in ``linux/nsfs.h``); every namespace
|
||||
#: created later draws one from ``0xF0000000`` up, so this value names the host's own namespace.
|
||||
_INITIAL_PID_NAMESPACE: Final[int] = 0xEFFFFFFC
|
||||
_NAMESPACE_MARK: Final[str] = "?pidns-"
|
||||
|
||||
|
||||
def host_name() -> str:
|
||||
"""
|
||||
The hostname recorded alongside an owner, so a marker written on another machine is never probed here.
|
||||
|
||||
Marker formats here hold the field to printable non-space ASCII, while ``socket.gethostname()`` reports whatever
|
||||
the kernel stores: a space, a newline that forges an extra marker line, or a byte no codec encodes, which Python
|
||||
hands back as a surrogate. A holder that publishes one raw writes a marker it cannot read back, and loses the lock
|
||||
to the first peer that ages it out as malformed. ``?`` is illegal in a hostname, so escaping it along with every
|
||||
out-of-grammar byte as ``?<hex>`` leaves a real name untouched and still keeps two hosts apart, which
|
||||
:func:`owner_is_stale` relies on to refuse to probe a foreign PID.
|
||||
|
||||
Linux appends the PID namespace, as Mercurial's lock prefix does. Two containers in one pod share the hostname and
|
||||
the boot id but not the PID namespace, so a PID recorded in one names another process, or none, in the other. The
|
||||
suffix starts with a ``?`` no escape can produce, so ``a`` in a namespace never reads as a host literally named
|
||||
``a?pidns-...``. The initial namespace adds nothing, which keeps a host's name what older releases recorded.
|
||||
|
||||
An over-long name ends in a digest of the part cut to fit the limit. A bare cut gives every name that shares the
|
||||
kept prefix one identity, and :func:`owner_is_stale` would then probe a local PID for another host's holder.
|
||||
"""
|
||||
name = (
|
||||
"".join(
|
||||
chr(byte) if byte in _HOST_NAME_VERBATIM else f"?{byte:02x}"
|
||||
for byte in socket.gethostname().encode("utf-8", "surrogateescape")
|
||||
)
|
||||
+ _pid_namespace_suffix()
|
||||
)
|
||||
if len(name) > _HOST_NAME_LIMIT:
|
||||
keep = _HOST_NAME_LIMIT - _HOST_NAME_DIGEST_LEN - 1
|
||||
name = f"{name[:keep]}-{hashlib.sha256(name[keep:].encode()).hexdigest()[:_HOST_NAME_DIGEST_LEN]}"
|
||||
# An escape is three characters and a kept byte is never '?', so a bare '?' can only mean an empty hostname.
|
||||
return name or "?"
|
||||
|
||||
|
||||
def _pid_namespace_suffix() -> str:
|
||||
try:
|
||||
namespace: Final = Path("/proc/self/ns/pid").stat().st_ino
|
||||
except OSError: # only Linux 3.8+ has the file, and a sandbox may hide it; nothing else tells namespaces apart
|
||||
return ""
|
||||
return "" if namespace == _INITIAL_PID_NAMESPACE else f"{_NAMESPACE_MARK}{namespace:x}"
|
||||
|
||||
|
||||
def owner_is_stale(pid: int, hostname: str, start_token: int | None) -> bool:
|
||||
"""
|
||||
Whether the recorded owner is provably gone, so reclaiming its marker cannot detach a live holder.
|
||||
|
||||
Fail closed: return ``True`` only when this process can prove the exact recorded owner is dead. A marker from
|
||||
another host cannot be probed; a live PID whose start token still matches, or whose token cannot be read, is the
|
||||
holder or is indistinguishable from it; a live PID whose start token differs is a recycled PID, so the process that
|
||||
wrote the marker is gone. PostgreSQL, Qt ``QLockFile`` and Mercurial all break a stale lock only on proof of death
|
||||
and treat an unreadable or foreign owner as still holding.
|
||||
"""
|
||||
if hostname != host_name():
|
||||
return False
|
||||
if process_alive(pid) and (
|
||||
start_token is None or (current := process_start_token(pid)) is None or _same_start(current, start_token)
|
||||
):
|
||||
return False
|
||||
# Only a bare hostname can be an older release's, written from a container that shares this host's name.
|
||||
return _NAMESPACE_MARK in hostname or not _owner_runs_elsewhere(pid, start_token)
|
||||
|
||||
|
||||
def owner_is_current_process(pid: int, hostname: str, start_token: int | None) -> bool:
|
||||
"""
|
||||
Whether the recorded owner is this process, so a marker a recycled PID left behind never reads as ours.
|
||||
|
||||
Fail closed: a recorded start token must match this process's. One that differs was written by an earlier process
|
||||
that reused this PID, and when this process cannot read its own token nothing proves the marker is ours. A marker
|
||||
without a token falls back to the PID and hostname.
|
||||
"""
|
||||
if pid != os.getpid() or hostname != host_name():
|
||||
return False
|
||||
return start_token is None or start_token == process_start_token(pid)
|
||||
|
||||
|
||||
def _same_start(current: int, recorded: int) -> bool:
|
||||
# A Linux token carries the boot id in its high 64 bits, or 0 there when the process could not read the boot id.
|
||||
# systemd's ProcSubset=pid hides /proc/sys from one unit, so one process may lack the boot id while its peer has
|
||||
# it. A missing boot id then proves nothing, and only starttime is compared. Other platforms' tokens fit in the
|
||||
# low bits.
|
||||
return current == recorded or (0 in {current >> 64, recorded >> 64} and current % 2**64 == recorded % 2**64)
|
||||
|
||||
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
_KERNEL32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
_KERNEL32.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
_KERNEL32.CloseHandle.restype = wintypes.BOOL
|
||||
_KERNEL32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
||||
_KERNEL32.OpenProcess.restype = wintypes.HANDLE
|
||||
_KERNEL32.GetProcessTimes.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
ctypes.POINTER(wintypes.FILETIME),
|
||||
ctypes.POINTER(wintypes.FILETIME),
|
||||
ctypes.POINTER(wintypes.FILETIME),
|
||||
ctypes.POINTER(wintypes.FILETIME),
|
||||
]
|
||||
_KERNEL32.GetProcessTimes.restype = wintypes.BOOL
|
||||
|
||||
_WIN_SYNCHRONIZE: Final[int] = 0x100000
|
||||
_WIN_PROCESS_QUERY_LIMITED_INFORMATION: Final[int] = 0x1000
|
||||
_WIN_ERROR_INVALID_PARAMETER: Final[int] = 87
|
||||
_WIN_INHERIT_HANDLE: Final[bool] = False
|
||||
|
||||
def process_alive(pid: int) -> bool:
|
||||
"""Whether a process with this PID exists, treating an access denial as proof it does."""
|
||||
handle = _KERNEL32.OpenProcess(_WIN_SYNCHRONIZE, _WIN_INHERIT_HANDLE, pid)
|
||||
if handle:
|
||||
_KERNEL32.CloseHandle(handle)
|
||||
return True
|
||||
return ctypes.get_last_error() != _WIN_ERROR_INVALID_PARAMETER
|
||||
|
||||
def process_start_token(pid: int) -> int | None:
|
||||
"""The process creation FILETIME as a 100ns tick count, or ``None`` when it cannot be read."""
|
||||
handle = _KERNEL32.OpenProcess(_WIN_PROCESS_QUERY_LIMITED_INFORMATION, _WIN_INHERIT_HANDLE, pid)
|
||||
if not handle:
|
||||
return None
|
||||
creation, exit_time, kernel_time, user_time = (wintypes.FILETIME() for _ in range(4))
|
||||
try:
|
||||
if not _KERNEL32.GetProcessTimes(
|
||||
handle,
|
||||
ctypes.byref(creation),
|
||||
ctypes.byref(exit_time),
|
||||
ctypes.byref(kernel_time),
|
||||
ctypes.byref(user_time),
|
||||
):
|
||||
return None # pragma: no cover # win32 GetProcessTimes failure path; not reproducible on a live handle
|
||||
finally:
|
||||
_KERNEL32.CloseHandle(handle)
|
||||
return (creation.dwHighDateTime << 32) | creation.dwLowDateTime
|
||||
|
||||
else: # pragma: win32 no cover
|
||||
|
||||
def process_alive(pid: int) -> bool:
|
||||
"""Whether a process with this PID exists, treating an access denial (``EPERM``) as proof it does."""
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError as error:
|
||||
if error.errno == ESRCH:
|
||||
return False
|
||||
if error.errno == EPERM:
|
||||
return True
|
||||
raise
|
||||
return True
|
||||
|
||||
if sys.platform in {"linux", "android"}: # pragma: linux cover
|
||||
# Termux/Android reports sys.platform == "android" but runs the Linux kernel, so /proc/<pid>/stat and the boot
|
||||
# id are the same reliable start-time source; treat it exactly like Linux rather than the tokenless fallback.
|
||||
# comm (field 2) is wrapped in parentheses and may itself contain spaces or a ')', so the fixed fields start
|
||||
# after the final ')'. starttime is field 22 overall, the twentieth of those trailing fields (index 19).
|
||||
_STARTTIME_INDEX: Final[int] = 19
|
||||
|
||||
def _read_boot_id() -> int:
|
||||
# starttime is measured in clock ticks since boot, so on its own it repeats across a reboot. Folding the
|
||||
# boot id into the high bits makes the Linux token reboot-safe like the absolute clocks macOS and Windows
|
||||
# expose, while staying a single integer so a 3.29 reader still parses the third marker line. 0 when this
|
||||
# process cannot read the boot id leaves the bare starttime, which _same_start compares alone.
|
||||
try:
|
||||
boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="ascii")
|
||||
return int(boot_id.strip().replace("-", ""), 16)
|
||||
except (OSError, ValueError):
|
||||
return 0
|
||||
|
||||
_BOOT_ID: Final[int] = _read_boot_id()
|
||||
#: ``TIME_NS_INIT_INO`` in ``linux/proc_ns.h``, fixed by the kernel ABI like the initial PID namespace's inode.
|
||||
_INITIAL_TIME_NAMESPACE: Final[int] = 0xEFFFFFFA
|
||||
|
||||
def process_start_token(pid: int) -> int | None:
|
||||
"""
|
||||
The ``/proc/<pid>/stat`` ``starttime`` folded with the boot id.
|
||||
|
||||
``None`` when the process is gone, or when this process runs in a time namespace with a boottime offset.
|
||||
Since Linux 5.6 the kernel shifts ``starttime`` by the reader's offset, so a value read there differs
|
||||
from what a peer outside reads.
|
||||
"""
|
||||
return _read_start_token(pid) if _starttime_unshifted() else None
|
||||
|
||||
def _starttime_unshifted() -> bool:
|
||||
# setns can move this process into another time namespace, so ask on every read.
|
||||
try:
|
||||
namespace: Final = Path("/proc/self/ns/time").stat().st_ino
|
||||
except FileNotFoundError: # no entry means no time namespaces on this kernel
|
||||
return True
|
||||
except OSError: # any other failure leaves the namespace unknown, so starttime is not comparable
|
||||
return False
|
||||
if namespace == _INITIAL_TIME_NAMESPACE:
|
||||
return True
|
||||
# A container runtime may give each container a time namespace without offsets. timens_offsets describes
|
||||
# the namespace this process's children get, its own unless it unshared one since.
|
||||
try:
|
||||
if Path("/proc/self/ns/time_for_children").stat().st_ino != namespace:
|
||||
return False
|
||||
offsets: Final = Path("/proc/self/timens_offsets").read_text(encoding="ascii")
|
||||
except OSError:
|
||||
return False
|
||||
# The clock goes by name since Linux 5.8 and by clock id before.
|
||||
return any(line.split() in (["boottime", "0", "0"], ["7", "0", "0"]) for line in offsets.splitlines())
|
||||
|
||||
def _read_start_token(pid: int) -> int | None:
|
||||
try:
|
||||
data = Path(f"/proc/{pid}/stat").read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
# psutil identifies a process by the same (pid, starttime) pair; the boot id extends that across reboots.
|
||||
fields = data[data.rfind(b")") + 1 :].split()
|
||||
if len(fields) <= _STARTTIME_INDEX: # pragma: no cover # a truncated /proc read, never seen in practice
|
||||
return None
|
||||
try:
|
||||
starttime = int(fields[_STARTTIME_INDEX])
|
||||
except ValueError: # pragma: no cover # /proc always renders starttime as an integer
|
||||
return None
|
||||
return (_BOOT_ID << 64) | starttime
|
||||
|
||||
def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool:
|
||||
"""
|
||||
Whether the owner runs in another PID namespace as *pid*.
|
||||
|
||||
An older release in a container sharing this hostname recorded its in-container PID, which means nothing
|
||||
here. Readers without a boottime offset read the same starttime and boot id. A task with the recorded
|
||||
start token whose innermost PID is *pid* is therefore the owner; one that started in the same tick is not.
|
||||
A reader with an offset cannot compare start tokens, so the innermost PID alone keeps the marker.
|
||||
"""
|
||||
if start_token is None or (_BOOT_ID and start_token >> 64 not in {0, _BOOT_ID}):
|
||||
return False
|
||||
unshifted: Final = _starttime_unshifted()
|
||||
try:
|
||||
with os.scandir("/proc") as entries:
|
||||
return any(
|
||||
entry.name.isdigit()
|
||||
and (task := int(entry.name)) != pid
|
||||
and (
|
||||
not unshifted
|
||||
or ((token := _read_start_token(task)) is not None and _same_start(token, start_token))
|
||||
)
|
||||
and _innermost_pid(task) == pid
|
||||
for entry in entries
|
||||
)
|
||||
except OSError:
|
||||
# A /proc this process cannot list proves nothing either way, which leaves 4.0.12's verdict.
|
||||
return False
|
||||
|
||||
def _innermost_pid(task: int) -> int | None:
|
||||
try:
|
||||
status: Final = Path(f"/proc/{task}/status").read_bytes()
|
||||
except OSError: # pragma: no cover # the task exited between the stat read and this one
|
||||
return None
|
||||
# The kernel ends a line only with "\n". A task names itself, and its Name line keeps "\r", "\v" and the
|
||||
# other separators that a text-mode read and str.splitlines() split at. A name could then forge a line.
|
||||
return next(
|
||||
(int(line.split()[-1]) for line in status.split(b"\n") if line.startswith(b"NSpid:")),
|
||||
None, # kernels before 4.1 record no namespace PIDs
|
||||
)
|
||||
|
||||
elif sys.platform == "darwin": # pragma: darwin cover
|
||||
import ctypes
|
||||
import struct
|
||||
|
||||
_LIBC: Final[ctypes.CDLL] = ctypes.CDLL(None, use_errno=True)
|
||||
_CTL_KERN: Final[int] = 1
|
||||
_KERN_PROC: Final[int] = 14
|
||||
_KERN_PROC_PID: Final[int] = 1
|
||||
# kinfo_proc opens with kp_proc.p_starttime (a struct timeval) at offset 0: int64 seconds, int32 microseconds.
|
||||
# The offset is fixed by the struct chain kinfo_proc -> extern_proc -> p_un, so a read at 0 is not a guess.
|
||||
_TIMEVAL_AT_ZERO: Final[str] = "<qi"
|
||||
_TIMEVAL_SIZE: Final[int] = struct.calcsize(_TIMEVAL_AT_ZERO)
|
||||
|
||||
def process_start_token(pid: int) -> int | None:
|
||||
"""The process start time in microseconds from ``sysctl(KERN_PROC_PID)``, or ``None`` when it is gone."""
|
||||
mib = (ctypes.c_int * 4)(_CTL_KERN, _KERN_PROC, _KERN_PROC_PID, pid)
|
||||
length = ctypes.c_size_t(0)
|
||||
# The size probe reports the kinfo_proc size for any PID, so the read below, not the probe, tells a live
|
||||
# process from a gone one: a gone PID leaves the fetch a zero-length success, so re-check the length after.
|
||||
if _LIBC.sysctl(mib, 4, None, ctypes.byref(length), None, 0) != 0:
|
||||
return None
|
||||
buffer = (ctypes.c_char * length.value)()
|
||||
if _LIBC.sysctl(mib, 4, buffer, ctypes.byref(length), None, 0) != 0 or length.value < _TIMEVAL_SIZE:
|
||||
return None
|
||||
seconds, microseconds = struct.unpack_from(_TIMEVAL_AT_ZERO, buffer.raw, 0)
|
||||
return seconds * 1_000_000 + microseconds
|
||||
|
||||
else: # pragma: no cover # a POSIX platform without a proven start-time source falls back to fail-closed liveness
|
||||
|
||||
def process_start_token(pid: int) -> int | None:
|
||||
"""No proven start-time source, so the owner carries no token and liveness rests on the PID alone."""
|
||||
del pid
|
||||
return None
|
||||
|
||||
|
||||
if sys.platform not in {"linux", "android"}: # pragma: linux no cover # pragma: android no cover
|
||||
|
||||
def _owner_runs_elsewhere(pid: int, start_token: int | None) -> bool:
|
||||
"""Without Linux PID namespaces, a PID that is gone or recycled leaves its owner nowhere else to run."""
|
||||
del pid, start_token
|
||||
return False
|
||||
|
||||
|
||||
__all__ = [
|
||||
"host_name",
|
||||
"owner_is_current_process",
|
||||
"owner_is_stale",
|
||||
"process_alive",
|
||||
"process_start_token",
|
||||
]
|
||||
@@ -0,0 +1,402 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = [
|
||||
"contextlib",
|
||||
"filelock._error",
|
||||
"filelock._identity",
|
||||
"filelock._soft",
|
||||
"filelock._util",
|
||||
"math",
|
||||
"secrets",
|
||||
"threading",
|
||||
"weakref",
|
||||
]
|
||||
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from threading import Event, Thread, current_thread
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
from weakref import WeakMethod
|
||||
|
||||
from filelock._api import _check_timeout_max, _seconds
|
||||
from filelock._error import LeaseSettingsMismatch
|
||||
from filelock._identity import owner_is_stale
|
||||
from filelock._marker import MarkerSoftFileLock, OwnerMode, OwnerRecord, parse_marker
|
||||
from filelock._soft import _read_lock_file
|
||||
from filelock._util import pin_inode, touch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Unpack
|
||||
|
||||
from filelock._api import LockOptions, _ExtraValue
|
||||
|
||||
#: ``evicted`` is a :class:`~filelock.SoftReadWriteLock` loss: a peer waited out the stale threshold and committed a
|
||||
#: generation without this holder.
|
||||
CompromiseReason = Literal["marker-missing", "owner-changed", "refresh-failed", "evicted"]
|
||||
|
||||
_RefreshOutcome = Literal["ok", "transient", "marker-missing", "owner-changed"]
|
||||
_DEFAULT_LEASE_DURATION: Final[float] = 30.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LeaseCompromise:
|
||||
"""Why a held lease stopped being this process's to hold, and the error that ended its refresh, if any."""
|
||||
|
||||
lock_file: str
|
||||
token: str
|
||||
reason: CompromiseReason
|
||||
error: Exception | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Heartbeat:
|
||||
"""A running heartbeat and the event that stops it, which only ever exist together."""
|
||||
|
||||
thread: Thread
|
||||
stop: Event
|
||||
|
||||
|
||||
@dataclass
|
||||
class _LeaseClaim:
|
||||
"""The state of one claim: its token, its heartbeat, and how that claim was lost."""
|
||||
|
||||
token: str | None = None
|
||||
compromise: LeaseCompromise | None = None
|
||||
heartbeat: _Heartbeat | None = None
|
||||
|
||||
|
||||
class SoftFileLease(MarkerSoftFileLock):
|
||||
"""
|
||||
Existence lock whose claim expires, so a peer may take it while the previous holder still runs.
|
||||
|
||||
A lease trades mutual exclusion for progress. The holder publishes a claim and refreshes it every
|
||||
``heartbeat_interval`` seconds; a contender takes the marker once it has seen no refresh for ``lease_duration``
|
||||
seconds of its own monotonic clock, so clock skew between hosts cannot expire a live claim. Nothing stops the
|
||||
expired holder: it keeps running, and it keeps using whatever the lock protects. Treat the lease as a hint about
|
||||
who *should* be working, not as a guarantee that only one worker is.
|
||||
|
||||
To make a protected resource reject a superseded holder, that resource must be linearizable and must fence on a
|
||||
monotonic generation it controls. :attr:`token` names a claim; it does not fence one. Where overlap is unacceptable,
|
||||
use :class:`StrictSoftFileLock <filelock.StrictSoftFileLock>` instead.
|
||||
|
||||
Every contender for a path must agree on ``lease_duration``. A contender that finds a claim published under a
|
||||
different duration raises :class:`LeaseSettingsMismatch <filelock.LeaseSettingsMismatch>` rather than apply its own
|
||||
expiry to a peer that never agreed to it.
|
||||
|
||||
Expiry reclaims less on Windows, which refuses to rename or delete a file another process holds open. A peer there
|
||||
takes an expired claim only once the previous holder's process exits and its handle closes; a holder that lives on
|
||||
but stops refreshing keeps the marker. Unix reclaims the marker either way.
|
||||
|
||||
``on_compromise`` fires from the heartbeat thread when a refresh fails, or when the marker vanishes or names another
|
||||
owner. The holder should stop touching the protected resource when it runs. Because it runs on that thread, a
|
||||
``release()`` inside it only takes effect when the lease was built with ``thread_local=False``; the default
|
||||
thread-local context hides the claim from every thread but the one that acquired it, so the release does nothing.
|
||||
Signal the acquiring thread instead when the context stays thread-local.
|
||||
|
||||
.. versionadded:: 3.30.0
|
||||
|
||||
"""
|
||||
|
||||
_owner_mode: OwnerMode = "lease"
|
||||
|
||||
#: lease_duration replaces the legacy age-based lifetime, so accepting both would give one lock two expiry clocks.
|
||||
_lifetime_supported: bool = False
|
||||
_lifetime_unsupported_reason: str = "lease_duration sets when a lease expires"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
lock_file: str | os.PathLike[str],
|
||||
*,
|
||||
lease_duration: float = _DEFAULT_LEASE_DURATION,
|
||||
heartbeat_interval: float | None = None,
|
||||
on_compromise: Callable[[LeaseCompromise], None] | None = None,
|
||||
**kwargs: Unpack[LockOptions],
|
||||
) -> None:
|
||||
"""
|
||||
Create a lease.
|
||||
|
||||
:param lease_duration: seconds a contender must see the marker go unrefreshed before it may take the claim.
|
||||
Every contender for the path must pass the same value.
|
||||
:param heartbeat_interval: seconds between refreshes. Defaults to a third of ``lease_duration``, leaving room
|
||||
for two missed refreshes before a peer may take the claim. Must be shorter than ``lease_duration`` and at
|
||||
most :data:`threading.TIMEOUT_MAX`.
|
||||
:param on_compromise: called from the heartbeat thread with a :class:`LeaseCompromise` when the claim is lost.
|
||||
:param kwargs: every other :class:`BaseFileLock <filelock.BaseFileLock>` option, ``timeout`` and ``mode`` among
|
||||
them. The metaclass passes them all by keyword, and taking them here lets
|
||||
:class:`AsyncSoftFileLease <filelock.AsyncSoftFileLease>` add the async plumbing a fixed signature would
|
||||
hide.
|
||||
|
||||
"""
|
||||
# A plain float: the marker records its repr, which NumPy's float64 spells in a form no parser reads.
|
||||
if not isfinite(lease_duration := _seconds("lease_duration", lease_duration)) or lease_duration <= 0:
|
||||
msg = f"lease_duration must be positive and finite, got {lease_duration!r}"
|
||||
raise ValueError(msg)
|
||||
heartbeat_interval = (
|
||||
lease_duration / 3 if heartbeat_interval is None else _seconds("heartbeat_interval", heartbeat_interval)
|
||||
)
|
||||
if not 0 < heartbeat_interval < lease_duration:
|
||||
msg = f"heartbeat_interval must be positive and below lease_duration, got {heartbeat_interval!r}"
|
||||
raise ValueError(msg)
|
||||
_check_timeout_max("heartbeat_interval", heartbeat_interval)
|
||||
super().__init__(lock_file, **kwargs)
|
||||
self._lease_duration = lease_duration
|
||||
self._heartbeat_interval = heartbeat_interval
|
||||
self._on_compromise = on_compromise
|
||||
# Two threads lazily creating a shared claim could each get one, and release() would miss the heartbeat's.
|
||||
if not self.is_thread_local():
|
||||
self._context.lease_claim = _LeaseClaim()
|
||||
|
||||
def _singleton_extra_mismatches(self, kwargs: Mapping[str, _ExtraValue], /) -> dict[str, tuple[str, str]]:
|
||||
mismatches = super()._singleton_extra_mismatches(kwargs)
|
||||
lease_duration: Final = _seconds(
|
||||
"lease_duration", cast("float", kwargs.get("lease_duration", _DEFAULT_LEASE_DURATION))
|
||||
)
|
||||
if lease_duration != self._lease_duration:
|
||||
mismatches["lease_duration"] = (str(lease_duration), str(self._lease_duration))
|
||||
# An omitted heartbeat_interval resolves as in __init__; a differing duration is already reported above.
|
||||
heartbeat_interval: Final = (
|
||||
self._lease_duration / 3
|
||||
if (passed := kwargs.get("heartbeat_interval")) is None
|
||||
else _seconds("heartbeat_interval", cast("float", passed))
|
||||
)
|
||||
if heartbeat_interval != self._heartbeat_interval:
|
||||
mismatches["heartbeat_interval"] = (str(passed), str(self._heartbeat_interval))
|
||||
# A callback compares by identity, as on_acquired does: two equal callables can close over different state.
|
||||
if (on_compromise := kwargs.get("on_compromise")) is not self._on_compromise:
|
||||
mismatches["on_compromise"] = (str(on_compromise), str(self._on_compromise))
|
||||
return mismatches
|
||||
|
||||
@property
|
||||
def _claim(self) -> _LeaseClaim:
|
||||
# Per hold, so another thread's failed acquire cannot stop the holder's heartbeat.
|
||||
state: Final = self._hold_state()
|
||||
if (claim := state.lease_claim) is None:
|
||||
claim = state.lease_claim = _LeaseClaim()
|
||||
return claim
|
||||
|
||||
@property
|
||||
def lease_duration(self) -> float:
|
||||
"""The staleness in seconds after which a contender may take this claim."""
|
||||
return self._lease_duration
|
||||
|
||||
@property
|
||||
def token(self) -> str | None:
|
||||
"""
|
||||
The token naming the claim this process published.
|
||||
|
||||
:returns: the token while the lease is held, ``None`` otherwise. It identifies a claim; it does not fence one.
|
||||
|
||||
"""
|
||||
return self._claim.token
|
||||
|
||||
@property
|
||||
def compromise(self) -> LeaseCompromise | None:
|
||||
"""
|
||||
The loss of claim the heartbeat observed.
|
||||
|
||||
The value outlives ``release()``, so a holder can check after leaving the ``with`` block whether its work may
|
||||
have overlapped a successor; the next acquisition resets it, as :attr:`SoftReadWriteLock.compromise
|
||||
<filelock.SoftReadWriteLock.compromise>` does.
|
||||
|
||||
:returns: the :class:`LeaseCompromise` of the latest claim, or ``None`` while that claim holds or held to its
|
||||
release
|
||||
|
||||
"""
|
||||
return self._claim.compromise
|
||||
|
||||
def _acquire(self) -> None:
|
||||
claim = self._claim
|
||||
self._stop_heartbeat() # no earlier claim's heartbeat outlives the acquisition of the next one
|
||||
# The published record reads the token, so it exists before the marker is written and goes back on failure.
|
||||
claim.token = token = secrets.token_hex(16)
|
||||
claim.compromise = None
|
||||
try:
|
||||
super()._acquire()
|
||||
except BaseException:
|
||||
claim.token = None
|
||||
raise
|
||||
# The context is thread-local by default, so the heartbeat thread cannot read the descriptor this one just
|
||||
# published, nor the claim this one owns. Hand it the fd, the inode it verified and the claim instead.
|
||||
if (fd := self._context.lock_file_fd) is not None and (
|
||||
identity := self._context.lock_file_fd_identity
|
||||
) is not None:
|
||||
self._start_heartbeat(claim, fd, identity, token)
|
||||
else:
|
||||
claim.token = None
|
||||
|
||||
def _release(self) -> None:
|
||||
self._stop_heartbeat()
|
||||
self._claim.token = None
|
||||
super()._release()
|
||||
|
||||
def _published_record(self) -> OwnerRecord:
|
||||
return super()._published_record()._replace(token=self._claim.token, lease_duration=self._lease_duration)
|
||||
|
||||
def _try_break_stale_lock(self) -> None:
|
||||
# A break can fail for reasons a contender must ride out rather than raise on: a peer broke the marker first,
|
||||
# or Windows refuses to rename a file whose holder still has it open. Poll again instead.
|
||||
with suppress(OSError), pin_inode(self._lock_path) as pinned:
|
||||
self._try_break_pinned_lock(pinned)
|
||||
|
||||
def _try_break_pinned_lock(self, pinned: int | None) -> None:
|
||||
if (peer := self._read_peer()) is None:
|
||||
# Not a readable protocol 2 lease record: a partial write, a foreign or legacy protocol 1 marker, or the
|
||||
# strict sentinel. The base self-heal evicts a genuinely malformed marker once it ages past the grace
|
||||
# window and leaves a legitimate legacy or strict holder in place, so a corrupt marker no longer wedges
|
||||
# every lease contender until its own timeout.
|
||||
super()._try_break_stale_lock()
|
||||
return
|
||||
owner, marker = peer
|
||||
if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing
|
||||
return # replaced before the read, so nothing keeps the number read from reuse
|
||||
# Only a peer that published a lease agreed to be superseded by age; an exclusive owner is reclaimed only once
|
||||
# it is provably dead, and a contract this version does not know never is. Raise the mismatch outside the read
|
||||
# so the suppression cannot swallow it.
|
||||
if owner.mode == "lease" and owner.lease_duration != self._lease_duration:
|
||||
msg = (
|
||||
f"{self.lock_file} holds a lease of {owner.lease_duration!r}s but this contender configured "
|
||||
f"{self._lease_duration!r}s; every contender for a path must agree on lease_duration"
|
||||
)
|
||||
raise LeaseSettingsMismatch(msg)
|
||||
# A dead or recycled owner is reclaimed at once, whether it held a lease or an exclusive marker; a live lease
|
||||
# owner whose marker stops being refreshed is superseded on the schedule every contender agreed to.
|
||||
if owner.mode != "unknown" and owner_is_stale(owner.pid, owner.hostname, owner.start):
|
||||
self._take_over(marker, owner.pid)
|
||||
return
|
||||
if owner.mode == "lease" and self._marker_unchanged_for(marker) >= self._lease_duration:
|
||||
self._take_over(marker, None)
|
||||
|
||||
def _read_peer(self) -> tuple[OwnerRecord, os.stat_result] | None:
|
||||
with suppress(OSError, ValueError):
|
||||
content, marker = _read_lock_file(self._lock_path)
|
||||
if (owner := parse_marker(content)) is not None:
|
||||
return owner, marker
|
||||
return None
|
||||
|
||||
def _start_heartbeat(self, claim: _LeaseClaim, fd: int, identity: tuple[int, int], token: str) -> None:
|
||||
# The thread watches the event it was handed rather than whatever the claim names later: a heartbeat that
|
||||
# outlives its join timeout would otherwise adopt the next acquisition's event and never stop.
|
||||
stop: Final = Event()
|
||||
loop: Final = _RefreshLoop(
|
||||
# A strong reference from a running thread would keep a dropped lease alive, so __del__ would never release
|
||||
# it and its marker would outlive the process. The callback stops a heartbeat that __del__ could not reach.
|
||||
report=WeakMethod(self._report_compromise, lambda _: stop.set()),
|
||||
lock_file=self._lock_path,
|
||||
claim=claim,
|
||||
fd=fd,
|
||||
identity=identity,
|
||||
token=token,
|
||||
stop=stop,
|
||||
interval=self._heartbeat_interval,
|
||||
duration=self._lease_duration,
|
||||
)
|
||||
thread: Final = Thread(target=loop.run, name=f"filelock-lease-{os.getpid()}", daemon=True)
|
||||
# Record the heartbeat before starting the thread so a release racing this acquire on a shared,
|
||||
# non-thread-local claim always sees it and sets the stop event; the thread then exits at its first wait
|
||||
# instead of outliving the release. A start that raises leaves the unstarted thread for _stop_heartbeat.
|
||||
claim.heartbeat = _Heartbeat(thread, stop)
|
||||
thread.start()
|
||||
|
||||
def _stop_heartbeat(self) -> None:
|
||||
claim = self._claim
|
||||
if (heartbeat := claim.heartbeat) is None:
|
||||
return
|
||||
heartbeat.stop.set()
|
||||
claim.heartbeat = None
|
||||
# An unstarted thread has nothing to join, on_compromise may release from the heartbeat thread itself, and
|
||||
# during finalization a daemon thread is frozen, so joining it raises PythonFinalizationError (3.14+).
|
||||
if heartbeat.thread.ident is not None and heartbeat.thread is not current_thread() and not sys.is_finalizing():
|
||||
heartbeat.thread.join(timeout=self._heartbeat_interval)
|
||||
|
||||
def _report_compromise(
|
||||
self,
|
||||
claim: _LeaseClaim,
|
||||
reason: CompromiseReason,
|
||||
error: OSError | None,
|
||||
token: str,
|
||||
) -> None:
|
||||
# Record it on the claim this heartbeat serves, not on self._claim: a thread-local claim read from the
|
||||
# heartbeat thread is a different, empty one, so the holder would never see the loss it is being told about.
|
||||
# The token is the one this thread published, not claim.token, which a release may already have cleared.
|
||||
claim.compromise = LeaseCompromise(lock_file=self.lock_file, token=token, reason=reason, error=error)
|
||||
if self._on_compromise is not None:
|
||||
self._on_compromise(claim.compromise)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RefreshLoop:
|
||||
"""What a heartbeat thread needs to refresh one claim, holding the lease itself only weakly."""
|
||||
|
||||
report: WeakMethod[Callable[[_LeaseClaim, CompromiseReason, OSError | None, str], None]]
|
||||
lock_file: str
|
||||
claim: _LeaseClaim
|
||||
fd: int
|
||||
identity: tuple[int, int]
|
||||
token: str
|
||||
stop: Event
|
||||
interval: float
|
||||
duration: float
|
||||
|
||||
def run(self) -> None:
|
||||
# A transient error (ESTALE / EIO on the NFS-style filesystems a lease targets) is not a loss. Like restic, call
|
||||
# the claim unrefreshable once failures run long enough that a contender could take it before the next success.
|
||||
last_success, missing = time.monotonic(), False
|
||||
while not self.stop.wait(self.interval):
|
||||
outcome, error = self._refresh()
|
||||
if outcome == "ok":
|
||||
last_success, missing = time.monotonic(), False
|
||||
continue
|
||||
# A peer's stale break exchanges a live marker aside for the length of its check, so only a second miss or
|
||||
# change in a row is a loss.
|
||||
if outcome in {"marker-missing", "owner-changed"} and not missing:
|
||||
missing = True
|
||||
continue
|
||||
if outcome == "transient" and time.monotonic() - last_success < self.duration - self.interval:
|
||||
continue
|
||||
if (report := self.report()) is not None:
|
||||
report(self.claim, "refresh-failed" if outcome == "transient" else outcome, error, self.token)
|
||||
return
|
||||
|
||||
def _refresh(self) -> tuple[_RefreshOutcome, OSError | None]:
|
||||
try:
|
||||
st = os.lstat(self.lock_file)
|
||||
except FileNotFoundError as error:
|
||||
return self._refresh_unnamed(error)
|
||||
except OSError as error:
|
||||
return "transient", error
|
||||
# A peer that took the expired claim replaced the marker, so the pathname now names its inode, not ours.
|
||||
if (st.st_dev, st.st_ino) != self.identity:
|
||||
return "owner-changed", None
|
||||
return self._touch()
|
||||
|
||||
def _refresh_unnamed(self, name_error: FileNotFoundError) -> tuple[_RefreshOutcome, OSError | None]:
|
||||
# A renamed directory takes the marker with it; the name is gone but the inode behind the descriptor keeps its
|
||||
# link.
|
||||
try:
|
||||
linked = os.fstat(self.fd).st_nlink > 0
|
||||
except OSError as error:
|
||||
return "transient", error
|
||||
if not linked:
|
||||
return "marker-missing", name_error
|
||||
return self._touch()
|
||||
|
||||
def _touch(self) -> tuple[_RefreshOutcome, OSError | None]:
|
||||
try:
|
||||
touch(self.lock_file, fd=self.fd)
|
||||
except OSError as error:
|
||||
return "transient", error
|
||||
return "ok", None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CompromiseReason",
|
||||
"LeaseCompromise",
|
||||
"SoftFileLease",
|
||||
"_LeaseClaim",
|
||||
]
|
||||
@@ -0,0 +1,189 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["contextlib", "filelock._identity", "filelock._util", "math"]
|
||||
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from contextlib import suppress
|
||||
from typing import Final, Literal, NamedTuple
|
||||
|
||||
from filelock._identity import host_name, owner_is_current_process, owner_is_stale, process_start_token
|
||||
from filelock._soft import SoftFileLock, _read_lock_file, parse_decimal
|
||||
from filelock._util import pin_inode, write_all
|
||||
|
||||
#: Protocol 1 is the legacy ``<pid>\n<hostname>\n[<start_token>\n]`` marker that :class:`SoftFileLock` still writes.
|
||||
#: Protocol 2 carries the owner mode and the lease claim. A protocol 1 reader treats a protocol 2 marker as malformed
|
||||
#: and evicts it after its grace period, so the two never guarantee mutual exclusion against each other.
|
||||
_PROTOCOL: Final[str] = "filelock/2"
|
||||
|
||||
_MAX_PID: Final[int] = 2**31 - 1
|
||||
|
||||
#: The forms ``repr`` gives a non-negative finite float or int; ``float()`` would also accept whitespace, underscores
|
||||
#: and non-ASCII digits.
|
||||
_DURATION: Final[re.Pattern[str]] = re.compile(r"[0-9]+(?:\.[0-9]+)?(?:e[+-][0-9]+)?", re.ASCII)
|
||||
|
||||
#: Preserve unknown contracts so contenders cannot mistake them for malformed, reclaimable markers.
|
||||
OwnerMode = Literal["lease", "exclusive", "unknown"]
|
||||
|
||||
|
||||
class MarkerSoftFileLock(SoftFileLock):
|
||||
"""An existence lock whose marker carries a protocol 2 owner record."""
|
||||
|
||||
#: Age-based expiry requires a lease contract; an exclusive holder grants none.
|
||||
_owner_mode: OwnerMode = "exclusive"
|
||||
|
||||
@property
|
||||
def owner(self) -> OwnerRecord | None:
|
||||
"""
|
||||
The owner named by the marker on disk.
|
||||
|
||||
:returns: the published record, or ``None`` when no marker exists or its record is malformed or protocol 1
|
||||
|
||||
"""
|
||||
return self._read_owner()
|
||||
|
||||
@property
|
||||
def pid(self) -> int | None:
|
||||
"""
|
||||
The PID of the process holding this lock, read from the marker.
|
||||
|
||||
:returns: the PID, or ``None`` when no marker exists or its record is unreadable
|
||||
|
||||
"""
|
||||
return None if (owner := self._read_owner()) is None else owner.pid
|
||||
|
||||
@property
|
||||
def is_lock_held_by_us(self) -> bool:
|
||||
"""
|
||||
Whether the marker on disk names this process.
|
||||
|
||||
:returns: ``True`` when the marker names this process's PID, hostname and, when recorded, start token
|
||||
|
||||
"""
|
||||
owner = self._read_owner()
|
||||
return owner is not None and owner_is_current_process(owner.pid, owner.hostname, owner.start)
|
||||
|
||||
def force_break(self) -> None:
|
||||
"""
|
||||
Remove the marker whoever holds it, so a later contender can acquire.
|
||||
|
||||
Forced breaking voids mutual exclusion: the previous holder keeps running and keeps using whatever the lock
|
||||
protects. Reserve it for an operator clearing a marker whose holder is known to be gone.
|
||||
"""
|
||||
self.break_lock()
|
||||
|
||||
def _try_break_stale_lock(self) -> None:
|
||||
with suppress(OSError, ValueError), pin_inode(self._lock_path) as pinned:
|
||||
content, marker = _read_lock_file(self._lock_path)
|
||||
owner: Final[OwnerRecord | None]
|
||||
if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing
|
||||
return # replaced before the read, so nothing keeps the number read from reuse
|
||||
if (owner := parse_marker(content)) is None:
|
||||
super()._try_break_stale_lock()
|
||||
elif owner.mode != "unknown" and owner_is_stale(owner.pid, owner.hostname, owner.start):
|
||||
self._take_over(marker, owner.pid)
|
||||
|
||||
@classmethod
|
||||
def _recorded_holder(cls, content: str | None) -> tuple[int, str] | None:
|
||||
if (owner := parse_marker(content)) is not None:
|
||||
return owner.pid, owner.hostname
|
||||
return super()._recorded_holder(content)
|
||||
|
||||
@classmethod
|
||||
def _marker_is_live(cls, content: str | None, breaker_pid: int) -> bool: # pragma: win32 no cover
|
||||
if (owner := parse_marker(content)) is None:
|
||||
return super()._marker_is_live(content, breaker_pid)
|
||||
return owner.mode == "unknown" or (
|
||||
(owner.pid != breaker_pid or owner.hostname != host_name())
|
||||
and not owner_is_stale(owner.pid, owner.hostname, owner.start)
|
||||
)
|
||||
|
||||
def _read_owner(self) -> OwnerRecord | None:
|
||||
with suppress(OSError, ValueError):
|
||||
return parse_marker(_read_lock_file(self._lock_path)[0])
|
||||
return None
|
||||
|
||||
def _write_lock_info(self, fd: int) -> None:
|
||||
write_all(fd, encode_marker(self._published_record()))
|
||||
|
||||
def _published_record(self) -> OwnerRecord:
|
||||
return OwnerRecord(
|
||||
pid=os.getpid(),
|
||||
hostname=host_name(),
|
||||
mode=self._owner_mode,
|
||||
start=process_start_token(os.getpid()),
|
||||
)
|
||||
|
||||
|
||||
class OwnerRecord(NamedTuple):
|
||||
"""Keep optional metadata when reading unknown lock modes."""
|
||||
|
||||
pid: int
|
||||
hostname: str
|
||||
mode: OwnerMode
|
||||
token: str | None = None
|
||||
lease_duration: float | None = None
|
||||
start: int | None = None
|
||||
|
||||
|
||||
def encode_marker(record: OwnerRecord) -> bytes:
|
||||
"""Render an owner record as the bytes a protocol 2 marker holds."""
|
||||
lines = [_PROTOCOL, f"pid={record.pid}", f"host={record.hostname}", f"mode={record.mode}"]
|
||||
if record.token is not None:
|
||||
lines.append(f"token={record.token}")
|
||||
if record.lease_duration is not None:
|
||||
lines.append(f"duration={record.lease_duration!r}")
|
||||
if record.start is not None:
|
||||
lines.append(f"start={record.start}")
|
||||
return "".join(f"{line}\n" for line in lines).encode()
|
||||
|
||||
|
||||
def parse_marker(content: str | None) -> OwnerRecord | None:
|
||||
"""Return the owner a protocol 2 marker names, or ``None`` when the record is malformed or protocol 1."""
|
||||
# A torn write may end in a prefix like "mode=exclu", an unknown contract nobody reclaims; malformed, it ages out.
|
||||
if not content or content[-1] != "\n" or not (lines := content.strip().splitlines()) or lines[0] != _PROTOCOL:
|
||||
return None
|
||||
fields: dict[str, str] = {}
|
||||
for line in lines[1:]:
|
||||
key, separator, value = line.partition("=")
|
||||
if not separator:
|
||||
return None
|
||||
fields[key] = value
|
||||
return _build_record(fields)
|
||||
|
||||
|
||||
def _build_record(fields: dict[str, str]) -> OwnerRecord | None:
|
||||
# An unknown key is a field a newer filelock published, so ignore it rather than read the record as malformed. An
|
||||
# unrecognized mode is the same story one level up: a contract this version does not implement. Reading it as
|
||||
# malformed would age the marker out of a live owner's hands, so keep it and let the caller refuse to reclaim it.
|
||||
# A record naming no mode at all states no contract and stays malformed.
|
||||
if (published := fields.get("mode")) is None:
|
||||
return None
|
||||
mode: Final[OwnerMode] = published if published in {"lease", "exclusive"} else "unknown"
|
||||
hostname = fields.get("host")
|
||||
if not hostname or "pid" not in fields or ("duration" in fields and not _DURATION.fullmatch(fields["duration"])):
|
||||
return None
|
||||
try:
|
||||
pid = parse_decimal(fields["pid"])
|
||||
duration = float(fields["duration"]) if "duration" in fields else None
|
||||
start = parse_decimal(fields["start"]) if "start" in fields else None
|
||||
except ValueError:
|
||||
return None
|
||||
if not 1 <= pid <= _MAX_PID:
|
||||
return None
|
||||
token = fields.get("token")
|
||||
# "1e400" passes the grammar but overflows to inf, which mismatches every configured duration and so wedges reclaim,
|
||||
# where a malformed marker ages out.
|
||||
if mode == "lease" and (not token or duration is None or not (math.isfinite(duration) and duration > 0)):
|
||||
return None
|
||||
return OwnerRecord(pid=pid, hostname=hostname, mode=mode, token=token, lease_duration=duration, start=start)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MarkerSoftFileLock",
|
||||
"OwnerMode",
|
||||
"OwnerRecord",
|
||||
"encode_marker",
|
||||
"parse_marker",
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,643 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = [
|
||||
"contextlib",
|
||||
"filelock._identity",
|
||||
"filelock._soft_protocol",
|
||||
"filelock._util",
|
||||
"itertools",
|
||||
"pathlib",
|
||||
"re",
|
||||
"secrets",
|
||||
"typing",
|
||||
]
|
||||
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from errno import EACCES, EEXIST, EPERM
|
||||
from itertools import islice
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from filelock._api import BaseFileLock, _raise_grouped_errors
|
||||
from filelock._identity import host_name, owner_is_current_process, owner_is_stale, process_start_token
|
||||
from filelock._soft_protocol import STRICT_SOFT_SENTINEL_RECORD
|
||||
from filelock._util import (
|
||||
break_lock_file,
|
||||
exchange_names,
|
||||
exchange_supported,
|
||||
pin_inode,
|
||||
raise_on_not_writable_file,
|
||||
remove_own_file,
|
||||
write_all,
|
||||
)
|
||||
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
from filelock._windows import open_read_without_oplock_wait
|
||||
from filelock._windows import unlink_without_oplock_wait as unlink_path
|
||||
else: # pragma: win32 no cover
|
||||
from filelock._util import unlink_path
|
||||
|
||||
MALFORMED_LOCK_AGE_THRESHOLD: Final[float] = 2.0
|
||||
_MAX_LOCK_FILE_SIZE: Final[int] = 1024
|
||||
_UNLINK_MAX_RETRIES: Final[int] = 10
|
||||
_MARKER_WITH_START_TOKEN_LINE_COUNT: Final[int] = 3
|
||||
#: Contenders act on the record in the marker, so no other account may write it, whatever the mode or umask.
|
||||
_OTHERS_WRITE: Final[int] = stat.S_IWGRP | stat.S_IWOTH
|
||||
#: All a SoftFileLock before 3.22 writes, and what a current one shows between creating its marker and writing it.
|
||||
_RECORDLESS_MARKER: Final[str] = ""
|
||||
_CLAIMS_IN_PROGRESS: Final[set[str]] = set()
|
||||
_CLAIMS_LOCK: Final[threading.Lock] = threading.Lock()
|
||||
|
||||
|
||||
class SoftFileLock(BaseFileLock):
|
||||
"""
|
||||
Cooperative file lock based on a shared existence marker.
|
||||
|
||||
Unlike :class:`UnixFileLock <filelock.UnixFileLock>` and :class:`WindowsFileLock <filelock.WindowsFileLock>`, this
|
||||
lock does not use OS-level locking primitives. Instead, it creates the lock file with ``O_CREAT | O_EXCL`` and
|
||||
treats its existence as the lock indicator. The filesystem must provide coherent exclusive creation and directory
|
||||
updates to each participating process. A crash can leave the marker behind.
|
||||
|
||||
The marker contains the holder's PID and hostname. A contender may remove it when it can no longer find a same-host
|
||||
process with that PID. A configured :attr:`~filelock.BaseFileLock.lifetime` also permits removal based on marker
|
||||
age, including while the holder remains alive. Age-based expiry can overlap protected operations and does not
|
||||
provide strict mutual exclusion.
|
||||
|
||||
"""
|
||||
|
||||
#: Existence locks reclaim by unlinking a pathname, so an age-based lease may break one; a native inode lock cannot.
|
||||
_lifetime_supported: bool = True
|
||||
|
||||
#: Age-based expiry preserves historical behavior but does not provide strict mutual exclusion.
|
||||
_lifetime_replacements: tuple[str, str] | None = ("StrictSoftFileLock", "SoftFileLease")
|
||||
|
||||
#: An existence lock unlinks its marker to release, so it cannot promise to keep the pathname.
|
||||
_preserve_lock_file_supported: bool = False
|
||||
|
||||
#: An existence lock keeps protocol state in its marker, so it cannot lend the descriptor to an on_acquired hook.
|
||||
_on_acquired_supported: bool = False
|
||||
|
||||
#: The marker's (mtime, inode, ctime) when this instance last saw it change, and when, on its monotonic clock.
|
||||
_marker_seen: tuple[tuple[float, int, int], float] | None = None
|
||||
|
||||
#: The identity of a marker a Windows release relinquished but has not unlinked yet.
|
||||
_marker_cleanup: tuple[int, int] | None = None
|
||||
|
||||
#: The descriptor of that marker while its close has not run yet, as a list so one C call can take it.
|
||||
_marker_fds: list[int]
|
||||
|
||||
def _acquire(self) -> None:
|
||||
self._finish_marker_cleanup()
|
||||
raise_on_not_writable_file(self._lock_path)
|
||||
# O_CREAT | O_EXCL makes the create fail with EEXIST when the file already exists, so a successful open
|
||||
# means this process now holds the lock.
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
|
||||
if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None: # pragma: needs o-nofollow
|
||||
flags |= o_nofollow
|
||||
try:
|
||||
fd = os.open(self._lock_path, flags, self._open_mode() & ~_OTHERS_WRITE)
|
||||
except OSError as exception:
|
||||
if not (
|
||||
exception.errno == EEXIST or (exception.errno == EACCES and sys.platform == "win32")
|
||||
): # pragma: win32 no cover
|
||||
raise
|
||||
self._try_break_stale_lock()
|
||||
return
|
||||
# Publish held state only once the record is on disk. On failure, cancellation included, close the descriptor
|
||||
# and unlink the path only while it still names the file we opened: after an expired lease a successor's
|
||||
# marker may sit at the same path. The guard starts at the open, so a signal handler that raises before
|
||||
# _mark_descriptor_owned leaves no marker behind.
|
||||
identity: tuple[int, int] | None = None
|
||||
try:
|
||||
identity = _file_identity(os.fstat(fd))
|
||||
self._mark_descriptor_pending(fd)
|
||||
self._apply_explicit_mode(fd)
|
||||
self._write_lock_info(fd)
|
||||
self._mark_descriptor_owned(fd, identity)
|
||||
except BaseException:
|
||||
self._mark_descriptor_released()
|
||||
try:
|
||||
if sys.platform != "win32": # pragma: win32 no cover
|
||||
# Before the close: our open descriptor keeps the inode allocated, so a successor's marker cannot
|
||||
# carry its number.
|
||||
with suppress(OSError):
|
||||
remove_own_file(self._lock_path, fd)
|
||||
finally:
|
||||
try:
|
||||
os.close(fd)
|
||||
finally:
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
self._unlink_held_marker(identity)
|
||||
raise
|
||||
|
||||
def _apply_explicit_mode(self, fd: int) -> None:
|
||||
# os.open filters the mode through the umask, so an explicit mode lands as given only through fchmod, the way
|
||||
# UnixFileLock applies it. Windows maps the mode to a read-only bit that os.open already set.
|
||||
if self.has_explicit_mode and sys.platform != "win32": # pragma: win32 no cover
|
||||
with suppress(PermissionError):
|
||||
os.fchmod(fd, self._context.mode & ~_OTHERS_WRITE)
|
||||
|
||||
def _try_break_stale_lock(self) -> None:
|
||||
with suppress(OSError, ValueError), pin_inode(self._lock_path) as pinned:
|
||||
content, marker = _read_lock_file(self._lock_path)
|
||||
if pinned not in {None, marker.st_ino}: # pragma: win32 no cover # Windows pins nothing
|
||||
return # another node replaced the pinned one before the read, and nothing keeps its number from reuse
|
||||
if content == STRICT_SOFT_SENTINEL_RECORD: # pragma: needs hard-link
|
||||
return
|
||||
# Such a marker is held for as long as its owner runs, so being unchanged proves nobody gone.
|
||||
if content == _RECORDLESS_MARKER:
|
||||
return
|
||||
holder = _parse_lock_holder(content)
|
||||
|
||||
if holder is None:
|
||||
# Unparsable: wrong line count, a non-integer PID or start token, empty, oversized or not UTF-8.
|
||||
# Self-heal only once the file is clearly not a half-written fresh lock (a peer between O_EXCL and
|
||||
# _write_lock_info), so the brief create-then-write window is never mistaken for a stale lock.
|
||||
if self._marker_unchanged_for(marker) >= MALFORMED_LOCK_AGE_THRESHOLD:
|
||||
self._take_over(marker, _placeholder_breaker(content))
|
||||
return
|
||||
|
||||
if owner_is_stale(*holder):
|
||||
self._take_over(marker, holder[0])
|
||||
|
||||
def _try_break_expired_lock(self) -> None:
|
||||
if (lifetime := self._context.lifetime) is None:
|
||||
return
|
||||
with suppress(OSError), pin_inode(self._lock_path) as pinned:
|
||||
# _read_lock_file takes the lstat, not stat: a symlink swapped in at the path must report its own mtime.
|
||||
content, marker = _read_lock_file(self._lock_path)
|
||||
if pinned not in {None, marker.st_ino} or time.time() - marker.st_mtime < lifetime:
|
||||
return
|
||||
holder = self._recorded_holder(content)
|
||||
self._take_over(marker, None if holder is None else holder[0])
|
||||
|
||||
def _take_over(self, marker: os.stat_result, breaker_pid: int | None) -> None:
|
||||
# Swap the judged marker for a complete marker of ours in one exchange, so the lock path never stands empty.
|
||||
# Once the file swapped out proves to be the one judged, this lock holds the path; the caller's pin keeps that
|
||||
# file's inode number from naming anything else. Only a peer's exchange can move our marker after that, and
|
||||
# the peer puts it back because what it took is not the file it judged. Every failure before the handoff takes
|
||||
# our marker off the path and puts back what the exchange took. Without an exchange, the portable break decides.
|
||||
if exchange_supported(): # pragma: win32 no cover
|
||||
self._take_over_by_exchange(marker, breaker_pid)
|
||||
else:
|
||||
break_lock_file(self._lock_path, marker.st_mtime, marker.st_ino)
|
||||
|
||||
def _take_over_by_exchange(self, marker: os.stat_result, breaker_pid: int | None) -> None: # pragma: win32 no cover
|
||||
if (current := os.lstat(self._lock_path)).st_mtime != marker.st_mtime or current.st_ino != marker.st_ino:
|
||||
return
|
||||
if breaker_pid is not None and self._restore_stranded_marker(breaker_pid, marker):
|
||||
return
|
||||
aside: Final = f"{self._lock_path}.break.{os.getpid()}.{secrets.token_hex(16)}"
|
||||
flags: Final = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd: Final = os.open(aside, flags, self._open_mode() & ~_OTHERS_WRITE)
|
||||
self._mark_descriptor_pending(fd)
|
||||
exchanged = held = False
|
||||
try:
|
||||
identity: Final = _file_identity(os.fstat(fd))
|
||||
self._apply_explicit_mode(fd)
|
||||
self._write_lock_info(fd)
|
||||
if not exchange_names(aside, self._lock_path):
|
||||
return # NFS, SMB and the like refuse it; renaming the marker aside would empty the path
|
||||
exchanged = True
|
||||
if (taken := os.lstat(aside)).st_mtime != marker.st_mtime or taken.st_ino != marker.st_ino:
|
||||
return # a peer recreated the marker, which the cleanup below swaps back
|
||||
Path(aside).unlink()
|
||||
exchanged = False
|
||||
self._mark_descriptor_owned(fd, identity)
|
||||
held = True
|
||||
finally:
|
||||
if not held:
|
||||
self._abandon_take_over(fd, aside, put_back=exchanged)
|
||||
|
||||
def _restore_stranded_marker(self, breaker_pid: int, marker: os.stat_result) -> bool: # pragma: win32 no cover
|
||||
# A breaker killed between its exchange and the put-back leaves its own marker at the path and the live
|
||||
# holder's marker under its break name. A recoverer first renames that record to a name of its own. Only one
|
||||
# contender wins the rename, so two of them never exchange the same record and reverse each other. A record
|
||||
# a dead recoverer holds is claimable again. A claim of this process that no thread is working on was
|
||||
# abandoned by a failed put-back, and the next poll takes it up again. Where the marker cannot be put back
|
||||
# the lock counts as held, as it does wherever no exchange is possible.
|
||||
if (candidates := self._stranded_candidates(breaker_pid)) is None:
|
||||
return True
|
||||
return any(self._recover_claimed(path, breaker_pid, marker, adopt=abandoned) for path, abandoned in candidates)
|
||||
|
||||
def _stranded_candidates(self, breaker_pid: int) -> list[tuple[str, bool]] | None: # pragma: win32 no cover
|
||||
directory, name = os.path.split(self._lock_path)
|
||||
stranded: Final = re.compile(rf"{re.escape(name)}\.break\.{breaker_pid}\.[0-9a-f]{{32}}")
|
||||
claimed: Final = re.compile(rf"{re.escape(name)}\.restore\.(\d+)\.[0-9a-f]{{32}}")
|
||||
candidates: list[tuple[str, bool]] = []
|
||||
with os.scandir(directory or os.curdir) as entries:
|
||||
for entry in entries:
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
continue
|
||||
path = str(Path(directory) / entry.name)
|
||||
if stranded.fullmatch(entry.name):
|
||||
candidates.append((path, False))
|
||||
elif found := claimed.fullmatch(entry.name):
|
||||
ours = int(found[1]) == os.getpid()
|
||||
if not (ours or owner_is_stale(int(found[1]), host_name(), None)):
|
||||
return None
|
||||
candidates.append((path, ours))
|
||||
return sorted(candidates)
|
||||
|
||||
def _recover_claimed(
|
||||
self, path: str, breaker_pid: int, marker: os.stat_result, *, adopt: bool
|
||||
) -> bool: # pragma: win32 no cover
|
||||
# One try/finally owns the registration from the first step to the last, so no interruption can leave it set.
|
||||
# Registered before the rename publishes the name, so a thread that scans meanwhile sees a claim in progress.
|
||||
own: Final = path if adopt else f"{self._lock_path}.restore.{os.getpid()}.{secrets.token_hex(16)}"
|
||||
if not _begin_claim(own):
|
||||
return True
|
||||
try:
|
||||
if not adopt:
|
||||
try:
|
||||
Path(path).rename(own)
|
||||
except OSError: # a peer that won the rename now holds the record, so the lock counts as held
|
||||
return True
|
||||
return self._restore_claimed(own, breaker_pid, marker)
|
||||
finally:
|
||||
_end_claim(own)
|
||||
|
||||
def _restore_claimed(self, own: str, breaker_pid: int, marker: os.stat_result) -> bool: # pragma: win32 no cover
|
||||
try:
|
||||
content, _ = _read_lock_file(own)
|
||||
except OSError:
|
||||
self._return_claim(own, breaker_pid)
|
||||
return True
|
||||
if not self._marker_is_live(content, breaker_pid):
|
||||
with suppress(OSError):
|
||||
Path(own).unlink()
|
||||
return False
|
||||
# The inode of the marker judged dead must still be at the path, or a peer has acted since.
|
||||
with suppress(OSError):
|
||||
current = os.lstat(self._lock_path)
|
||||
if (current.st_ino, current.st_mtime) == (marker.st_ino, marker.st_mtime) and exchange_names(
|
||||
own, self._lock_path
|
||||
):
|
||||
if os.lstat(own).st_ino == marker.st_ino:
|
||||
Path(own).unlink()
|
||||
return True
|
||||
# A peer replaced the path after the check, so the exchange took its marker: swap it straight back.
|
||||
exchange_names(own, self._lock_path)
|
||||
self._return_claim(own, breaker_pid)
|
||||
return True
|
||||
|
||||
def _return_claim(self, own: str, breaker_pid: int) -> None: # pragma: win32 no cover
|
||||
# A refused rename leaves the claim under this process's PID, and the next poll retries it.
|
||||
with suppress(OSError):
|
||||
Path(own).rename(f"{self._lock_path}.break.{breaker_pid}.{secrets.token_hex(16)}")
|
||||
|
||||
@classmethod
|
||||
def _marker_is_live(cls, content: str | None, breaker_pid: int) -> bool: # pragma: win32 no cover
|
||||
if content == STRICT_SOFT_SENTINEL_RECORD: # pragma: needs hard-link
|
||||
return True
|
||||
if content == _RECORDLESS_MARKER:
|
||||
# An empty marker is a pre-3.22 holder for as long as it exists, as the stale check treats it.
|
||||
return True
|
||||
# The breaker's own marker names its PID on this host; the same number on another host is a holder.
|
||||
return (
|
||||
(holder := _parse_lock_holder(content)) is not None
|
||||
and (holder[0] != breaker_pid or holder[1] != host_name())
|
||||
and not owner_is_stale(*holder)
|
||||
)
|
||||
|
||||
def _abandon_take_over(self, fd: int, aside: str, *, put_back: bool) -> None: # pragma: win32 no cover
|
||||
self._mark_descriptor_released()
|
||||
try:
|
||||
if put_back:
|
||||
exchange_names(aside, self._lock_path)
|
||||
finally:
|
||||
try:
|
||||
# Whichever name still holds our marker loses it: the descriptor keeps its inode number ours.
|
||||
own: Final = os.fstat(fd)
|
||||
for name in (self._lock_path, aside):
|
||||
with suppress(FileNotFoundError):
|
||||
if os.path.samestat(os.lstat(name), own):
|
||||
Path(name).unlink()
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def _marker_unchanged_for(self, marker: os.stat_result) -> float:
|
||||
# The mtime is the writer's or file server's clock, so against time.time() a contender whose clock runs ahead
|
||||
# would expire a fresh marker. Only watch it for change, timed on this process's monotonic clock. The owner of a
|
||||
# file sets its mtime, and ext4 reuses a freed inode number, so a new marker can repeat both. The change time
|
||||
# cannot be set and moves with every create, which keeps a new marker from inheriting an old one's age.
|
||||
now: Final = time.monotonic()
|
||||
key: Final = marker.st_mtime, marker.st_ino, marker.st_ctime_ns
|
||||
if (seen := self._marker_seen) is None or seen[0] != key:
|
||||
self._marker_seen = key, now
|
||||
return 0.0
|
||||
return now - seen[1]
|
||||
|
||||
def _unprobeable_holder(self) -> str | None:
|
||||
# This host cannot prove such a holder dead, so whoever decides whether it is gone needs its name.
|
||||
with suppress(OSError, ValueError):
|
||||
if (content := _read_lock_file(self._lock_path)[0]) == _RECORDLESS_MARKER:
|
||||
return "an owner that recorded no PID (a filelock release before 3.22, or one still writing its record)"
|
||||
if (holder := self._recorded_holder(content)) and holder[1] != host_name():
|
||||
return f"pid {holder[0]} on {holder[1]}"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _recorded_holder(cls, content: str | None) -> tuple[int, str] | None:
|
||||
return None if (holder := _parse_lock_holder(content)) is None else holder[:2]
|
||||
|
||||
@staticmethod
|
||||
def _write_lock_info(fd: int) -> None:
|
||||
# No suppression: a write failure must reach the acquisition rollback so it never publishes a half-written
|
||||
# marker as held state. The optional third line is this process's start token, absent when the platform
|
||||
# exposes no proven start time, in which case a reader falls back to PID-only liveness.
|
||||
info = f"{os.getpid()}\n{host_name()}\n"
|
||||
if (token := process_start_token(os.getpid())) is not None:
|
||||
info += f"{token}\n"
|
||||
write_all(fd, info.encode())
|
||||
|
||||
@property
|
||||
def pid(self) -> int | None:
|
||||
"""
|
||||
The PID of the process holding this lock, read from the lock file.
|
||||
|
||||
:returns: the PID as an integer, or ``None`` if the lock file does not exist or cannot be parsed
|
||||
|
||||
"""
|
||||
with suppress(OSError, ValueError):
|
||||
holder = _parse_lock_holder(_read_lock_file(self._lock_path)[0])
|
||||
if holder is not None:
|
||||
return holder[0]
|
||||
return None
|
||||
|
||||
@property
|
||||
def is_lock_held_by_us(self) -> bool:
|
||||
"""
|
||||
Whether this lock is held by the current process.
|
||||
|
||||
:returns: ``True`` if the lock file exists and names the current process's PID, hostname and, when recorded,
|
||||
start token
|
||||
|
||||
"""
|
||||
with suppress(OSError, ValueError):
|
||||
if (holder := _parse_lock_holder(_read_lock_file(self._lock_path)[0])) is not None:
|
||||
return owner_is_current_process(*holder)
|
||||
return False
|
||||
|
||||
def break_lock(self) -> None:
|
||||
"""Forcibly break the lock by removing the lock file, regardless of who holds it."""
|
||||
with suppress(OSError):
|
||||
Path(self._lock_path).unlink()
|
||||
|
||||
def _release_current(self, *, force: bool) -> None:
|
||||
try:
|
||||
super()._release_current(force=force)
|
||||
finally:
|
||||
# A release a handler cut short after the descriptor closed still owes the unlink on Windows.
|
||||
self._finish_marker_cleanup()
|
||||
|
||||
def _release(self) -> None:
|
||||
fd = self._context.lock_file_fd
|
||||
assert fd is not None # ruff:ignore[assert] # _release runs only while held, so the descriptor is set
|
||||
# Capture the held file's identity before closing so cleanup can refuse to unlink a successor's marker. A
|
||||
# supported lifetime lease lets a peer break our expired marker and create its own at this path before we
|
||||
# release; unlinking by path alone would then delete the successor's lock.
|
||||
identity: tuple[int, int] | None = None
|
||||
with suppress(OSError):
|
||||
identity = _file_identity(os.fstat(fd))
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
self._release_then_unlink(fd, identity)
|
||||
else: # pragma: win32 no cover
|
||||
# Our open descriptor keeps the inode allocated, so its number cannot name a successor's marker yet, and an
|
||||
# interrupt before the unlink lands leaves the lock held for the retried release to finish.
|
||||
unlinked: Final = self._unlink_open_marker(fd, identity)
|
||||
# A failed close may already have released and recycled the descriptor number. Relinquish it before the
|
||||
# one close attempt so no later release can close an unrelated descriptor that reused the same integer.
|
||||
self._mark_descriptor_released()
|
||||
try:
|
||||
self._close_released_fd(fd, default_suppresses=False)
|
||||
finally:
|
||||
# A share that refuses to delete an open file gets the unlink after the close; the marker still exists
|
||||
# until then, so its inode cannot be reused either.
|
||||
if not unlinked:
|
||||
self._unlink_held_marker(identity)
|
||||
|
||||
def _release_then_unlink(self, fd: int, identity: tuple[int, int] | None) -> None: # pragma: win32 cover
|
||||
# Windows cannot unlink an open file, so the unlink outlives the hold until it lands, here or on a later acquire
|
||||
# or release. Only the gate's holder touches the pending record, so two threads never overwrite each other's.
|
||||
with self._transition_lock:
|
||||
self._marker_fds = [fd]
|
||||
self._marker_cleanup = identity
|
||||
self._mark_descriptor_released()
|
||||
try:
|
||||
self._close_marker_fd()
|
||||
# Marker cleanup must also run for control-flow exceptions, and both failures must remain observable.
|
||||
except BaseException as close_error:
|
||||
try:
|
||||
self._finish_marker_cleanup()
|
||||
except BaseException as cleanup_error: # ruff:ignore[blind-except] # keep control-flow failures
|
||||
_raise_grouped_errors(
|
||||
"lock descriptor close and marker cleanup both failed",
|
||||
close_error,
|
||||
cleanup_error,
|
||||
)
|
||||
raise
|
||||
self._finish_marker_cleanup()
|
||||
|
||||
def _marker_names(self) -> tuple[str, ...]:
|
||||
# After a rename of the lock's directory only the caller's spelling, resolved through the working directory,
|
||||
# still reaches the marker this hold opened.
|
||||
return tuple(dict.fromkeys((self._lock_path, self.lock_file)))
|
||||
|
||||
def _unlink_open_marker(self, fd: int, identity: tuple[int, int] | None) -> bool: # pragma: win32 no cover
|
||||
if identity is None:
|
||||
return True
|
||||
replaced = False
|
||||
for name in self._marker_names():
|
||||
try:
|
||||
if _file_identity(os.lstat(name)) == identity:
|
||||
Path(name).unlink()
|
||||
return True
|
||||
replaced = True
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except OSError:
|
||||
return False
|
||||
if replaced:
|
||||
try:
|
||||
_leave_released_record(fd)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _finish_marker_cleanup(self) -> None:
|
||||
# Never waits: whoever holds the gate is acquiring or releasing this lock and finishes the cleanup itself.
|
||||
if self._marker_cleanup is not None and self._transition_lock.acquire(blocking=False): # pragma: win32 cover
|
||||
try:
|
||||
# A close error here belongs to the release an interrupt cut short, not to this caller.
|
||||
with suppress(OSError):
|
||||
self._close_marker_fd()
|
||||
if (identity := self._marker_cleanup) is not None: # pragma: no branch # the gate's holder clears it
|
||||
self._windows_unlink_if_ours(identity)
|
||||
finally:
|
||||
self._transition_lock.release()
|
||||
|
||||
def _close_marker_fd(self) -> None: # pragma: win32 cover
|
||||
# Popping and closing is one chain of C calls that no signal handler can split, so a retried release closes the
|
||||
# descriptor at most once; a second close could hit an unrelated file that reused the number.
|
||||
try:
|
||||
list(map(os.close, islice(iter(vars(self).setdefault("_marker_fds", []).pop, None), 1)))
|
||||
except IndexError:
|
||||
return
|
||||
except OSError:
|
||||
if not self._suppresses_close_error(default_suppresses=False):
|
||||
raise
|
||||
|
||||
def _unlink_held_marker(self, identity: tuple[int, int] | None) -> None:
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
if identity is not None:
|
||||
self._windows_unlink_if_ours(identity)
|
||||
else: # pragma: win32 no cover
|
||||
for name in self._marker_names():
|
||||
with suppress(OSError):
|
||||
if _file_identity(os.lstat(name)) == identity:
|
||||
Path(name).unlink()
|
||||
return
|
||||
|
||||
def _windows_unlink_if_ours(self, identity: tuple[int, int]) -> None: # pragma: win32 cover
|
||||
retry_delay = 0.001
|
||||
for attempt in range(_UNLINK_MAX_RETRIES):
|
||||
# Windows doesn't immediately release file handles after close, causing EACCES/EPERM on unlink. Recheck
|
||||
# identity each attempt: a failed unlink leaves a window for a successor to replace the marker at this path.
|
||||
try:
|
||||
for name in self._marker_names():
|
||||
with suppress(FileNotFoundError):
|
||||
if _file_identity(os.lstat(name)) == identity:
|
||||
# Forgotten before the unlink: Windows does not promise unique file IDs, so no retry may
|
||||
# reach a successor that reused ours.
|
||||
self._marker_cleanup = None
|
||||
unlink_path(name)
|
||||
break
|
||||
except OSError as exc:
|
||||
if exc.errno in {EACCES, EPERM}:
|
||||
# The file is still ours, so the next acquire or release tries again if these attempts run out.
|
||||
self._marker_cleanup = identity
|
||||
if attempt < _UNLINK_MAX_RETRIES - 1:
|
||||
time.sleep(retry_delay)
|
||||
retry_delay *= 2
|
||||
continue
|
||||
self._marker_cleanup = None
|
||||
return
|
||||
|
||||
|
||||
def _file_identity(st: os.stat_result) -> tuple[int, int]:
|
||||
# (st_dev, st_ino) names the concrete inode behind a path, so a marker recreated at the same pathname after an
|
||||
# expired lease reads as a different file. CPython populates both on Windows from the volume serial and file index.
|
||||
return st.st_dev, st.st_ino
|
||||
|
||||
|
||||
def _begin_claim(name: str | Path) -> bool: # pragma: win32 no cover
|
||||
# Keyed by the file name alone: its random token identifies the claim, and the spelling of the directory varies.
|
||||
with _CLAIMS_LOCK:
|
||||
if (key := Path(name).name) in _CLAIMS_IN_PROGRESS:
|
||||
return False
|
||||
_CLAIMS_IN_PROGRESS.add(key)
|
||||
return True
|
||||
|
||||
|
||||
def _end_claim(name: str | Path) -> None: # pragma: win32 no cover
|
||||
with _CLAIMS_LOCK:
|
||||
_CLAIMS_IN_PROGRESS.discard(Path(name).name)
|
||||
|
||||
|
||||
def _placeholder_breaker(content: str | None) -> int | None: # pragma: win32 no cover
|
||||
# The placeholder a break leaves at the path while it works names the breaker that may have died there.
|
||||
if content is None or not content.startswith(placeholder := "filelock/break\n") or not content.endswith("\n"):
|
||||
return None
|
||||
with suppress(ValueError):
|
||||
return parse_decimal(content.removeprefix(placeholder)[:-1])
|
||||
return None
|
||||
|
||||
|
||||
def _leave_released_record(fd: int) -> None:
|
||||
# A peer's exchange holds our marker under its break name and will put it back. Overwriting the record lets
|
||||
# contenders age it out instead of waiting on this live PID. The write comes before the truncate so the file is
|
||||
# never empty.
|
||||
if sys.platform != "win32": # pragma: win32 no cover
|
||||
os.pwrite(fd, released := b"filelock/released\n", 0)
|
||||
os.ftruncate(fd, len(released))
|
||||
|
||||
|
||||
def _read_lock_file(path: str) -> tuple[str | None, os.stat_result]:
|
||||
# A legitimate lock file is always a regular file. Classify the path with lstat first, so any other node (symlink,
|
||||
# FIFO, socket, device) is reported as a malformed lock the caller can evict, without an os.open that would follow
|
||||
# a symlink, stall on a FIFO, or fail on a socket and leave acquisition wedged. The stat result still flows back
|
||||
# for the identity-checked stale break. lstat, not stat, so a hostile symlink is never followed onto its target.
|
||||
st = os.lstat(path)
|
||||
if not stat.S_ISREG(st.st_mode): # pragma: needs fifo
|
||||
return None, st
|
||||
# Re-check on the opened handle: O_NOFOLLOW refuses a symlink swapped in after the lstat, O_NONBLOCK stops a FIFO
|
||||
# swapped in from stalling the open, and the fstat catches any other non-regular replacement race before we read.
|
||||
# The capped read stops a huge regular file (e.g. one filled from /dev/zero) from exhausting memory.
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
# os.open would wait for an oplock holder to answer its break, past the caller's deadline.
|
||||
fd = open_read_without_oplock_wait(path)
|
||||
else: # pragma: win32 no cover
|
||||
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
|
||||
try:
|
||||
st = os.fstat(fd)
|
||||
if not stat.S_ISREG(st.st_mode): # pragma: no cover # only a non-regular node swapped in after the lstat
|
||||
return None, st
|
||||
data = os.read(fd, _MAX_LOCK_FILE_SIZE + 1)
|
||||
finally:
|
||||
os.close(fd)
|
||||
if len(data) <= _MAX_LOCK_FILE_SIZE:
|
||||
with suppress(UnicodeDecodeError):
|
||||
return data.decode("utf-8"), st
|
||||
return None, st
|
||||
|
||||
|
||||
def _parse_lock_holder(content: str | None) -> tuple[int, str, int | None] | None:
|
||||
# A well-formed lock file is "<pid>\n<hostname>\n" with an optional "<start_token>\n" third line naming the
|
||||
# holder's process start instant (a filelock 3.29 marker wrote this only on Windows; every platform writes it now).
|
||||
# Anything else (wrong line count, a non-integer PID or start token, empty or unreadable content) is unparsable;
|
||||
# returning None lets the caller treat it as a malformed lock to self-heal rather than a holder.
|
||||
# Every writer since 3.22 ends each line with "\n", so a record without one was cut by a crash or read mid-write. As
|
||||
# a holder it would name a foreign host no one reclaims, or a start token that lets a contender break a live lock.
|
||||
if not content or not content.endswith("\n") or len(lines := content.strip().splitlines()) not in {2, 3}:
|
||||
return None
|
||||
try:
|
||||
pid = parse_decimal(lines[0])
|
||||
start_token = parse_decimal(lines[2]) if len(lines) == _MARKER_WITH_START_TOKEN_LINE_COUNT else None
|
||||
except ValueError:
|
||||
return None
|
||||
# A pid outside the valid range is a malformed lock, not a holder. Without this, a non-positive pid
|
||||
# reaches os.kill() where 0 / -1 mean "the caller's own process group / every process" so a dead
|
||||
# holder reads as alive and the lock is never reclaimed, while an oversized pid raises OverflowError
|
||||
# (not OSError/ValueError) out of the self-heal path. _parse_marker_bytes already enforces this range.
|
||||
if not 1 <= pid <= 2**31 - 1:
|
||||
return None
|
||||
return pid, lines[1], start_token
|
||||
|
||||
|
||||
def parse_decimal(text: str) -> int:
|
||||
"""
|
||||
Read a PID or start token as a writer formats it: ASCII digits only.
|
||||
|
||||
:raises ValueError: for anything else ``int()`` would accept, such as ``1_000``, surrounding whitespace, a sign or
|
||||
non-ASCII digits; no filelock writes those
|
||||
|
||||
"""
|
||||
if not (text.isascii() and text.isdigit()):
|
||||
msg = f"not a decimal integer: {text!r}"
|
||||
raise ValueError(msg)
|
||||
return int(text)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MALFORMED_LOCK_AGE_THRESHOLD",
|
||||
"SoftFileLock",
|
||||
"parse_decimal",
|
||||
]
|
||||
@@ -0,0 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["typing"]
|
||||
|
||||
from typing import Final
|
||||
|
||||
STRICT_SOFT_SENTINEL_RECORD: Final[str] = "1\nfilelock-strict-v1\x00\n0\n"
|
||||
|
||||
__all__ = [
|
||||
"STRICT_SOFT_SENTINEL_RECORD",
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["filelock._soft_rw._async"]
|
||||
|
||||
from filelock._soft_rw._async import AsyncAcquireSoftReadWriteReturnProxy, AsyncSoftReadWriteLock
|
||||
from filelock._soft_rw._sync import SoftReadWriteLock
|
||||
|
||||
__all__ = [
|
||||
"AsyncAcquireSoftReadWriteReturnProxy",
|
||||
"AsyncSoftReadWriteLock",
|
||||
"SoftReadWriteLock",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,373 @@
|
||||
"""Async wrapper around :class:`SoftReadWriteLock` for use with ``asyncio``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["asyncio", "filelock._async", "filelock._error", "functools", "threading"]
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import os
|
||||
import threading
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar
|
||||
|
||||
from filelock._async import (
|
||||
_TASKS_THREAD_ID,
|
||||
_BackendOutcome,
|
||||
_capture_call,
|
||||
_drain_future,
|
||||
_future_result,
|
||||
_raise_cancelled_error,
|
||||
_task_owners_for,
|
||||
_wait_until_done,
|
||||
)
|
||||
from filelock._error import Timeout
|
||||
from filelock._soft_rw._sync import SoftReadWriteLock, _lost_hold_error
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator, Callable
|
||||
from concurrent import futures
|
||||
from types import TracebackType
|
||||
|
||||
from filelock._api import AcquireReturnProxy
|
||||
from filelock._lease import LeaseCompromise
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
_R = TypeVar("_R")
|
||||
|
||||
|
||||
class AsyncSoftReadWriteLock:
|
||||
"""
|
||||
Async wrapper around :class:`SoftReadWriteLock` for ``asyncio`` applications.
|
||||
|
||||
The sync class's blocking filesystem operations run on a thread pool via ``loop.run_in_executor()``. Each
|
||||
``asyncio`` task owns its own hold. Tasks share a read lock, and a task asking for the write lock waits until the
|
||||
other holders release, with waiting writers ahead of new readers. Upgrading or downgrading a held lock raises
|
||||
:class:`RuntimeError`. The underlying :class:`SoftReadWriteLock` handles forks, the heartbeat, and stale eviction.
|
||||
Singleton wrappers share one sync lock and its task holds, and closing one ends only the holds taken through it.
|
||||
|
||||
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
|
||||
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
|
||||
:param is_singleton: if ``True``, reuse existing :class:`SoftReadWriteLock` instances per resolved path
|
||||
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
|
||||
:param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to
|
||||
``3 * heartbeat_interval``
|
||||
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
|
||||
:param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is
|
||||
lost
|
||||
:param loop: event loop for ``run_in_executor``; ``None`` uses the running loop
|
||||
:param executor: executor for ``run_in_executor``; ``None`` uses the default executor
|
||||
|
||||
.. versionadded:: 3.27.0
|
||||
|
||||
"""
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
|
||||
self,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = True,
|
||||
heartbeat_interval: float = 30.0,
|
||||
stale_threshold: float | None = None,
|
||||
poll_interval: float = 0.25,
|
||||
on_compromise: Callable[[LeaseCompromise], None] | None = None,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
executor: futures.Executor | None = None,
|
||||
) -> None:
|
||||
self._creator_pid = os.getpid()
|
||||
self._closed = False
|
||||
self._owns_backend = not is_singleton
|
||||
#: Marks the task holds taken through this wrapper, which its close() ends.
|
||||
self._via: Final = object()
|
||||
self._lock = SoftReadWriteLock(
|
||||
lock_file,
|
||||
timeout,
|
||||
blocking=blocking,
|
||||
is_singleton=is_singleton,
|
||||
heartbeat_interval=heartbeat_interval,
|
||||
stale_threshold=stale_threshold,
|
||||
poll_interval=poll_interval,
|
||||
on_compromise=on_compromise,
|
||||
)
|
||||
self._owners = _task_owners_for(self._lock)
|
||||
self._loop = loop
|
||||
self._executor = executor
|
||||
|
||||
@property
|
||||
def lock_file(self) -> str:
|
||||
"""The path to the lock file passed to the constructor."""
|
||||
return self._lock.lock_file
|
||||
|
||||
@property
|
||||
def timeout(self) -> float:
|
||||
"""The default timeout applied when ``acquire_read`` / ``acquire_write`` is called without one."""
|
||||
return self._lock.timeout
|
||||
|
||||
@property
|
||||
def blocking(self) -> bool:
|
||||
"""Whether ``acquire_*`` defaults to blocking; ``False`` makes contention raise immediately."""
|
||||
return self._lock.blocking
|
||||
|
||||
@property
|
||||
def generation(self) -> int | None:
|
||||
"""The generation the current hold was granted at, a fencing token; ``None`` when no lock is held."""
|
||||
return self._lock.generation
|
||||
|
||||
@property
|
||||
def compromise(self) -> LeaseCompromise | None:
|
||||
"""How the current hold was lost, or ``None`` while it stands; acquiring raises until the hold is released."""
|
||||
return self._lock.compromise
|
||||
|
||||
@property
|
||||
def loop(self) -> asyncio.AbstractEventLoop | None:
|
||||
"""The event loop used for ``run_in_executor``, or ``None`` for the running loop."""
|
||||
return self._loop
|
||||
|
||||
@property
|
||||
def executor(self) -> futures.Executor | None:
|
||||
"""The executor used for ``run_in_executor``, or ``None`` for the default executor."""
|
||||
return self._executor
|
||||
|
||||
@asynccontextmanager
|
||||
async def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
|
||||
"""
|
||||
Async context manager that acquires and releases a shared read lock.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the write lock
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
await self.acquire_read(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await self.release()
|
||||
|
||||
@asynccontextmanager
|
||||
async def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> AsyncGenerator[None]:
|
||||
"""
|
||||
Async context manager that acquires and releases an exclusive write lock.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the read lock
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
await self.acquire_write(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await self.release()
|
||||
|
||||
async def acquire_read(
|
||||
self, timeout: float | None = None, *, blocking: bool | None = None
|
||||
) -> AsyncAcquireSoftReadWriteReturnProxy:
|
||||
"""
|
||||
Acquire a shared read lock.
|
||||
|
||||
See :meth:`SoftReadWriteLock.acquire_read` for reentrancy / upgrade / fork semantics. The blocking work runs
|
||||
inside ``run_in_executor`` so other coroutines on the same loop keep progressing while this call waits.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:returns: a proxy usable as an async context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the write lock, if this instance was invalidated by
|
||||
:func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self._raise_if_unusable()
|
||||
await self._acquire("read", timeout, blocking=blocking)
|
||||
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
|
||||
|
||||
async def acquire_write(
|
||||
self, timeout: float | None = None, *, blocking: bool | None = None
|
||||
) -> AsyncAcquireSoftReadWriteReturnProxy:
|
||||
"""
|
||||
Acquire an exclusive write lock.
|
||||
|
||||
See :meth:`SoftReadWriteLock.acquire_write` for the writer-preferring semantics. The blocking work runs inside
|
||||
``run_in_executor``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:returns: a proxy usable as an async context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if the calling task already holds the read lock, if this instance was invalidated by
|
||||
:func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self._raise_if_unusable()
|
||||
await self._acquire("write", timeout, blocking=blocking)
|
||||
return AsyncAcquireSoftReadWriteReturnProxy(lock=self)
|
||||
|
||||
async def _acquire(self, mode: Literal["read", "write"], timeout: float | None, *, blocking: bool | None) -> None:
|
||||
blocking = self._lock.blocking if blocking is None else blocking
|
||||
|
||||
async def enter(remaining: float) -> None:
|
||||
# A close() may have landed while this task waited for the table.
|
||||
self._raise_if_unusable()
|
||||
await self._run_acquire(
|
||||
functools.partial(
|
||||
self._lock._acquire_level_of, # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
_TASKS_THREAD_ID,
|
||||
mode,
|
||||
remaining,
|
||||
blocking=blocking,
|
||||
)
|
||||
)
|
||||
|
||||
def joinable() -> bool:
|
||||
self._raise_if_unusable()
|
||||
if (compromise := self._lock.compromise) is not None:
|
||||
raise _lost_hold_error(self.lock_file, mode, compromise)
|
||||
return self._backend_holds()
|
||||
|
||||
await self._owners.acquire(
|
||||
mode,
|
||||
timeout=self._lock.timeout if timeout is None else timeout,
|
||||
blocking=blocking,
|
||||
lock_file=self.lock_file,
|
||||
via=self._via,
|
||||
enter=enter,
|
||||
joinable=joinable,
|
||||
)
|
||||
|
||||
async def release(self, *, force: bool = False) -> None:
|
||||
"""
|
||||
Release one level of the calling task's hold.
|
||||
|
||||
The wrapper releases the lock once no task holds it.
|
||||
|
||||
:param force: if ``True``, end every level the calling task took through this wrapper. Levels taken through
|
||||
other wrappers on the same path stay. A wrapper that holds no level for the task releases nothing.
|
||||
|
||||
:raises RuntimeError: if the calling task holds no level through this wrapper and *force* is ``False``
|
||||
|
||||
"""
|
||||
if self._creator_pid == os.getpid():
|
||||
await self._owners.release(
|
||||
force=force,
|
||||
lock_file=self.lock_file,
|
||||
via=self._via,
|
||||
leave=functools.partial(self._submit, self._leave_backend),
|
||||
still_held=self._backend_holds,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""
|
||||
End the holds of the tasks that acquired through this instance and release the lock once no task holds it.
|
||||
|
||||
Idempotent. After calling this method the instance can no longer acquire locks. With ``is_singleton=False`` it
|
||||
also closes the underlying :class:`SoftReadWriteLock`. A singleton wrapper leaves the shared one open, so tasks
|
||||
that acquired through another wrapper and synchronous users of the same path keep their holds. The next acquire
|
||||
notices a synchronous ``close()`` or forced release of that shared lock. One that lands in the few bytecodes
|
||||
between that check and the admission still admits the task into the ended hold.
|
||||
|
||||
"""
|
||||
if self._creator_pid != os.getpid() or self._closed:
|
||||
return
|
||||
# Refuse acquires from here on, so none is admitted into a hold the backend is letting go of.
|
||||
self._closed = True
|
||||
try:
|
||||
await self._owners.retire(
|
||||
None if self._owns_backend else self._via,
|
||||
leave=functools.partial(self._submit, self._lock.close if self._owns_backend else self._leave_backend),
|
||||
still_held=self._backend_holds,
|
||||
)
|
||||
except BaseException as error:
|
||||
# Only a failed backend call can leave the level held, and then a retried close() has to find it.
|
||||
if not isinstance(error, asyncio.CancelledError) and self._backend_holds():
|
||||
self._closed = False
|
||||
raise
|
||||
|
||||
def _raise_if_unusable(self) -> None:
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
msg = f"AsyncSoftReadWriteLock on {self.lock_file} was inherited across fork; construct a new instance"
|
||||
raise RuntimeError(msg)
|
||||
if self._closed:
|
||||
msg = f"AsyncSoftReadWriteLock on {self.lock_file} has been closed"
|
||||
raise RuntimeError(msg)
|
||||
|
||||
def _backend_holds(self) -> bool:
|
||||
# A sync close() or forced release on a shared singleton ends the tasks' level without the table taking part.
|
||||
return self._lock._holds_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
|
||||
def _leave_backend(self) -> None:
|
||||
# Only the tasks' level: a sync user of the same singleton keeps its own.
|
||||
self._lock._release_level_of(_TASKS_THREAD_ID) # ruff:ignore[private-member-access] # the wrapper owns this backend hold
|
||||
|
||||
async def _run_acquire(self, acquire: Callable[..., AcquireReturnProxy]) -> None:
|
||||
# run_in_executor cannot recall work the pool already started. A sync acquire that won the lock before it saw
|
||||
# the cancellation has published its claim and started the heartbeat, which keeps the record fresh so no peer
|
||||
# evicts it. Wait the call out and hand the claim back, as AsyncReadWriteLock does.
|
||||
abandoned = threading.Event()
|
||||
acquire_future = self._submit(acquire, cancel_check=abandoned.is_set)
|
||||
try:
|
||||
await _wait_until_done(acquire_future)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
# Stops a backend wait on another process's hold, which would otherwise outlast the caller by as long.
|
||||
abandoned.set()
|
||||
try:
|
||||
await _drain_future(acquire_future)
|
||||
except Timeout:
|
||||
# Nothing was taken, so the caller's cancellation is the whole outcome.
|
||||
raise cancellation from None
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
_raise_cancelled_error(cancellation, error)
|
||||
try:
|
||||
await _drain_future(self._submit(self._leave_backend))
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
_raise_cancelled_error(cancellation, error)
|
||||
raise
|
||||
_future_result(acquire_future)
|
||||
|
||||
def _submit(
|
||||
self, func: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> asyncio.Future[_BackendOutcome[_R]]:
|
||||
loop = self._loop or asyncio.get_running_loop()
|
||||
return loop.run_in_executor(self._executor, _capture_call, functools.partial(func, *args, **kwargs))
|
||||
|
||||
|
||||
class AsyncAcquireSoftReadWriteReturnProxy:
|
||||
"""Async context-aware object that releases an :class:`AsyncSoftReadWriteLock` on exit."""
|
||||
|
||||
def __init__(self, lock: AsyncSoftReadWriteLock) -> None:
|
||||
self.lock = lock
|
||||
|
||||
async def __aenter__(self) -> AsyncSoftReadWriteLock:
|
||||
return self.lock
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
await self.lock.release()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AsyncAcquireSoftReadWriteReturnProxy",
|
||||
"AsyncSoftReadWriteLock",
|
||||
]
|
||||
@@ -0,0 +1,638 @@
|
||||
"""
|
||||
The generation-log protocol behind :class:`~filelock.SoftReadWriteLock`.
|
||||
|
||||
The lock's whole state is a sequence of immutable snapshots, ``gen/<N>``, each naming the writer and the readers that
|
||||
hold the lock at generation ``N``. Every transition is one atomic publication of the next snapshot: a participant reads
|
||||
the latest generation, derives the successor it wants, and links a fully written temporary file to ``gen/<N+1>``. The
|
||||
link fails if a peer got there first, so the participant re-reads and tries again. No step is ever half done: a crash
|
||||
leaves at most an unlinked temporary file, never a state that another participant must repair before it can proceed.
|
||||
|
||||
Liveness rests on a nonce, not a clock. Each participant keeps ``holders/<token>`` and rewrites its nonce every
|
||||
heartbeat. A contender records the bytes it read there and the moment it read them on its own monotonic clock; a
|
||||
member whose bytes have not changed for ``stale_threshold`` is evicted by committing a snapshot without it. Two hosts
|
||||
never compare clocks, so a skewed file server or a suspended virtual machine cannot make a live holder look dead.
|
||||
|
||||
The generation number is monotonic and unique to one transition within one incarnation of the log, so the generation at
|
||||
which a hold was granted fences it: a resource that rejects anything carrying a lower generation than the highest it has
|
||||
accepted refuses a holder that paused past its expiry and resumed. A random epoch file names the incarnation, so a
|
||||
participant that outlives a removal of the log reads the new one instead of continuing the old numbering into it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["contextlib", "filelock._error", "filelock._identity", "math", "pathlib", "secrets"]
|
||||
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol
|
||||
|
||||
from filelock._error import SoftFileLockProtocolError
|
||||
from filelock._identity import host_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
Mode = Literal["read", "write"]
|
||||
HeartbeatOutcome = Literal["ok", "lost", "transient"]
|
||||
|
||||
_PROTOCOL: Final[str] = "filelock-rw/1"
|
||||
_GENERATION_DIGITS: Final[int] = 20
|
||||
_TOKEN_HEX_LENGTH: Final[int] = 32
|
||||
#: A snapshot line is ``reader=`` plus a token, so this bounds a hostile record without capping real reader counts.
|
||||
_MAX_RECORD_SIZE: Final[int] = 1 << 20
|
||||
#: Snapshots older than this many generations behind the latest are removed by whoever commits. The window is also how
|
||||
#: far ``latest`` probes past a gap before it trusts that nothing newer exists; ``gen/HEAD``, not the window, is what
|
||||
#: keeps a listing served further behind from passing an old generation off as the head.
|
||||
_RETAINED_GENERATIONS: Final[int] = 64
|
||||
_GENERATIONS_DIRECTORY: Final[str] = "gen"
|
||||
#: A random token created once per log: removing ``gen/`` removes it, so a participant can tell a log that was wiped and
|
||||
#: restarted from the one it remembers.
|
||||
_EPOCH_NAME: Final[str] = "epoch"
|
||||
#: The epoch plus a copy of the newest snapshot, renamed into place after each commit. NFS revalidates a file opened by
|
||||
#: name but may serve a listing from cache, so this is the head a stale client cannot miss; it may lag, never lead.
|
||||
_HEAD_NAME: Final[str] = "HEAD"
|
||||
_HOLDERS_DIRECTORY: Final[str] = "holders"
|
||||
_TEMPORARY_PREFIX: Final[str] = ".commit-"
|
||||
|
||||
|
||||
class TreeReplacedError(RuntimeError):
|
||||
"""
|
||||
Raised by a ``Files`` implementation when a protocol directory is not a real directory (a symlink, a junction, a
|
||||
file) or belongs to another account.
|
||||
"""
|
||||
|
||||
|
||||
class Files(Protocol):
|
||||
"""The filesystem operations the protocol runs on; :mod:`._storage` implements them over :mod:`os`."""
|
||||
|
||||
def read(self, path: str) -> bytes | None:
|
||||
"""The file's bytes, ``None`` when it does not exist, or ``b""`` when it is not a regular file."""
|
||||
|
||||
def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]:
|
||||
"""
|
||||
Create the file exclusively with the whole record written; raise ``FileExistsError`` when it exists.
|
||||
|
||||
The returned device and inode (the kernel's identity for a file, independent of its name) let ``overwrite``
|
||||
check that the name still leads to this file.
|
||||
|
||||
*durable* flushes it to stable storage first, for a record that outlives the host that wrote it: a generation a
|
||||
crash left empty would wedge every other participant. A holder record dies with its writer, so it skips that.
|
||||
"""
|
||||
|
||||
def link(self, source: str, target: str) -> bool:
|
||||
"""Hard-link *source* to *target* without replacing; ``True`` when *target* names *source*'s file afterwards."""
|
||||
|
||||
def replace(self, source: str, target: str) -> bool:
|
||||
"""Rename *source* over *target* atomically; ``False`` when the platform refused because *target* is in use."""
|
||||
|
||||
def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool:
|
||||
"""
|
||||
Rewrite from offset zero the file ``create`` returned *identity* for.
|
||||
|
||||
``False`` when *path* no longer leads to that file.
|
||||
"""
|
||||
|
||||
def unlink(self, path: str) -> None:
|
||||
"""Remove the file, tolerating one that is already gone."""
|
||||
|
||||
def listdir(self, path: str) -> list[str]:
|
||||
"""The names in the directory, or an empty list when it does not exist."""
|
||||
|
||||
def prepare(self, root: str, *, force: bool = False) -> None:
|
||||
"""
|
||||
Create the protocol directories under *root*, refusing anything at those paths that is not a directory.
|
||||
|
||||
Done once per lock; *force* repeats it after a write found a directory missing.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Snapshot:
|
||||
"""The lock's state at one generation: which token writes, which tokens read."""
|
||||
|
||||
generation: int
|
||||
writer: str | None
|
||||
readers: frozenset[str]
|
||||
|
||||
@property
|
||||
def members(self) -> frozenset[str]:
|
||||
"""Every token this generation names."""
|
||||
return self.readers if self.writer is None else self.readers | {self.writer}
|
||||
|
||||
def encode(self) -> bytes:
|
||||
lines = [_PROTOCOL, f"generation={self.generation}"]
|
||||
if self.writer is not None:
|
||||
lines.append(f"writer={self.writer}")
|
||||
lines.extend(f"reader={token}" for token in sorted(self.readers))
|
||||
return "".join(f"{line}\n" for line in lines).encode("ascii")
|
||||
|
||||
|
||||
def parse_snapshot(data: bytes) -> Snapshot | None:
|
||||
"""The snapshot a ``gen/<N>`` record encodes, or ``None`` when the record is malformed."""
|
||||
if not data or len(data) > _MAX_RECORD_SIZE:
|
||||
return None
|
||||
try:
|
||||
lines = data.decode("ascii").split("\n")
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
if lines[0] != _PROTOCOL or lines[-1]:
|
||||
return None
|
||||
generation: int | None = None
|
||||
writer: str | None = None
|
||||
readers: set[str] = set()
|
||||
for line in lines[1:-1]:
|
||||
key, _, value = line.partition("=")
|
||||
if (
|
||||
key == "generation"
|
||||
and generation is None
|
||||
and len(value) <= _GENERATION_DIGITS
|
||||
and value.isdigit()
|
||||
and str(int(value)) == value
|
||||
):
|
||||
generation = int(value)
|
||||
elif key == "writer" and writer is None and is_token(value):
|
||||
writer = value
|
||||
elif key == "reader" and is_token(value) and value not in readers:
|
||||
readers.add(value)
|
||||
else:
|
||||
return None
|
||||
return None if generation is None else Snapshot(generation=generation, writer=writer, readers=frozenset(readers))
|
||||
|
||||
|
||||
def is_token(value: str) -> bool:
|
||||
"""Whether *value* is a claim token: exactly 32 lowercase hex digits."""
|
||||
return len(value) == _TOKEN_HEX_LENGTH and all(character in "0123456789abcdef" for character in value)
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
"""A fresh claim token."""
|
||||
return secrets.token_hex(_TOKEN_HEX_LENGTH // 2)
|
||||
|
||||
|
||||
def encode_holder(token: str, nonce: str, lease: float) -> bytes:
|
||||
"""
|
||||
The ``holders/<token>`` record; only the nonce changes between heartbeats, so the length stays constant.
|
||||
|
||||
*lease* is the writer's own ``stale_threshold``: a peer configured tighter must not read this holder's longer
|
||||
heartbeat interval as death, so evictors wait the longer of their threshold and this one.
|
||||
"""
|
||||
return (
|
||||
f"{_PROTOCOL}\ntoken={token}\npid={os.getpid()}\nhost={host_name()}\nlease={float(lease)!r}\nnonce={nonce}\n"
|
||||
).encode("ascii")
|
||||
|
||||
|
||||
def holder_lease(record: bytes | None) -> float | None:
|
||||
"""The ``lease`` a holder record carries, or ``None`` when it has none (an older release) or it is malformed."""
|
||||
try:
|
||||
text = next(
|
||||
line for line in (record or b"").decode("ascii").split("\n") if line.startswith("lease=")
|
||||
).removeprefix("lease=")
|
||||
lease = float(text)
|
||||
except (StopIteration, ValueError): # UnicodeDecodeError is a ValueError
|
||||
return None
|
||||
# Only the exact spelling repr() writes: float() alone also takes "1_000", " 1", and non-ASCII digits.
|
||||
return lease if isfinite(lease) and lease > 0 and repr(lease) == text else None
|
||||
|
||||
|
||||
class Ledger:
|
||||
"""
|
||||
How long each observed value has stayed the same, measured on this observer's monotonic clock.
|
||||
|
||||
A value that has not changed for ``stale_threshold`` belongs to a participant that stopped refreshing it. The clock
|
||||
is the observer's own and only ever measures the gap between two of its own observations, which is what lets two
|
||||
hosts agree on liveness without agreeing on the time.
|
||||
"""
|
||||
|
||||
def __init__(self, clock: Callable[[], float]) -> None:
|
||||
self._clock = clock
|
||||
self._seen: dict[str, tuple[bytes | None, float]] = {}
|
||||
|
||||
def observe(self, key: str, value: bytes | None, *, asked: float | None = None) -> float:
|
||||
"""
|
||||
Record *value* for *key* and return for how long it has stayed the same.
|
||||
|
||||
*asked* is the clock reading taken before the caller fetched *value*. An unchanged value counts as unchanged up
|
||||
to *asked*, so a pause between the fetch and this call adds no time.
|
||||
"""
|
||||
now = self._clock()
|
||||
if (seen := self._seen.get(key)) is None or seen[0] != value:
|
||||
self._seen[key] = (value, now)
|
||||
return 0.0
|
||||
return (now if asked is None else asked) - seen[1]
|
||||
|
||||
def forget(self, key: str) -> None:
|
||||
self._seen.pop(key, None)
|
||||
|
||||
def retain(self, keys: set[str]) -> None:
|
||||
"""Drop every key not in *keys*: peers that left would otherwise stay in the ledger for the lock's lifetime."""
|
||||
self._seen = {key: seen for key, seen in self._seen.items() if key in keys}
|
||||
|
||||
|
||||
class GenerationLog:
|
||||
"""The ordered snapshots under ``gen/``, found by listing and probing, advanced by linking the next one."""
|
||||
|
||||
def __init__(self, files: Files, lock_file: str, root: str) -> None:
|
||||
self._files = files
|
||||
self._lock_file = lock_file
|
||||
self._directory = Path(root, _GENERATIONS_DIRECTORY)
|
||||
self._epoch_path = str(self._directory / _EPOCH_NAME)
|
||||
self._head_path = str(self._directory / _HEAD_NAME)
|
||||
self._epoch: bytes | None = None
|
||||
self._latest: Snapshot | None = None
|
||||
|
||||
def latest(self) -> Snapshot:
|
||||
"""
|
||||
The newest snapshot.
|
||||
|
||||
Starts from the newest readable generation among those ``gen/HEAD``, the listing, or memory name, then probes
|
||||
forward by name across the retained window. ``HEAD`` only names a generation, and that generation must still
|
||||
read back: a committer stalled before its rename can leave ``HEAD`` on a generation compacted since. A
|
||||
generation compacted between the listing and the read, and a hole left by a committer that died before it
|
||||
compacted, are covered by the probe. Memory and ``HEAD`` are bound to the log's epoch, so a log wiped and
|
||||
restarted is never continued from a generation it no longer holds. Only a log that names generations, now or as
|
||||
remembered, of which none can be read is refused, because that is a client so far behind the log that nothing
|
||||
it reads can be trusted.
|
||||
"""
|
||||
# When everything the first pass named is gone, peers compacted past it while this client looked, or the log was
|
||||
# replaced; one more pass settles whether this client cannot see the head at all.
|
||||
if (latest := self._find()) is None and (latest := self._find()) is None:
|
||||
raise SoftFileLockProtocolError(self._lock_file, None, "generation log names no readable snapshot")
|
||||
self._latest = latest
|
||||
return latest
|
||||
|
||||
def _find(self) -> Snapshot | None:
|
||||
if (epoch := self._incarnation()) != self._epoch:
|
||||
self._epoch, self._latest = epoch, None
|
||||
named = set(self._list())
|
||||
# Generation 0 is the empty log, which has no record to read back.
|
||||
if self._latest is not None and self._latest.generation:
|
||||
named.add(self._latest.generation)
|
||||
if head := self._head_generation():
|
||||
named.add(head)
|
||||
for generation in sorted(named, reverse=True):
|
||||
if (start := self._read(generation)) is not None:
|
||||
# Compaction keeps the surviving generations contiguous, so the first missing name past a readable HEAD
|
||||
# generation is the end; probing the whole window on every poll cost 64 failed opens.
|
||||
return self._probe(start, patience=1 if generation == head else _RETAINED_GENERATIONS)
|
||||
newest = max(named, default=0)
|
||||
found = self._probe(Snapshot(generation=newest, writer=None, readers=frozenset()))
|
||||
return found if found.generation > newest or not named else None
|
||||
|
||||
def _head_generation(self) -> int | None:
|
||||
# HEAD is never flushed, so a crash can leave it unreadable; readers verify what it names anyway, so a torn one
|
||||
# only sends them to the window probe.
|
||||
if (record := self._head_record()) is None or (snapshot := parse_snapshot(record)) is None:
|
||||
return None
|
||||
return snapshot.generation
|
||||
|
||||
def _head_record(self) -> bytes | None:
|
||||
if (data := self._files.read(self._head_path)) is None:
|
||||
return None
|
||||
# A HEAD from another epoch landed after its log was removed; a log whose epoch was never written runs without.
|
||||
epoch, newline, record = data.partition(b"\n")
|
||||
return record if self._valid_epoch() == epoch + newline else None
|
||||
|
||||
def _valid_epoch(self) -> bytes | None:
|
||||
epoch = self._epoch
|
||||
return epoch if epoch is not None and is_token(epoch.decode("ascii", "replace").removesuffix("\n")) else None
|
||||
|
||||
def _incarnation(self) -> bytes | None:
|
||||
if (epoch := self._files.read(self._epoch_path)) is None:
|
||||
# Exclusive creation, not a link: reading the log must keep working where os.link does not exist. A peer
|
||||
# that reads the file before its token is written takes the empty value for the epoch, and its next commit
|
||||
# sees the change and re-reads, so a partial read costs one retry.
|
||||
try:
|
||||
self._files.create(self._epoch_path, f"{new_token()}\n".encode("ascii"), durable=True)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
# No gen/ yet: there is no log to tell apart, and the first participant's prepare() creates it.
|
||||
return None
|
||||
epoch = self._files.read(self._epoch_path)
|
||||
return epoch
|
||||
|
||||
def _list(self) -> list[int]:
|
||||
listed = self._listed()
|
||||
for generation in listed[:-_RETAINED_GENERATIONS]:
|
||||
self._files.unlink(self._path(generation))
|
||||
return listed[-_RETAINED_GENERATIONS:]
|
||||
|
||||
def _listed(self) -> list[int]:
|
||||
# isdigit() alone admits Unicode digits: superscripts int() rejects, fullwidth ones it parses to another number.
|
||||
return sorted(
|
||||
int(name)
|
||||
for name in self._files.listdir(str(self._directory))
|
||||
if len(name) == _GENERATION_DIGITS and name.isascii() and name.isdigit()
|
||||
)
|
||||
|
||||
def _probe(self, start: Snapshot, patience: int = _RETAINED_GENERATIONS) -> Snapshot:
|
||||
latest = start
|
||||
generation = start.generation
|
||||
gap = 0
|
||||
while gap < patience:
|
||||
generation += 1
|
||||
if (snapshot := self._read(generation)) is None:
|
||||
gap += 1
|
||||
continue
|
||||
latest = snapshot
|
||||
gap = 0
|
||||
return latest
|
||||
|
||||
def _read(self, generation: int) -> Snapshot | None:
|
||||
if (data := self._files.read(self._path(generation))) is None:
|
||||
return None
|
||||
if (snapshot := parse_snapshot(data)) is None or snapshot.generation != generation:
|
||||
name = self._name(generation)
|
||||
raise SoftFileLockProtocolError(self._lock_file, name, "malformed generation record")
|
||||
return snapshot
|
||||
|
||||
def commit(self, successor: Snapshot) -> bool:
|
||||
"""
|
||||
Publish *successor* as the next generation; ``False`` when a peer published that generation first.
|
||||
|
||||
The record is complete before it gets its public name, and a no-replace hard link is the compare-and-swap: only
|
||||
one link to ``gen/<N+1>`` can ever succeed. The link's return code is not trusted, because an NFS retry can
|
||||
report a link that did land as failed; the file's identity after the call is what decides. A log whose epoch
|
||||
changed since the snapshot was read is a different log, so the commit is refused as lost and the caller
|
||||
re-reads; that narrows a wipe racing this commit to the moment between the epoch check and the link. A slot
|
||||
compaction freed is refused the same way, before the link and again after it, since a committer can stall in
|
||||
between. After the link a surviving predecessor proves little: it may be a record another stalled committer
|
||||
linked into its own freed slot and has yet to remove. A slot comes free once the log is a whole window past it,
|
||||
and the head stays, so a record the listing or ``HEAD`` leads by a window is refused as well. A record refused
|
||||
after it landed is removed when nothing was built on it. One that peers lead by less than a window stays, or its
|
||||
slot would come free early.
|
||||
"""
|
||||
if self._files.read(self._epoch_path) != self._epoch or not self._extends_the_log(successor.generation):
|
||||
return False
|
||||
if not self._publish(
|
||||
self._path(successor.generation), data := successor.encode(), self._files.link, durable=True
|
||||
):
|
||||
return False
|
||||
lead = max([*self._listed(), self._head_generation() or 0]) - successor.generation
|
||||
if lead >= _RETAINED_GENERATIONS or not self._extends_the_log(successor.generation):
|
||||
if not 0 < lead < _RETAINED_GENERATIONS:
|
||||
self._files.unlink(self._path(successor.generation))
|
||||
return False
|
||||
self._latest = successor
|
||||
# Readers verify the generation HEAD names, so a failed rename costs a probe, not the commit. A late one would
|
||||
# move HEAD back past later commits, so HEAD only ever moves forward.
|
||||
with suppress(OSError):
|
||||
if (
|
||||
(epoch := self._valid_epoch()) is not None
|
||||
and self._files.read(self._epoch_path) == epoch
|
||||
and (self._head_generation() or 0) < successor.generation
|
||||
):
|
||||
self._publish(self._head_path, epoch + data, self._files.replace, durable=False)
|
||||
if (expired := successor.generation - _RETAINED_GENERATIONS - 1) > 0:
|
||||
self._compact_through(expired)
|
||||
return True
|
||||
|
||||
def _extends_the_log(self, generation: int) -> bool:
|
||||
# Compaction removes oldest first, so a missing predecessor means compaction freed this slot. The first
|
||||
# generation has none; a HEAD of this epoch means the log already moved past it.
|
||||
if generation > 1:
|
||||
return self._files.read(self._path(generation - 1)) is not None
|
||||
return self._head_record() is None
|
||||
|
||||
def _publish(self, target: str, data: bytes, place: Callable[[str, str], bool], *, durable: bool) -> bool:
|
||||
temporary = str(self._directory / f"{_TEMPORARY_PREFIX}{new_token()}")
|
||||
try:
|
||||
self._files.create(temporary, data, durable=durable)
|
||||
except FileNotFoundError:
|
||||
# No gen/: the log was removed, or nobody has prepared it yet. Either way there is nothing to publish into
|
||||
# until a participant's prepare() creates it, and the caller re-reads whatever log exists by then.
|
||||
return False
|
||||
try:
|
||||
return place(temporary, target)
|
||||
finally:
|
||||
self._files.unlink(temporary)
|
||||
|
||||
def _compact_through(self, expired: int) -> None:
|
||||
# Oldest first, so survivors stay contiguous: a stalled committer's leftover would otherwise sit below a hole.
|
||||
oldest = expired
|
||||
while oldest > 1 and self._files.read(self._path(oldest - 1)) is not None:
|
||||
oldest -= 1
|
||||
for generation in range(oldest, expired + 1):
|
||||
self._files.unlink(self._path(generation))
|
||||
|
||||
def _path(self, generation: int) -> str:
|
||||
return str(self._directory / self._name(generation))
|
||||
|
||||
@staticmethod
|
||||
def _name(generation: int) -> str:
|
||||
return f"{generation:0{_GENERATION_DIGITS}d}"
|
||||
|
||||
|
||||
class Participant:
|
||||
"""
|
||||
One acquisition, from publishing a claim through holding the lock to leaving.
|
||||
|
||||
``advance`` is polled until it reports the lock granted, ``heartbeat`` runs on the holder's schedule, and ``leave``
|
||||
commits the participant out. None of them sleep: the caller owns the deadline and the poll interval. The ledger and
|
||||
the log outlive one acquisition: how long a peer's record has stayed unchanged is knowledge every attempt against
|
||||
the same lock shares, or a caller whose attempts are each shorter than the stale threshold could never evict.
|
||||
"""
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # one value per protocol input; the lock builds this once per acquisition
|
||||
self,
|
||||
files: Files,
|
||||
root: str,
|
||||
mode: Mode,
|
||||
*,
|
||||
stale_threshold: float,
|
||||
clock: Callable[[], float],
|
||||
ledger: Ledger,
|
||||
log: GenerationLog,
|
||||
) -> None:
|
||||
self.token: Final[str] = new_token()
|
||||
self.mode: Final[Mode] = mode
|
||||
self.generation: int | None = None
|
||||
self._files = files
|
||||
self._root = root
|
||||
self._stale_threshold = stale_threshold
|
||||
self._clock = clock
|
||||
self._ledger = ledger
|
||||
self._log = log
|
||||
self._holder = self._holder_path(self.token)
|
||||
self._holder_identity: tuple[int, int] | None = None
|
||||
self._entered = False
|
||||
self._next_sweep = clock()
|
||||
|
||||
def publish(self) -> None:
|
||||
"""Create the holder record peers watch for liveness; it exists before the token appears in any snapshot."""
|
||||
self._files.prepare(self._root)
|
||||
self._create_holder()
|
||||
|
||||
def _create_holder(self) -> None:
|
||||
try:
|
||||
self._holder_identity = self._files.create(self._holder, self._record())
|
||||
except FileNotFoundError:
|
||||
# Removing the whole log takes the directories with it.
|
||||
self._files.prepare(self._root, force=True)
|
||||
self._holder_identity = self._files.create(self._holder, self._record())
|
||||
|
||||
def _record(self) -> bytes:
|
||||
return encode_holder(self.token, new_token(), self._stale_threshold)
|
||||
|
||||
def advance(self) -> bool:
|
||||
"""
|
||||
Take the next step toward holding the lock; ``True`` once the lock is granted.
|
||||
|
||||
A reader enters as soon as no live writer is named. A writer enters as soon as no live writer is named, which
|
||||
blocks every new reader, then waits for the named readers to leave. Stale members are evicted in the same commit
|
||||
that admits this participant, and a writer's final admission is itself a commit, so no admission can rest on a
|
||||
snapshot a peer is about to supersede.
|
||||
"""
|
||||
# A lost race is not a reason to wait: re-read and try again before handing the poll interval back to the
|
||||
# caller.
|
||||
for _ in range(4):
|
||||
latest = self._log.latest()
|
||||
self._keep_claim_fresh(latest)
|
||||
self._sweep(latest)
|
||||
stale = frozenset(token for token in latest.members - {self.token} if self._is_stale(token))
|
||||
writer = None if latest.writer in stale else latest.writer
|
||||
readers = latest.readers - stale
|
||||
if not self._entered:
|
||||
if writer is not None:
|
||||
# A live writer blocks entry in both modes; only an eviction is worth committing.
|
||||
if stale:
|
||||
self._commit(latest, writer, readers, stale)
|
||||
return False
|
||||
writer, readers = (self.token, readers) if self.mode == "write" else (None, readers | {self.token})
|
||||
granted = self.mode == "read" or not readers
|
||||
if self._entered and not stale and not granted:
|
||||
return False
|
||||
if (generation := self._commit(latest, writer, readers, stale)) is None:
|
||||
continue
|
||||
self._entered = True
|
||||
if granted:
|
||||
self.generation = generation
|
||||
return granted
|
||||
return False
|
||||
|
||||
def _keep_claim_fresh(self, latest: Snapshot) -> None:
|
||||
# The snapshot is the truth about membership: a commit the filesystem reported lost may have landed, and a peer
|
||||
# may have evicted this participant while it waited to drain, so both directions are taken from it. A wait can
|
||||
# outlast the stale threshold, so the nonce changes on every poll; a peer then never reads a patient contender
|
||||
# as a corpse. A record that has gone missing was taken by an evictor or a sweeper that did, or by a removal of
|
||||
# the whole log.
|
||||
self._entered = self.token in latest.members
|
||||
if not self._refresh_nonce():
|
||||
with suppress(FileExistsError):
|
||||
self._create_holder()
|
||||
|
||||
def _sweep(self, latest: Snapshot) -> None:
|
||||
# A participant that crashed before its first commit leaves a record no snapshot names, and a commit that
|
||||
# crashed between creating and unlinking its temporary file leaves that too. Neither blocks anyone, so they are
|
||||
# collected on the same unchanged-for-a-threshold rule as a member, and only every half threshold at most.
|
||||
if (now := self._clock()) < self._next_sweep:
|
||||
return
|
||||
self._next_sweep = now + self._stale_threshold / 2
|
||||
watched = set(latest.members)
|
||||
holders = Path(self._root, _HOLDERS_DIRECTORY)
|
||||
for name in self._files.listdir(str(holders)):
|
||||
if name != self.token and name not in latest.members and is_token(name):
|
||||
watched.add(key := f"holder:{name}")
|
||||
self._collect(key, str(holders / name))
|
||||
generations = Path(self._root, _GENERATIONS_DIRECTORY)
|
||||
for name in self._files.listdir(str(generations)):
|
||||
if name.startswith(_TEMPORARY_PREFIX):
|
||||
watched.add(key := f"temporary:{name}")
|
||||
self._collect(key, str(generations / name))
|
||||
self._ledger.retain(watched)
|
||||
|
||||
def _collect(self, key: str, path: str) -> None:
|
||||
if self._unchanged_for_its_lease(key, path):
|
||||
self._files.unlink(path)
|
||||
self._ledger.forget(key)
|
||||
|
||||
def _commit(
|
||||
self, latest: Snapshot, writer: str | None, readers: frozenset[str], stale: frozenset[str]
|
||||
) -> int | None:
|
||||
successor = Snapshot(generation=latest.generation + 1, writer=writer, readers=readers)
|
||||
if not self._log.commit(successor):
|
||||
return None
|
||||
for token in stale:
|
||||
self._files.unlink(self._holder_path(token))
|
||||
self._ledger.forget(token)
|
||||
return successor.generation
|
||||
|
||||
def _is_stale(self, token: str) -> bool:
|
||||
return self._unchanged_for_its_lease(token, self._holder_path(token))
|
||||
|
||||
def _unchanged_for_its_lease(self, key: str, path: str) -> bool:
|
||||
asked = self._clock()
|
||||
record = self._files.read(path)
|
||||
return self._ledger.observe(key, record, asked=asked) >= max(self._stale_threshold, holder_lease(record) or 0.0)
|
||||
|
||||
def _holder_path(self, token: str) -> str:
|
||||
return str(Path(self._root, _HOLDERS_DIRECTORY, token))
|
||||
|
||||
def _refresh_nonce(self) -> bool:
|
||||
if (identity := self._holder_identity) is None:
|
||||
return False
|
||||
return self._files.overwrite(self._holder, self._record(), identity)
|
||||
|
||||
def heartbeat(self) -> tuple[HeartbeatOutcome, OSError | None]:
|
||||
"""
|
||||
Refresh the nonce and confirm the latest snapshot still names this participant.
|
||||
|
||||
``lost`` means a peer evicted this participant or removed its holder record: the lock is no longer held, and the
|
||||
holder must stop using what it protects. ``transient`` is a filesystem error refreshing the nonce, worth
|
||||
retrying on the next tick. A failure to read the log or to sweep decides nothing about this holder's liveness,
|
||||
so it is not counted against it.
|
||||
"""
|
||||
try:
|
||||
if not self._refresh_nonce():
|
||||
return "lost", None
|
||||
except TreeReplacedError:
|
||||
return "lost", None
|
||||
except OSError as error:
|
||||
return "transient", error
|
||||
try:
|
||||
latest = self._log.latest()
|
||||
self._sweep(latest)
|
||||
except (OSError, TreeReplacedError):
|
||||
return "ok", None
|
||||
return ("ok" if self.token in latest.members else "lost"), None
|
||||
|
||||
def leave(self) -> None:
|
||||
"""Commit this participant out of the latest snapshot, then remove its holder record."""
|
||||
# The record goes even when the log cannot be read: a contender that never entered then leaves nothing behind,
|
||||
# and a member still named ages out the same way with or without one.
|
||||
# A tree that was swapped out leaves no record this participant can reach, so there is nothing to commit out of.
|
||||
with suppress(TreeReplacedError):
|
||||
try:
|
||||
while self.token in (latest := self._log.latest()).members:
|
||||
successor = Snapshot(
|
||||
generation=latest.generation + 1,
|
||||
writer=None if latest.writer == self.token else latest.writer,
|
||||
readers=latest.readers - {self.token},
|
||||
)
|
||||
if self._log.commit(successor):
|
||||
break
|
||||
finally:
|
||||
self._entered = False
|
||||
self.generation = None
|
||||
self._files.unlink(self._holder)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Files",
|
||||
"GenerationLog",
|
||||
"HeartbeatOutcome",
|
||||
"Ledger",
|
||||
"Mode",
|
||||
"Participant",
|
||||
"Snapshot",
|
||||
"TreeReplacedError",
|
||||
"encode_holder",
|
||||
"holder_lease",
|
||||
"is_token",
|
||||
"new_token",
|
||||
"parse_snapshot",
|
||||
]
|
||||
@@ -0,0 +1,278 @@
|
||||
"""The :class:`~._protocol.Files` implementation over :mod:`os`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["filelock._soft_rw._protocol", "filelock._strict", "filelock._util", "pathlib"]
|
||||
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager, suppress
|
||||
from errno import ELOOP, ENOENT, ENOTDIR, ENXIO, ESTALE
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from filelock._soft_rw._protocol import (
|
||||
_GENERATIONS_DIRECTORY,
|
||||
_HOLDERS_DIRECTORY,
|
||||
_MAX_RECORD_SIZE,
|
||||
TreeReplacedError,
|
||||
)
|
||||
from filelock._strict import _link_no_follow, _pin_directories, _raise_if_hard_links_unsupported
|
||||
from filelock._util import ensure_directory_exists, foreign_directory_owner, write_all
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
_O_NOFOLLOW: Final[int] = getattr(os, "O_NOFOLLOW", 0)
|
||||
#: Probed once at import, as :mod:`filelock._strict` does, because a per-call check flips when a test mocks the
|
||||
#: function. The set lists os.rename, whose implementation os.replace shares.
|
||||
_SUPPORTS_DIR_FD: Final[bool] = (
|
||||
hasattr(os, "link")
|
||||
and {os.link, os.open, os.stat, os.mkdir, os.rename, os.unlink} <= os.supports_dir_fd
|
||||
and os.scandir in os.supports_fd
|
||||
)
|
||||
#: Windows opens descriptors in text mode by default, which rewrites newlines; every record here is exact bytes.
|
||||
_O_BINARY: Final[int] = getattr(os, "O_BINARY", 0)
|
||||
#: ESTALE is what an NFS client reports for a file a peer unlinked out from under its cached handle: gone, not broken.
|
||||
_MISSING_ERRNOS: Final[frozenset[int]] = frozenset({ENOENT, ESTALE})
|
||||
#: Windows refuses an open with EACCES while another process is deleting the file or holds it open without sharing;
|
||||
#: both clear within moments, so an open is retried this long before the refusal is taken as real.
|
||||
_WINDOWS_OPEN_GRACE: Final[float] = 0.5
|
||||
|
||||
|
||||
class OsFiles:
|
||||
"""Record files under a lock's protocol root, opened without following symlinks and owned by this UID alone."""
|
||||
|
||||
def __init__(self, lock_file: str) -> None:
|
||||
self._lock_file = lock_file
|
||||
self._prepared = False
|
||||
self._identities: dict[str, tuple[int, int]] = {}
|
||||
|
||||
def read(self, path: str) -> bytes | None:
|
||||
# O_NONBLOCK keeps an open of a FIFO planted at the path from blocking; the regular-file check below rejects it.
|
||||
if (fd := self._open(path, os.O_RDONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY)) is None:
|
||||
return None
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
return b""
|
||||
chunks: list[bytes] = []
|
||||
remaining = _MAX_RECORD_SIZE + 1
|
||||
while remaining > 0 and (chunk := os.read(fd, remaining)):
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
return b"".join(chunks)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def create(self, path: str, data: bytes, *, durable: bool = False) -> tuple[int, int]:
|
||||
with self._relative(path) as (name, dir_fd):
|
||||
fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _O_NOFOLLOW | _O_BINARY, 0o600, dir_fd=dir_fd)
|
||||
try:
|
||||
write_all(fd, data)
|
||||
if durable:
|
||||
os.fsync(fd)
|
||||
created = os.fstat(fd)
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
# The name is unique to this call, so removing it cannot take a record from anyone else.
|
||||
self.unlink(path)
|
||||
raise
|
||||
os.close(fd)
|
||||
return created.st_dev, created.st_ino
|
||||
|
||||
def link(self, source: str, target: str) -> bool:
|
||||
failure: NotImplementedError | OSError | None = None
|
||||
with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd):
|
||||
try:
|
||||
_link_no_follow(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd)
|
||||
except FileExistsError:
|
||||
pass
|
||||
except (NotImplementedError, OSError) as error:
|
||||
_raise_if_hard_links_unsupported(self._lock_file, error)
|
||||
failure = error
|
||||
# The file's identity decides, not the call's result: an NFS retransmit can report a link that landed as
|
||||
# failed, and an EEXIST can be this very link answered twice. Any other error on a link that did not land is
|
||||
# a fault, not a peer winning the race, and a retry would spin.
|
||||
if (identity := _identity(source_name, source_fd)) is not None and identity == _identity(
|
||||
target_name, target_fd
|
||||
):
|
||||
return True
|
||||
if failure is not None:
|
||||
raise failure
|
||||
return False
|
||||
|
||||
def replace(self, source: str, target: str) -> bool:
|
||||
try:
|
||||
with self._relative(source) as (source_name, source_fd), self._relative(target) as (target_name, target_fd):
|
||||
os.replace(source_name, target_name, src_dir_fd=source_fd, dst_dir_fd=target_fd)
|
||||
except PermissionError:
|
||||
# Windows refuses to rename over a file a reader holds open; callers tolerate a target one rename behind.
|
||||
return False
|
||||
return True
|
||||
|
||||
def overwrite(self, path: str, data: bytes, identity: tuple[int, int]) -> bool:
|
||||
# O_NONBLOCK keeps an open of a FIFO (a named pipe) at the path from blocking, as in read. With no reader the
|
||||
# open fails with ENXIO.
|
||||
try:
|
||||
fd = self._open(path, os.O_WRONLY | _O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0) | _O_BINARY)
|
||||
except OSError as error:
|
||||
if error.errno != ENXIO:
|
||||
raise
|
||||
return False # pragma: needs fifo
|
||||
if fd is None:
|
||||
return False
|
||||
try:
|
||||
# O_NOFOLLOW refuses a symlink but not a hard link (a second name for the same file) or another file renamed
|
||||
# over the name, and a write through either lands in a file that is not the record. The protocol does not
|
||||
# link holder records, so a second name marks a file that inherited a removed record's inode number.
|
||||
opened = os.fstat(fd)
|
||||
if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != identity or opened.st_nlink != 1:
|
||||
return False
|
||||
write_all(fd, data)
|
||||
finally:
|
||||
os.close(fd)
|
||||
return True
|
||||
|
||||
def unlink(self, path: str) -> None:
|
||||
try:
|
||||
with self._relative(path) as (name, dir_fd):
|
||||
os.unlink(name, dir_fd=dir_fd)
|
||||
except OSError as error:
|
||||
# Windows refuses to remove a file another process holds open; the record then outlives its owner until
|
||||
# that process exits, which only delays the cleanup a later participant repeats.
|
||||
if error.errno not in _MISSING_ERRNOS and not isinstance(error, PermissionError):
|
||||
raise
|
||||
|
||||
def listdir(self, path: str) -> list[str]:
|
||||
try:
|
||||
with self._directory(path) as dir_fd, os.scandir(path if dir_fd is None else dir_fd) as entries:
|
||||
return [entry.name for entry in entries]
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
|
||||
def prepare(self, root: str, *, force: bool = False) -> None:
|
||||
if self._prepared and not force:
|
||||
return
|
||||
if force:
|
||||
self._identities.clear()
|
||||
ensure_directory_exists(root)
|
||||
for directory, depth in (
|
||||
(Path(root), 0),
|
||||
(Path(root, _GENERATIONS_DIRECTORY), 1),
|
||||
(Path(root, _HOLDERS_DIRECTORY), 1),
|
||||
):
|
||||
with self._relative(str(directory), depth) as (name, dir_fd):
|
||||
with suppress(FileExistsError):
|
||||
os.mkdir(name, 0o700, dir_fd=dir_fd)
|
||||
# mkdir has no O_NOFOLLOW, so check what the name resolves to before the lock creates anything in it.
|
||||
found = os.stat(name, dir_fd=dir_fd, follow_symlinks=False)
|
||||
parent = directory.parent.stat() if dir_fd is None else os.fstat(dir_fd)
|
||||
if (
|
||||
not stat.S_ISDIR(found.st_mode)
|
||||
or getattr(found, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT
|
||||
):
|
||||
msg = f"{directory} exists but is not a directory or is a symlink; refusing to use it"
|
||||
raise TreeReplacedError(msg)
|
||||
owner = foreign_directory_owner(found, parent)
|
||||
if owner is not None: # pragma: needs foreign-sticky-tmp
|
||||
msg = f"{directory} belongs to uid {owner}, not to this process's account or the parent's owner"
|
||||
raise TreeReplacedError(msg)
|
||||
self._identities[str(directory)] = found.st_dev, found.st_ino
|
||||
self._prepared = True
|
||||
|
||||
def _open(self, path: str, flags: int) -> int | None:
|
||||
# Missing is the one answer a caller acts on without raising. A refused open is real on POSIX, where nothing
|
||||
# this lock does makes its own records unreadable; on Windows it is usually another process mid-delete or
|
||||
# holding the file without sharing, so it is retried through a short grace before it counts, and a file still
|
||||
# refusing after that has been in delete-pending for longer than any deletion takes, so it is on its way out:
|
||||
# missing.
|
||||
deadline: float | None = None
|
||||
while True:
|
||||
if isinstance(opened := self._attempt_open(path, flags), int):
|
||||
return opened
|
||||
if opened.errno in _MISSING_ERRNOS:
|
||||
return None
|
||||
if not isinstance(opened, PermissionError) or sys.platform != "win32":
|
||||
raise opened
|
||||
if deadline is None: # pragma: win32 cover
|
||||
deadline = time.monotonic() + _WINDOWS_OPEN_GRACE
|
||||
elif time.monotonic() >= deadline: # pragma: win32 cover
|
||||
return None
|
||||
time.sleep(0.002) # pragma: win32 cover
|
||||
|
||||
def _attempt_open(self, path: str, flags: int) -> int | OSError:
|
||||
try:
|
||||
with self._relative(path) as (name, dir_fd):
|
||||
return os.open(name, flags, dir_fd=dir_fd)
|
||||
except OSError as error:
|
||||
return error
|
||||
|
||||
@contextmanager
|
||||
def _relative(self, path: str, depth: int = 2) -> Generator[tuple[str, int | None]]:
|
||||
directory, name = os.path.split(path)
|
||||
with self._directory(directory, depth) as dir_fd:
|
||||
yield (path if dir_fd is None else name), dir_fd
|
||||
|
||||
@contextmanager
|
||||
def _directory(self, path: str, depth: int = 2) -> Generator[int | None]:
|
||||
# O_NOFOLLOW applies to the last path component. The check prepare() made on <root> and <root>/gen holds until a
|
||||
# process that can write the lock's directory renames the tree aside and puts one with a symlinked gen in its
|
||||
# place. Each call therefore opens the root and then the record's directory without following symlinks and
|
||||
# works inside what it opened. No descriptor outlives the call, so a later call finds a log removed and started
|
||||
# again by name, as before. None means descriptors cannot anchor a call and the call uses the whole path.
|
||||
if depth and _SUPPORTS_DIR_FD: # pragma: needs link-dir-fd
|
||||
with self._relative(path, depth - 1) as (name, parent_fd):
|
||||
try:
|
||||
fd = os.open(name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | _O_NOFOLLOW, dir_fd=parent_fd)
|
||||
except OSError as error:
|
||||
# NetBSD reports EFTYPE rather than ELOOP when O_NOFOLLOW refuses a symlink.
|
||||
if error.errno not in {ELOOP, ENOTDIR, getattr(errno, "EFTYPE", ELOOP)}:
|
||||
raise
|
||||
msg = f"{path} is not a directory or is a symlink; refusing to use it"
|
||||
raise TreeReplacedError(msg) from error
|
||||
try:
|
||||
self._admit(fd, path, parent_fd)
|
||||
except BaseException:
|
||||
os.close(fd)
|
||||
raise
|
||||
try:
|
||||
yield fd
|
||||
finally:
|
||||
os.close(fd)
|
||||
else:
|
||||
# Without dir_fd, hold each of those directories open instead, so no process can rename or replace one.
|
||||
try:
|
||||
pins = _pin_directories(*[Path(path), *Path(path).parents][:depth][::-1])
|
||||
except NotADirectoryError as error: # pragma: win32 cover
|
||||
msg = f"{error.filename} is not a directory or is a symlink; refusing to use it"
|
||||
raise TreeReplacedError(msg) from error
|
||||
with pins:
|
||||
yield None
|
||||
|
||||
def _admit(self, fd: int, path: str, parent_fd: int | None) -> None: # pragma: needs link-dir-fd
|
||||
# A directory other than the one prepare() checked may belong to another account: the tree can be renamed aside
|
||||
# and recreated while this object lives, and the ownership check has to hold for what is opened now.
|
||||
found = os.fstat(fd)
|
||||
if (found.st_dev, found.st_ino) == self._identities.get(path):
|
||||
return
|
||||
parent = os.fstat(parent_fd) if parent_fd is not None else Path(path).parent.stat()
|
||||
if (owner := foreign_directory_owner(found, parent)) is not None: # pragma: needs foreign-sticky-tmp
|
||||
msg = f"{path} belongs to uid {owner}, not to this process's account or the parent's owner"
|
||||
raise TreeReplacedError(msg)
|
||||
self._identities[path] = found.st_dev, found.st_ino
|
||||
|
||||
|
||||
def _identity(path: str, dir_fd: int | None) -> tuple[int, int] | None:
|
||||
try:
|
||||
st = os.stat(path, dir_fd=dir_fd, follow_symlinks=False)
|
||||
except OSError:
|
||||
return None
|
||||
return st.st_dev, st.st_ino
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OsFiles",
|
||||
]
|
||||
@@ -0,0 +1,793 @@
|
||||
"""Cross-process and cross-host reader/writer lock over a generation log of immutable snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = [
|
||||
"filelock._error",
|
||||
"filelock._lease",
|
||||
"filelock._soft_rw._protocol",
|
||||
"filelock._soft_rw._storage",
|
||||
"math",
|
||||
]
|
||||
|
||||
import atexit
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from math import isfinite
|
||||
from typing import TYPE_CHECKING, ClassVar, Final
|
||||
from weakref import WeakValueDictionary
|
||||
|
||||
from filelock._api import (
|
||||
AcquireReturnProxy,
|
||||
_acquire_thread_lock,
|
||||
_canonical,
|
||||
_check_timeout_max,
|
||||
_ensure_current_process,
|
||||
_register_fork_class,
|
||||
_register_fork_object,
|
||||
_resolve_read_write_timeout,
|
||||
_seconds,
|
||||
_timeout_behavior,
|
||||
)
|
||||
from filelock._error import Timeout
|
||||
from filelock._lease import LeaseCompromise
|
||||
from filelock._soft_rw._protocol import GenerationLog, Ledger, Mode, Participant
|
||||
from filelock._soft_rw._storage import OsFiles
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable, Generator
|
||||
|
||||
from filelock._lease import CompromiseReason
|
||||
|
||||
_ALL_INSTANCES: Final[WeakValueDictionary[int, SoftReadWriteLock]] = WeakValueDictionary()
|
||||
_ALL_INSTANCES_LOCK: threading.Lock = threading.Lock()
|
||||
|
||||
|
||||
class _SoftRWMeta(type):
|
||||
_instances: WeakValueDictionary[str, SoftReadWriteLock]
|
||||
_instances_lock: threading.RLock
|
||||
_instances_under_construction: set[str]
|
||||
|
||||
def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters
|
||||
cls,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = True,
|
||||
heartbeat_interval: float = 30.0,
|
||||
stale_threshold: float | None = None,
|
||||
poll_interval: float = 0.25,
|
||||
on_compromise: Callable[[LeaseCompromise], None] | None = None,
|
||||
) -> SoftReadWriteLock:
|
||||
_ensure_current_process()
|
||||
timeout = _seconds("timeout", timeout)
|
||||
# Passed through only when set, so a subclass that declares its own constructor without it keeps working.
|
||||
extra = {} if on_compromise is None else {"on_compromise": on_compromise}
|
||||
if not is_singleton:
|
||||
return super().__call__(
|
||||
lock_file,
|
||||
timeout,
|
||||
blocking=blocking,
|
||||
is_singleton=is_singleton,
|
||||
heartbeat_interval=heartbeat_interval,
|
||||
stale_threshold=stale_threshold,
|
||||
poll_interval=poll_interval,
|
||||
**extra,
|
||||
)
|
||||
|
||||
# Keyed like the log root, so one instance never spans two logs.
|
||||
normalized = _canonical(lock_file)
|
||||
with cls._instances_lock:
|
||||
instance = cls._instances.get(normalized)
|
||||
if instance is None:
|
||||
if normalized in cls._instances_under_construction:
|
||||
msg = f"Singleton lock construction is already active for {lock_file!s}"
|
||||
raise RuntimeError(msg)
|
||||
construction_pid = os.getpid()
|
||||
construction_registry = cls._instances_under_construction
|
||||
construction_registry.add(normalized)
|
||||
try:
|
||||
instance = super().__call__(
|
||||
lock_file,
|
||||
timeout,
|
||||
blocking=blocking,
|
||||
is_singleton=is_singleton,
|
||||
heartbeat_interval=heartbeat_interval,
|
||||
stale_threshold=stale_threshold,
|
||||
poll_interval=poll_interval,
|
||||
**extra,
|
||||
)
|
||||
finally:
|
||||
construction_registry.discard(normalized)
|
||||
if os.getpid() != construction_pid: # pragma: needs fork
|
||||
msg = "Lock construction cannot continue after fork; construct a new lock in the child"
|
||||
raise RuntimeError(msg)
|
||||
cls._instances[normalized] = instance
|
||||
elif (
|
||||
_timeout_behavior(instance.timeout, any_negative=False)
|
||||
!= _timeout_behavior(timeout, any_negative=False)
|
||||
or instance.blocking != blocking
|
||||
):
|
||||
msg = (
|
||||
f"Singleton lock created with timeout={instance.timeout}, blocking={instance.blocking},"
|
||||
f" cannot be changed to timeout={timeout}, blocking={blocking}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
elif on_compromise is not None and instance._on_compromise != on_compromise: # ruff: ignore[private-member-access] # the metaclass owns the singleton it compares
|
||||
# A tuning difference keeps the first caller's values; a different loss callback is a different safety
|
||||
# contract, and returning the cached instance would silently drop it.
|
||||
msg = f"Singleton lock created with a different on_compromise callback for {lock_file!s}"
|
||||
raise ValueError(msg)
|
||||
else:
|
||||
_validate_intervals(heartbeat_interval, stale_threshold, poll_interval)
|
||||
return instance
|
||||
|
||||
|
||||
class SoftReadWriteLock(metaclass=_SoftRWMeta):
|
||||
"""
|
||||
Cross-process and cross-host reader/writer lock for shared filesystems.
|
||||
|
||||
Use this class instead of :class:`~filelock.ReadWriteLock` when the lock file lives on a network filesystem (NFS,
|
||||
Lustre, HPC cluster shared storage). ``ReadWriteLock`` is backed by SQLite and cannot run on NFS because SQLite's
|
||||
``fcntl`` locking is unreliable there.
|
||||
|
||||
The lock's state is a log of immutable snapshots under ``foo.lock.rw/gen/<N>``, each naming the writer and the
|
||||
readers holding the lock at generation ``N``. Every acquire, release, and eviction publishes the next snapshot with
|
||||
one atomic no-replace hard link, so no transition can be left half done by a crash on any host. Each participant
|
||||
also keeps ``foo.lock.rw/holders/<token>``, a record whose nonce a daemon heartbeat thread rewrites every
|
||||
``heartbeat_interval`` seconds. A contender evicts a member whose record has not changed for ``stale_threshold``
|
||||
seconds of the contender's own monotonic clock; no clock is ever compared across hosts, so skew between hosts or
|
||||
against the file server cannot make a live holder look dead.
|
||||
|
||||
Writer acquire is writer-preferring: a writer enters the snapshot as soon as no live writer is named, which blocks
|
||||
any new reader, then waits for the named readers to leave. Writer starvation is impossible.
|
||||
|
||||
An expired holder is not fenced by the lock: a process paused past ``stale_threshold`` resumes believing it holds
|
||||
the lock until its next heartbeat reports the loss through ``on_compromise``. :attr:`generation` is a monotonic
|
||||
fencing token for the protected resource: a store that rejects writes carrying a lower generation than the highest
|
||||
it has accepted refuses such a holder.
|
||||
|
||||
Reentrancy, upgrade/downgrade rules, thread pinning, and singleton caching by canonical path match
|
||||
:class:`~filelock.ReadWriteLock`.
|
||||
|
||||
Forking invalidates the inherited instance in the child so the child cannot double-own the lock with its parent;
|
||||
``release()`` on that instance is a no-op, and the child must construct a new instance if it needs a lock.
|
||||
|
||||
Trust boundary: protects against same-UID non-cooperating processes (one host or cross-host) and same-host
|
||||
different-UID users via ``0o600`` / ``0o700`` permissions. Does not protect against root compromise or multi-tenant
|
||||
mounts where hostile co-tenants share the UID.
|
||||
|
||||
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
|
||||
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately on contention
|
||||
:param is_singleton: if ``True``, reuse existing instances for the same canonical path
|
||||
:param heartbeat_interval: seconds between heartbeat refreshes; default 30 s
|
||||
:param stale_threshold: seconds a holder record may stay unchanged before a contender evicts it; defaults to
|
||||
``3 * heartbeat_interval``, matching etcd's ``LeaseKeepAlive`` convention
|
||||
:param poll_interval: seconds between acquire retries under contention; default 0.25 s
|
||||
:param on_compromise: called from the heartbeat thread with a :class:`~filelock.LeaseCompromise` when the hold is
|
||||
lost: a peer evicted it, or refreshes failed for long enough that a peer could have
|
||||
|
||||
.. versionadded:: 3.27.0
|
||||
|
||||
.. versionchanged:: 3.33.0
|
||||
|
||||
The on-disk protocol is a generation log; earlier releases cannot share a lock path with this one.
|
||||
|
||||
"""
|
||||
|
||||
_instances: WeakValueDictionary[str, SoftReadWriteLock] = WeakValueDictionary()
|
||||
_instances_lock = threading.RLock()
|
||||
_instances_under_construction: ClassVar[set[str]] = set()
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None:
|
||||
"""Give each lock subclass its own singleton registry and lock."""
|
||||
super().__init_subclass__(**kwargs)
|
||||
cls._instances = WeakValueDictionary()
|
||||
cls._instances_lock = threading.RLock()
|
||||
cls._instances_under_construction = set()
|
||||
_register_fork_class(cls)
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
|
||||
self,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = True, # ruff:ignore[unused-method-argument] # consumed by _SoftRWMeta.__call__
|
||||
heartbeat_interval: float = 30.0,
|
||||
stale_threshold: float | None = None,
|
||||
poll_interval: float = 0.25,
|
||||
on_compromise: Callable[[LeaseCompromise], None] | None = None,
|
||||
) -> None:
|
||||
self._creator_pid = os.getpid()
|
||||
heartbeat_interval, stale_threshold, poll_interval = _validate_intervals(
|
||||
heartbeat_interval, stale_threshold, poll_interval
|
||||
)
|
||||
|
||||
self.lock_file: str = os.fspath(lock_file)
|
||||
self.timeout: float = timeout
|
||||
self.blocking: bool = blocking
|
||||
self.heartbeat_interval: float = heartbeat_interval
|
||||
self.stale_threshold: float = stale_threshold
|
||||
self.poll_interval: float = poll_interval
|
||||
|
||||
# Resolved once: a relative lock path must keep naming the same log after the process changes directory. The
|
||||
# final component stays unresolved, as in 4.0.12, so every release shares one log on a symlinked lock path.
|
||||
self._root = f"{_canonical(self.lock_file)}.rw"
|
||||
self._files = OsFiles(self.lock_file)
|
||||
self._ledger = Ledger(time.monotonic)
|
||||
self._log = GenerationLog(self._files, self.lock_file, self._root)
|
||||
self._on_compromise = on_compromise
|
||||
self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock())
|
||||
self._hold: _Hold | None = None
|
||||
self._compromise: LeaseCompromise | None = None
|
||||
self._closed: bool = False
|
||||
|
||||
with _ALL_INSTANCES_LOCK:
|
||||
_ALL_INSTANCES[id(self)] = self
|
||||
_register_fork_object(self)
|
||||
|
||||
@classmethod
|
||||
def _reset_class_after_fork(cls) -> None: # pragma: forked child
|
||||
global _ALL_INSTANCES_LOCK # ruff:ignore[global-statement] # rebinds the module lock to a fresh one in the fork child
|
||||
_ALL_INSTANCES_LOCK = threading.Lock()
|
||||
cls._instances = WeakValueDictionary()
|
||||
cls._instances_lock = threading.RLock()
|
||||
cls._instances_under_construction = set()
|
||||
|
||||
@property
|
||||
def generation(self) -> int | None:
|
||||
"""
|
||||
The generation at which the current hold was granted, or ``None`` when no lock is held.
|
||||
|
||||
Generations are monotonic across all participants and hosts, so this is a fencing token: pass it to the
|
||||
protected resource and have the resource reject any operation carrying a lower generation than the highest it
|
||||
has accepted. That refuses a holder that paused past ``stale_threshold``, was evicted, and resumed. Removing the
|
||||
``<lock_file>.rw`` directory starts a new log whose generations begin again at 1.
|
||||
|
||||
.. versionadded:: 3.33.0
|
||||
|
||||
"""
|
||||
with self._locks.internal:
|
||||
return None if self._hold is None else self._hold.participant.generation
|
||||
|
||||
@property
|
||||
def compromise(self) -> LeaseCompromise | None:
|
||||
"""
|
||||
How the current hold was lost, or ``None`` while it stands.
|
||||
|
||||
Set by the heartbeat thread once a peer has evicted this holder or refreshes have failed for long enough that a
|
||||
peer could have. The holder should stop using the protected resource when it is set. Until the lost hold is
|
||||
released, acquiring on this instance raises :class:`RuntimeError` instead of joining it.
|
||||
|
||||
.. versionadded:: 3.33.0
|
||||
|
||||
"""
|
||||
with self._locks.internal:
|
||||
return self._compromise
|
||||
|
||||
@contextmanager
|
||||
def read_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
|
||||
"""
|
||||
Context manager that acquires and releases a shared read lock.
|
||||
|
||||
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if a write lock is already held on this instance
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self.acquire_read(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
@contextmanager
|
||||
def write_lock(self, timeout: float | None = None, *, blocking: bool | None = None) -> Generator[None]:
|
||||
"""
|
||||
Context manager that acquires and releases an exclusive write lock.
|
||||
|
||||
Falls back to instance defaults for *timeout* and *blocking* when ``None``.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately; ``None`` uses the instance default
|
||||
|
||||
:raises RuntimeError: if a read lock is already held, or a write lock is held by a different thread
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
|
||||
"""
|
||||
self.acquire_write(timeout, blocking=blocking)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
def acquire_read(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
|
||||
"""
|
||||
Acquire a shared read lock.
|
||||
|
||||
If this instance already holds a read lock, the lock level is incremented (reentrant). Attempting to acquire a
|
||||
read lock while holding a write lock raises :class:`RuntimeError` (downgrade not allowed). On the 0→1
|
||||
transition the reader publishes its holder record, enters the next snapshot once no live writer is named, and
|
||||
starts a daemon heartbeat thread that rewrites the record's nonce every ``heartbeat_interval`` seconds so peers
|
||||
on other hosts do not evict it.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
|
||||
indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
|
||||
``None`` uses the instance default
|
||||
|
||||
:returns: a proxy that can be used as a context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if a write lock is already held on this instance, if this instance was invalidated by
|
||||
:func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
:raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem
|
||||
refuses the no-replace hard links the protocol commits with
|
||||
|
||||
"""
|
||||
return self._acquire("read", timeout, blocking=blocking)
|
||||
|
||||
def acquire_write(self, timeout: float | None = None, *, blocking: bool | None = None) -> AcquireReturnProxy:
|
||||
"""
|
||||
Acquire an exclusive write lock.
|
||||
|
||||
If this instance already holds a write lock from the same thread, the lock level is incremented (reentrant).
|
||||
Attempting to acquire a write lock while holding a read lock raises :class:`RuntimeError` (upgrade not
|
||||
allowed). Write locks are pinned to the acquiring thread: a different thread trying to re-enter also raises
|
||||
:class:`RuntimeError`.
|
||||
|
||||
A writer enters the next snapshot as its writer as soon as no live writer is named, which blocks every new
|
||||
reader on every host, then waits for the readers that snapshot names to leave. Writer starvation is impossible:
|
||||
new readers see the named writer and wait behind it.
|
||||
|
||||
:param timeout: maximum wait time in seconds, or ``None`` to use the instance default; ``-1`` means block
|
||||
indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable;
|
||||
``None`` uses the instance default
|
||||
|
||||
:returns: a proxy that can be used as a context manager to release the lock
|
||||
|
||||
:raises RuntimeError: if a read lock is already held, if a write lock is held by a different thread, if this
|
||||
instance was invalidated by :func:`os.fork`, or if :meth:`close` was called
|
||||
:raises Timeout: if the lock cannot be acquired within *timeout* seconds
|
||||
:raises TypeError: if *timeout* is a :class:`bool` or not a real number
|
||||
:raises ValueError: if a blocking call gets a ``nan`` *timeout* or a negative one other than ``-1``
|
||||
:raises SoftFileLockProtocolError: if a snapshot cannot be read without risking overlap, or the filesystem
|
||||
refuses the no-replace hard links the protocol commits with
|
||||
|
||||
"""
|
||||
return self._acquire("write", timeout, blocking=blocking)
|
||||
|
||||
@classmethod
|
||||
def get_lock(
|
||||
cls,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
*,
|
||||
blocking: bool = True,
|
||||
) -> SoftReadWriteLock:
|
||||
"""
|
||||
Return the singleton :class:`SoftReadWriteLock` for *lock_file*.
|
||||
|
||||
:param lock_file: path to the lock file; the protocol directory lives next to it as ``<lock_file>.rw``
|
||||
:param timeout: maximum wait time in seconds; ``-1`` means block indefinitely
|
||||
:param blocking: if ``False``, raise :class:`~filelock.Timeout` immediately when the lock is unavailable
|
||||
|
||||
:returns: the singleton lock instance
|
||||
|
||||
:raises ValueError: if an instance already exists for this path with different *timeout* or *blocking* values
|
||||
|
||||
"""
|
||||
return cls(lock_file, timeout, blocking=blocking)
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Release any held lock and mark the instance closed.
|
||||
|
||||
Idempotent. After calling this method the instance can no longer acquire locks — subsequent acquires raise
|
||||
:class:`RuntimeError`, and so does an acquire another thread is still waiting in, which gives up its claim
|
||||
instead of taking the lock. A fork-invalidated instance is closed without raising.
|
||||
"""
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
return
|
||||
# Closed first, under the lock an acquire publishes its hold under: an acquire still waiting then sees the flag
|
||||
# and leaves instead of publishing, and one that published before it is released below.
|
||||
with self._locks.internal:
|
||||
self._closed = True
|
||||
self.release(force=True)
|
||||
|
||||
def release(self, *, force: bool = False) -> None:
|
||||
"""
|
||||
Release one level of the current lock.
|
||||
|
||||
When the lock level reaches zero the heartbeat thread is stopped, the participant is committed out of the next
|
||||
snapshot, and its holder record is removed. On a fork-invalidated instance (that is, the child of a
|
||||
:func:`os.fork` call made while the parent held a lock) this method is a no-op so inherited ``with`` blocks can
|
||||
unwind cleanly in the child.
|
||||
|
||||
:param force: if ``True``, release the lock completely regardless of the current lock level
|
||||
|
||||
:raises RuntimeError: if no lock is currently held and *force* is ``False``, or if the calling thread holds no
|
||||
level of the current lock and *force* is ``False``
|
||||
|
||||
"""
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
return
|
||||
self._release(threading.get_ident(), force=force)
|
||||
|
||||
def _acquire_level_of(
|
||||
self, holder: int, mode: Mode, timeout: float, *, blocking: bool, cancel_check: Callable[[], bool]
|
||||
) -> AcquireReturnProxy:
|
||||
# Records the level under *holder* instead of the calling thread: an async wrapper's executor thread does not
|
||||
# identify the tasks it holds for. Gives up with Timeout once *cancel_check* is true.
|
||||
return self._acquire(mode, timeout, blocking=blocking, holder=holder, cancel_check=cancel_check)
|
||||
|
||||
def _release_level_of(self, holder: int) -> None:
|
||||
self._release(holder, force=False)
|
||||
|
||||
def _holds_level_of(self, holder: int) -> bool:
|
||||
# Read without the internal lock: an async wrapper asks from its event loop, which must not wait out a release
|
||||
# running on another thread.
|
||||
return (hold := self._hold) is not None and (holder == hold.write_thread_id or holder in hold.read_depths)
|
||||
|
||||
def _release(self, thread_id: int, *, force: bool) -> None:
|
||||
with self._locks.internal:
|
||||
hold = self._hold
|
||||
if hold is None:
|
||||
if force:
|
||||
return
|
||||
msg = f"Cannot release a lock on {self.lock_file} (lock id: {id(self)}) that is not held"
|
||||
raise RuntimeError(msg)
|
||||
if force:
|
||||
hold.level = 0
|
||||
else:
|
||||
self._raise_if_not_holder(hold, thread_id)
|
||||
hold.leave_one_level(thread_id)
|
||||
if hold.level > 0:
|
||||
return
|
||||
self._hold = None
|
||||
|
||||
# Stop the heartbeat before leaving, so a late tick cannot re-read a snapshot this participant just left and
|
||||
# report an eviction that was its own release. on_compromise runs on that thread and may be what called here,
|
||||
# in which case there is nothing to join.
|
||||
hold.heartbeat_stop.set()
|
||||
try:
|
||||
if hold.heartbeat_thread is not threading.current_thread():
|
||||
hold.heartbeat_thread.join(timeout=min(self.heartbeat_interval + 1.0, threading.TIMEOUT_MAX))
|
||||
finally:
|
||||
# The hold is already gone from this instance, so a join that raises must not strand its record too.
|
||||
hold.participant.leave()
|
||||
|
||||
def _raise_if_not_holder(self, hold: _Hold, thread_id: int) -> None:
|
||||
# The hold's own heartbeat thread may release it, since on_compromise runs there.
|
||||
if thread_id == hold.heartbeat_thread.ident:
|
||||
return
|
||||
# Another thread's release would leave the generation log under the holder and let a peer in.
|
||||
if hold.mode == "write" and thread_id != hold.write_thread_id:
|
||||
msg = (
|
||||
f"Cannot release write lock on {self.lock_file} (lock id: {id(self)}) "
|
||||
f"from thread {thread_id} while it is held by thread {hold.write_thread_id}"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
# Readers share one participant, so a thread that holds no read level would end another's hold.
|
||||
if hold.mode == "read" and thread_id not in hold.read_depths:
|
||||
msg = (
|
||||
f"Cannot release read lock on {self.lock_file} (lock id: {id(self)}) "
|
||||
f"from thread {thread_id}, which does not hold it"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
|
||||
def _acquire(
|
||||
self,
|
||||
mode: Mode,
|
||||
timeout: float | None,
|
||||
*,
|
||||
blocking: bool | None,
|
||||
holder: int | None = None,
|
||||
cancel_check: Callable[[], bool] | None = None,
|
||||
) -> AcquireReturnProxy:
|
||||
if self._creator_pid != os.getpid(): # pragma: forked child
|
||||
msg = f"SoftReadWriteLock on {self.lock_file} was invalidated by fork(); construct a new instance"
|
||||
raise RuntimeError(msg)
|
||||
timeout = self.timeout if timeout is None else timeout
|
||||
blocking = self.blocking if blocking is None else blocking
|
||||
holder = threading.get_ident() if holder is None else holder
|
||||
|
||||
with self._locks.internal:
|
||||
if self._closed:
|
||||
raise self._closed_error()
|
||||
timeout = _resolve_read_write_timeout(timeout, blocking=blocking)
|
||||
if self._hold is not None:
|
||||
return self._validate_reentrant(mode, holder)
|
||||
|
||||
deadline = None if timeout == -1 else time.perf_counter() + timeout
|
||||
if not _acquire_thread_lock(
|
||||
self._locks.transaction,
|
||||
blocking=blocking,
|
||||
timeout=timeout,
|
||||
poll_interval=self.poll_interval,
|
||||
cancel_check=cancel_check,
|
||||
):
|
||||
raise Timeout(self.lock_file) from None
|
||||
try:
|
||||
return self._do_acquire_inner(mode, deadline, holder, blocking=blocking, cancel_check=cancel_check)
|
||||
finally:
|
||||
self._locks.transaction.release()
|
||||
|
||||
def _closed_error(self) -> RuntimeError:
|
||||
return RuntimeError(f"SoftReadWriteLock on {self.lock_file} has been closed")
|
||||
|
||||
def _do_acquire_inner(
|
||||
self,
|
||||
mode: Mode,
|
||||
deadline: float | None,
|
||||
holder: int,
|
||||
*,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
) -> AcquireReturnProxy:
|
||||
with self._locks.internal:
|
||||
if self._hold is not None:
|
||||
return self._validate_reentrant(mode, holder)
|
||||
participant = Participant(
|
||||
self._files,
|
||||
self._root,
|
||||
mode,
|
||||
stale_threshold=self.stale_threshold,
|
||||
clock=time.monotonic,
|
||||
ledger=self._ledger,
|
||||
log=self._log,
|
||||
)
|
||||
participant.publish()
|
||||
try:
|
||||
self._wait_for(participant.advance, deadline=deadline, blocking=blocking, cancel_check=cancel_check)
|
||||
except BaseException:
|
||||
# A contender that gives up must not stay named in the snapshot: a writer left there would block every
|
||||
# reader until a peer waits out the stale threshold. Its own token is all leave() touches, so it cannot
|
||||
# undo a peer's claim.
|
||||
with suppress(OSError):
|
||||
participant.leave()
|
||||
raise
|
||||
stop_event = threading.Event()
|
||||
hold = _Hold(
|
||||
level=1,
|
||||
mode=mode,
|
||||
write_thread_id=holder if mode == "write" else None,
|
||||
read_depths={holder: 1} if mode == "read" else {},
|
||||
participant=participant,
|
||||
heartbeat_thread=_HeartbeatThread(
|
||||
refresh=lambda: self._refresh(participant, hold),
|
||||
interval=self.heartbeat_interval,
|
||||
stop_event=stop_event,
|
||||
name=f"filelock-heartbeat-{id(self):x}",
|
||||
),
|
||||
heartbeat_stop=stop_event,
|
||||
last_refresh=time.monotonic(),
|
||||
)
|
||||
# Publish the hold and start its heartbeat under one internal-lock section, so a concurrent release() never
|
||||
# observes a hold whose thread has not started and joins it. If the OS refuses the thread, clear the hold and
|
||||
# leave: left in place, a peer evicts the unrefreshed record and acquires while this instance still believes it
|
||||
# holds the lock. A close() that landed while this acquire waited gets the same treatment: published, the hold
|
||||
# would keep its heartbeat and its slot on an instance nobody can release through.
|
||||
start_error: BaseException | None = None
|
||||
with self._locks.internal:
|
||||
if self._closed:
|
||||
start_error = self._closed_error()
|
||||
else:
|
||||
self._hold = hold
|
||||
self._compromise = None
|
||||
try:
|
||||
hold.heartbeat_thread.start()
|
||||
except BaseException as error: # ruff:ignore[blind-except] # clear the slot below and re-raise
|
||||
self._hold = None
|
||||
start_error = error
|
||||
if start_error is not None:
|
||||
participant.leave()
|
||||
raise start_error
|
||||
return AcquireReturnProxy(lock=self)
|
||||
|
||||
def _validate_reentrant(self, mode: Mode, holder: int) -> AcquireReturnProxy:
|
||||
hold = self._hold
|
||||
assert hold is not None # ruff:ignore[assert] # callers dispatch here only inside the self._hold is not None branch
|
||||
if hold.mode != mode:
|
||||
opposite = "write" if mode == "read" else "read"
|
||||
direction = "downgrade" if mode == "read" else "upgrade"
|
||||
msg = (
|
||||
f"Cannot acquire {mode} lock on {self.lock_file} (lock id: {id(self)}): "
|
||||
f"already holding a {opposite} lock ({direction} not allowed)"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
if mode == "write" and holder != hold.write_thread_id:
|
||||
msg = (
|
||||
f"Cannot acquire write lock on {self.lock_file} (lock id: {id(self)}) "
|
||||
f"from thread {holder} while it is held by thread {hold.write_thread_id}"
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
# Publishing a hold clears the loss recorded for the one before, so a loss seen here is this hold's. Its
|
||||
# heartbeat has stopped and a peer may hold the lock, so a new level must not bypass the log.
|
||||
if (compromise := self._compromise) is not None:
|
||||
raise _lost_hold_error(self.lock_file, mode, compromise)
|
||||
if mode == "read":
|
||||
hold.read_depths[holder] = hold.read_depths.get(holder, 0) + 1
|
||||
hold.level += 1
|
||||
return AcquireReturnProxy(lock=self)
|
||||
|
||||
def _wait_for(
|
||||
self,
|
||||
predicate: Callable[[], bool],
|
||||
*,
|
||||
deadline: float | None,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
) -> None:
|
||||
while not predicate():
|
||||
# The flag is read without the lock here only to stop waiting early; publishing the hold re-checks it.
|
||||
if self._closed:
|
||||
raise self._closed_error()
|
||||
now = time.perf_counter()
|
||||
if not blocking or (deadline is not None and now >= deadline):
|
||||
raise Timeout(self.lock_file)
|
||||
# An abandoned acquire gives up like a timed-out one, so the caller's rollback withdraws its claim.
|
||||
if cancel_check is not None and cancel_check():
|
||||
raise Timeout(self.lock_file)
|
||||
sleep_for = self.poll_interval
|
||||
if deadline is not None:
|
||||
sleep_for = min(sleep_for, max(deadline - now, 0.0))
|
||||
time.sleep(sleep_for)
|
||||
|
||||
def _refresh(self, participant: Participant, hold: _Hold) -> bool:
|
||||
# The loop ends at the first loss, so the holder hears about it once. A transient filesystem error (ESTALE /
|
||||
# EIO on the NFS-style filesystems this lock targets) is not a loss: retry rather than report a false
|
||||
# compromise. Report the record unrefreshable only once failures have run long enough that a peer could evict
|
||||
# it before the next success would land, a margin before the record actually ages out.
|
||||
try:
|
||||
outcome, error = participant.heartbeat()
|
||||
except Exception as failure: # ruff:ignore[blind-except] # the thread would end with the hold still published
|
||||
self._report_compromise(hold, "refresh-failed", failure)
|
||||
return False
|
||||
if outcome == "ok":
|
||||
hold.last_refresh = time.monotonic()
|
||||
return True
|
||||
if outcome == "lost":
|
||||
self._report_compromise(hold, "evicted", None)
|
||||
return False
|
||||
if time.monotonic() - hold.last_refresh >= self.stale_threshold - self.heartbeat_interval:
|
||||
self._report_compromise(hold, "refresh-failed", error)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _report_compromise(self, hold: _Hold, reason: CompromiseReason, error: Exception | None) -> None:
|
||||
# A tick that outlived its release's join must not stamp the old hold's loss onto whatever was acquired since.
|
||||
with self._locks.internal:
|
||||
if self._hold is not hold:
|
||||
return
|
||||
compromise = LeaseCompromise(
|
||||
lock_file=self.lock_file, token=hold.participant.token, reason=reason, error=error
|
||||
)
|
||||
self._compromise = compromise
|
||||
if self._on_compromise is not None:
|
||||
self._on_compromise(compromise)
|
||||
|
||||
def _reset_after_fork_in_child(self) -> None: # pragma: forked child
|
||||
self._locks = _Locks(internal=threading.Lock(), transaction=threading.Lock())
|
||||
self._hold = None
|
||||
|
||||
|
||||
class _HeartbeatThread(threading.Thread):
|
||||
def __init__(
|
||||
self,
|
||||
refresh: Callable[[], bool],
|
||||
interval: float,
|
||||
stop_event: threading.Event,
|
||||
name: str,
|
||||
) -> None:
|
||||
super().__init__(name=name, daemon=True)
|
||||
self._refresh = refresh
|
||||
self._interval = interval
|
||||
self._stop_event = stop_event
|
||||
|
||||
def run(self) -> None:
|
||||
# Set on every exit, an on_compromise that raises included, so nothing waits on a thread that is gone.
|
||||
try:
|
||||
while not self._stop_event.wait(self._interval):
|
||||
if not self._refresh():
|
||||
return
|
||||
finally:
|
||||
self._stop_event.set()
|
||||
|
||||
|
||||
def _lost_hold_error(lock_file: str, mode: Mode, compromise: LeaseCompromise) -> RuntimeError:
|
||||
return RuntimeError(
|
||||
f"Cannot acquire {mode} lock on {lock_file}: the current hold was lost ({compromise.reason}), "
|
||||
"release it before acquiring again"
|
||||
)
|
||||
|
||||
|
||||
def _validate_intervals(
|
||||
heartbeat_interval: float, stale_threshold: float | None, poll_interval: float
|
||||
) -> tuple[float, float, float]:
|
||||
heartbeat_interval = _seconds("heartbeat_interval", heartbeat_interval)
|
||||
if not isfinite(heartbeat_interval) or heartbeat_interval <= 0:
|
||||
msg = f"heartbeat_interval must be positive and finite, got {heartbeat_interval}"
|
||||
raise ValueError(msg)
|
||||
# The heartbeat thread waits this long between refreshes, and Event.wait overflows past TIMEOUT_MAX.
|
||||
_check_timeout_max("heartbeat_interval", heartbeat_interval)
|
||||
stale_threshold = (
|
||||
heartbeat_interval * 3 if stale_threshold is None else _seconds("stale_threshold", stale_threshold)
|
||||
)
|
||||
if not isfinite(stale_threshold) or stale_threshold <= heartbeat_interval:
|
||||
msg = (
|
||||
f"stale_threshold must exceed heartbeat_interval ({heartbeat_interval}) "
|
||||
f"and be finite, got {stale_threshold}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
if not isfinite(poll_interval := _seconds("poll_interval", poll_interval)) or poll_interval <= 0:
|
||||
msg = f"poll_interval must be positive and finite, got {poll_interval}"
|
||||
raise ValueError(msg)
|
||||
_check_timeout_max("poll_interval", poll_interval)
|
||||
# A waiting writer refreshes its record once per poll, so a poll slower than the threshold reads as dead.
|
||||
if poll_interval >= stale_threshold:
|
||||
msg = f"poll_interval must be below stale_threshold ({stale_threshold}), got {poll_interval}"
|
||||
raise ValueError(msg)
|
||||
return heartbeat_interval, stale_threshold, poll_interval
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Locks:
|
||||
internal: threading.Lock
|
||||
transaction: threading.Lock
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Hold:
|
||||
"""Everything that exists only while a lock is held; ``None`` when the instance has no lock."""
|
||||
|
||||
level: int
|
||||
mode: Mode
|
||||
write_thread_id: int | None
|
||||
#: Read levels per thread, so a thread can give back only the levels it took.
|
||||
read_depths: dict[int, int]
|
||||
participant: Participant
|
||||
heartbeat_thread: _HeartbeatThread
|
||||
heartbeat_stop: threading.Event
|
||||
last_refresh: float
|
||||
|
||||
def leave_one_level(self, thread_id: int) -> None:
|
||||
self.level -= 1
|
||||
if self.mode == "read":
|
||||
# The heartbeat thread holds no level of its own, so its release gives back one of a reader's.
|
||||
owner = thread_id if thread_id in self.read_depths else next(iter(self.read_depths))
|
||||
if depth := self.read_depths.pop(owner) - 1:
|
||||
self.read_depths[owner] = depth
|
||||
|
||||
|
||||
def _cleanup_all_instances() -> None: # pragma: no cover - runs from atexit at interpreter shutdown
|
||||
for instance in list(_ALL_INSTANCES.values()):
|
||||
with suppress(Exception):
|
||||
instance.release(force=True)
|
||||
|
||||
|
||||
atexit.register(_cleanup_all_instances)
|
||||
_register_fork_class(SoftReadWriteLock)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SoftReadWriteLock",
|
||||
"_lost_hold_error",
|
||||
]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,218 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["contextlib", "filelock._util", "pathlib", "typing", "warnings"]
|
||||
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from errno import EACCES, EAGAIN, ENOSYS, EWOULDBLOCK
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from filelock._api import BaseFileLock
|
||||
from filelock._util import remove_own_file
|
||||
|
||||
has_fcntl = False
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
|
||||
class UnixFileLock(BaseFileLock):
|
||||
"""Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems."""
|
||||
|
||||
def _acquire(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def _release(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
else: # pragma: win32 no cover
|
||||
try:
|
||||
import fcntl
|
||||
|
||||
_ = (fcntl.flock, fcntl.LOCK_EX, fcntl.LOCK_NB, fcntl.LOCK_UN)
|
||||
except (ImportError, AttributeError):
|
||||
_FCNTL_UNAVAILABLE: Final[str] = "fcntl is unavailable"
|
||||
|
||||
def _lock_fd_nonblocking(_fd: int) -> bool:
|
||||
raise OSError(ENOSYS, _FCNTL_UNAVAILABLE)
|
||||
|
||||
def _unlock_fd(_fd: int) -> None:
|
||||
raise OSError(ENOSYS, _FCNTL_UNAVAILABLE)
|
||||
|
||||
else:
|
||||
has_fcntl = True
|
||||
# Contention errnos for a nonblocking flock. EAGAIN/EWOULDBLOCK are the usual "held elsewhere" codes; some
|
||||
# filesystems report EACCES instead, so treat it as contention too rather than a permanent error.
|
||||
_CONTENTION_ERRNOS: Final[frozenset[int]] = frozenset({EACCES, EAGAIN, EWOULDBLOCK})
|
||||
|
||||
def _lock_fd_nonblocking(fd: int) -> bool:
|
||||
# One nonblocking exclusive flock attempt shared by UnixFileLock and lock_descriptor, so both contend on
|
||||
# the same lock and classify errors identically. The caller owns fd; this never closes it.
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as exception:
|
||||
if exception.errno in _CONTENTION_ERRNOS:
|
||||
return False
|
||||
raise
|
||||
return True
|
||||
|
||||
def _unlock_fd(fd: int) -> None:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
|
||||
def _flock_is_missing(fd: int) -> bool:
|
||||
try:
|
||||
_lock_fd_nonblocking(fd)
|
||||
except OSError as exception:
|
||||
return exception.errno == ENOSYS
|
||||
return False # a flock the probe won is released when the caller closes fd
|
||||
|
||||
class UnixFileLock(BaseFileLock):
|
||||
"""
|
||||
Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.
|
||||
|
||||
We leave the lock file in place after release. Unlinking a locked file on Unix splits
|
||||
waiters across inodes and breaks mutual exclusion for processes that coordinate via the
|
||||
same path.
|
||||
"""
|
||||
|
||||
def _acquire(self) -> None:
|
||||
if (missing_flock := self._acquire_native()) is not None:
|
||||
fd, exception, created = missing_flock
|
||||
self._switch_to_soft_lock(fd, exception, created=created)
|
||||
|
||||
def _acquire_native(self) -> tuple[int, OSError, bool] | None:
|
||||
if (opened := self._open_lock_file()) is None:
|
||||
return None
|
||||
fd, created = opened
|
||||
self._mark_descriptor_pending(fd)
|
||||
if created:
|
||||
# Before the flock: a peer may win it first, and only the creator may set the mode. Outside the flock
|
||||
# handler below: an fchmod that answers ENOSYS says nothing about flock.
|
||||
try:
|
||||
self._apply_explicit_mode(fd)
|
||||
except BaseException:
|
||||
self._mark_descriptor_released()
|
||||
os.close(fd)
|
||||
raise
|
||||
# One guard covers the flock through the recording of the hold. A signal handler that raises in between
|
||||
# would otherwise leave an unreferenced descriptor, and its flock, open until the process exits.
|
||||
locked = False
|
||||
try:
|
||||
locked = _lock_fd_nonblocking(fd)
|
||||
if locked and self._finalize_locked_fd(fd):
|
||||
return None
|
||||
except BaseException as exception:
|
||||
if not locked and isinstance(exception, OSError) and exception.errno == ENOSYS:
|
||||
return fd, exception, created
|
||||
# Contention returns False from _lock_fd_nonblocking, so a raise here is a failure.
|
||||
self._mark_descriptor_released()
|
||||
os.close(fd)
|
||||
raise
|
||||
self._mark_descriptor_released()
|
||||
os.close(fd) # contention or an unlinked inode; let the retry loop try again
|
||||
return None
|
||||
|
||||
def _open_lock_file(self) -> tuple[int, bool] | None:
|
||||
# No O_TRUNC: a contender that loses the flock must not truncate the holder's file (#591), and neither may
|
||||
# the winner, since a file renamed or hard-linked to the lock path is indistinguishable from a lock file.
|
||||
# O_EXCL tells whether this attempt created the inode: only then does it get the explicit mode, and the
|
||||
# ENOSYS fallback removes only its own placeholder. O_NONBLOCK: a lease (Linux F_SETLEASE) on the file would
|
||||
# otherwise block the open for up to fs.lease-break-time, beyond timeout, blocking=False and cancel_check.
|
||||
open_flags = os.O_RDWR | os.O_NONBLOCK
|
||||
if (o_nofollow := getattr(os, "O_NOFOLLOW", None)) is not None:
|
||||
open_flags |= o_nofollow
|
||||
open_mode = self._open_mode()
|
||||
try:
|
||||
return os.open(self._lock_path, open_flags | os.O_CREAT | os.O_EXCL, open_mode), True
|
||||
except FileExistsError:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
# On FUSE/NFS a split LOOKUP + CREATE lets a concurrent unlink() race the create, so ENOENT is transient
|
||||
# contention. An empty or separator-terminated path (Path() strips the separator) fails this way on
|
||||
# every attempt on macOS, so re-raise rather than retry forever.
|
||||
if self._lock_path and not self._lock_path.endswith(os.sep) and Path(self._lock_path).parent.exists():
|
||||
return None
|
||||
raise
|
||||
except PermissionError:
|
||||
# Sticky-bit dirs (e.g. /tmp): O_CREAT fails if another user owns the file (#317).
|
||||
if not Path(self._lock_path).exists():
|
||||
raise
|
||||
return self._open_existing_lock_file(open_flags, open_mode)
|
||||
|
||||
def _open_existing_lock_file(self, open_flags: int, open_mode: int) -> tuple[int, bool] | None:
|
||||
try:
|
||||
return os.open(self._lock_path, open_flags, open_mode), False
|
||||
except FileNotFoundError:
|
||||
return None # unlinked since the create attempt; let the retry loop start over
|
||||
except BlockingIOError:
|
||||
return None # a lease is breaking: contention, so the retry loop keeps the deadline
|
||||
except PermissionError:
|
||||
# A marker another account left is readable but not writable. Without flock, waiting on it only needs
|
||||
# reading it; with flock the lock writes its file, so the denial stands.
|
||||
try:
|
||||
fd = os.open(self.lock_file, open_flags & ~os.O_RDWR | os.O_RDONLY)
|
||||
except (FileNotFoundError, BlockingIOError):
|
||||
return None # unlinked, or leased: the kernel checks permission before the lease
|
||||
if _flock_is_missing(fd):
|
||||
return fd, False
|
||||
os.close(fd)
|
||||
raise
|
||||
|
||||
def _switch_to_soft_lock(self, fd: int, missing_flock: OSError, *, created: bool) -> None:
|
||||
# The filesystem does not implement flock. Fail closed when the caller opted out of existence-lock
|
||||
# semantics (#603), asked to preserve the pathname (#605), or set an on_acquired hook (#607), none of which
|
||||
# a soft lock can honor.
|
||||
falls_back: Final = self._fallback_to_soft and not self._preserve_lock_file and self._on_acquired is None
|
||||
try:
|
||||
# A file this attempt did not create may be a peer's live marker, so it stays. Our own placeholder goes
|
||||
# before the close: the open descriptor keeps its inode number from a peer's marker, and an exchange
|
||||
# puts back a peer's marker that replaced it.
|
||||
if created and falls_back:
|
||||
with suppress(OSError):
|
||||
remove_own_file(self._lock_path, fd)
|
||||
finally:
|
||||
self._mark_descriptor_released()
|
||||
os.close(fd)
|
||||
if not falls_back:
|
||||
raise missing_flock
|
||||
self._fallback_to_soft_lock()
|
||||
self._acquire()
|
||||
|
||||
def _finalize_locked_fd(self, fd: int) -> bool:
|
||||
# Runs with the flock held; the caller closes fd on failure. A concurrent _release() may have unlinked
|
||||
# the file between our open() and flock(), which st_nlink 0 shows. That lock guards no name, so report
|
||||
# it and let the retry loop start over.
|
||||
if (stat_result := os.fstat(fd)).st_nlink == 0:
|
||||
return False
|
||||
self._mark_descriptor_owned(fd, (stat_result.st_dev, stat_result.st_ino))
|
||||
return True
|
||||
|
||||
def _apply_explicit_mode(self, fd: int) -> None:
|
||||
if self.has_explicit_mode:
|
||||
with suppress(PermissionError):
|
||||
os.fchmod(fd, self._context.mode)
|
||||
|
||||
def _fallback_to_soft_lock(self: BaseFileLock) -> None:
|
||||
# Import lazily: this runs only on the rare flock fallback, and asyncio imports _unix, so a
|
||||
# module-level import of it here would cycle.
|
||||
from filelock._soft import SoftFileLock # ruff:ignore[import-outside-top-level]
|
||||
|
||||
warnings.warn("flock not supported on this filesystem, falling back to SoftFileLock", stacklevel=2)
|
||||
from filelock.asyncio import AsyncSoftFileLock, BaseAsyncFileLock # ruff:ignore[import-outside-top-level]
|
||||
|
||||
self.__class__ = AsyncSoftFileLock if isinstance(self, BaseAsyncFileLock) else SoftFileLock
|
||||
|
||||
def _release(self) -> None:
|
||||
fd = cast("int", self._context.lock_file_fd)
|
||||
# Retain the descriptor until flock succeeds: a failed unlock leaves the kernel lock held, so is_locked
|
||||
# must keep reporting held for a retry. Once flock commits, clear held state and close as post-unlock
|
||||
# cleanup; a close failure (EIO on FUSE/Docker bind mounts) does not make the kernel lock held again.
|
||||
_unlock_fd(fd)
|
||||
self._mark_descriptor_released()
|
||||
self._close_released_fd(fd, default_suppresses=True)
|
||||
|
||||
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
__all__ = ["UnixFileLock", "has_fcntl"]
|
||||
else: # pragma: win32 no cover
|
||||
__all__ = ["UnixFileLock", "_lock_fd_nonblocking", "_unlock_fd", "has_fcntl"]
|
||||
@@ -0,0 +1,399 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["pathlib", "platform", "secrets"]
|
||||
|
||||
import errno
|
||||
import os
|
||||
import platform
|
||||
import secrets
|
||||
import stat
|
||||
import sys
|
||||
from contextlib import contextmanager, suppress
|
||||
from errno import EACCES, EINVAL, EIO, EISDIR, ENOSYS, EXDEV
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import ctypes
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import AbstractContextManager
|
||||
|
||||
|
||||
def write_all(fd: int, data: bytes) -> None:
|
||||
"""
|
||||
Write the whole buffer to *fd*, looping over the short writes POSIX lets ``os.write`` make.
|
||||
|
||||
A caller that stopped after one short write would leave a truncated marker behind. A peer that reads between two
|
||||
writes sees a prefix of the record; each caller has to tolerate that. Lock records need not survive a power loss, so
|
||||
we skip ``fsync``.
|
||||
|
||||
:param fd: file descriptor open for writing.
|
||||
:param data: bytes to write in full.
|
||||
|
||||
:raises OSError: if a write reports zero progress before the record is complete.
|
||||
|
||||
"""
|
||||
remaining = memoryview(data)
|
||||
while remaining:
|
||||
if (written := os.write(fd, remaining)) == 0:
|
||||
raise OSError(EIO, "os.write wrote 0 bytes before the record was complete")
|
||||
remaining = remaining[written:]
|
||||
|
||||
|
||||
def raise_on_not_writable_file(filename: str) -> None:
|
||||
"""
|
||||
Raise an exception if attempting to open the file for writing would fail.
|
||||
|
||||
Separates files that can never be written from files that are writable but currently locked.
|
||||
|
||||
:param filename: file to check
|
||||
|
||||
:raises OSError: as if the file was opened for writing.
|
||||
|
||||
"""
|
||||
try:
|
||||
# lstat, not stat: settles exists-and-writable in one syscall, and a hostile symlink at the lock path would
|
||||
# make stat inspect the link target, letting an attacker turn a contended acquire into a misleading
|
||||
# PermissionError / IsADirectoryError and probe that target's attributes. The real open passes O_NOFOLLOW and
|
||||
# refuses the symlink anyway.
|
||||
file_stat = os.lstat(filename)
|
||||
except OSError:
|
||||
return # does not exist, or an error the caller cannot act on
|
||||
|
||||
if stat.S_ISDIR(file_stat.st_mode):
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
raise PermissionError(EACCES, "Permission denied", filename)
|
||||
raise IsADirectoryError(EISDIR, "Is a directory", filename) # pragma: win32 no cover
|
||||
|
||||
# No mtime guard: the old `if st_mtime != 0` skip covered NFS/Linux quirks where os.lstat returned an all-zero
|
||||
# struct, which it no longer does. Skipping on mtime 0 let a read-only file or a directory at the lock path pass
|
||||
# as missing, so acquire() blocked forever on an open that cannot succeed.
|
||||
# Match open() credentials where supported; group permissions and ACLs can grant access without S_IWUSR. Do not
|
||||
# follow a symlink, as the lstat above does not: one swapped in since would be judged by its target.
|
||||
if not (file_stat.st_mode & stat.S_IWUSR) and not os.access(
|
||||
filename,
|
||||
os.W_OK,
|
||||
effective_ids=os.access in os.supports_effective_ids,
|
||||
follow_symlinks=os.access not in os.supports_follow_symlinks,
|
||||
):
|
||||
try:
|
||||
os.lstat(filename)
|
||||
except FileNotFoundError:
|
||||
# A holder may unlink its marker between lstat() and access(); let the caller attempt creation.
|
||||
return
|
||||
raise PermissionError(EACCES, "Permission denied", filename)
|
||||
|
||||
|
||||
def ensure_directory_exists(filename: Path | str) -> None:
|
||||
"""
|
||||
Ensure the directory containing the file exists (create it if necessary).
|
||||
|
||||
:param filename: file.
|
||||
|
||||
"""
|
||||
Path(filename).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def pin_inode(lock_file: str) -> Generator[int | None]:
|
||||
"""
|
||||
Keep the file at *lock_file* allocated for the length of the block and yield its inode number.
|
||||
|
||||
An inode is the kernel's identity for a file, independent of its name, and its number names a file only while the
|
||||
inode stays allocated. ext4 hands a freed number to the next file created. Once a stale marker is unlinked, the next
|
||||
holder's marker can therefore carry the number a contender read a moment ago. A caller that reads the marker inside
|
||||
the block, gives up unless it read the yielded inode, and breaks the marker before leaving can rely on the number.
|
||||
|
||||
:param lock_file: path to the lock file about to be read and perhaps broken.
|
||||
|
||||
:returns: the pinned inode number, or ``None`` where nothing can be pinned. Windows cannot rename a file this
|
||||
process holds open, so it needs no pin. Without ``O_PATH`` a symlink or a socket cannot be opened.
|
||||
|
||||
:raises FileNotFoundError: if the path is gone.
|
||||
|
||||
"""
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
yield None
|
||||
else: # pragma: win32 no cover
|
||||
# O_PATH pins any node, a symlink or socket included, without read permission and without the filesystem seeing
|
||||
# an open; elsewhere a nonblocking read-only open pins what it can.
|
||||
flags: Final = getattr(os, "O_PATH", os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)) | getattr(os, "O_NOFOLLOW", 0)
|
||||
try:
|
||||
fd = os.open(lock_file, flags)
|
||||
except FileNotFoundError:
|
||||
raise
|
||||
except OSError:
|
||||
yield None
|
||||
return
|
||||
try:
|
||||
yield os.fstat(fd).st_ino
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def remove_own_file(lock_file: str, fd: int) -> None: # pragma: win32 no cover
|
||||
"""
|
||||
Remove *lock_file* while it still names the file open as *fd*; the caller closes the descriptor afterwards.
|
||||
|
||||
Where the platform has an atomic exchange, a stale break takes the file through it, so a peer's marker found at the
|
||||
path goes back. Otherwise the path is unlinked by name if it still names this file. The open descriptor keeps the
|
||||
inode allocated, so a matching number cannot belong to a successor's marker.
|
||||
|
||||
:raises OSError: if the descriptor cannot be read or the unlink fails.
|
||||
|
||||
"""
|
||||
own: Final = os.fstat(fd)
|
||||
with suppress(FileNotFoundError):
|
||||
break_lock_file(lock_file, own.st_mtime, own.st_ino)
|
||||
if os.path.samestat(os.lstat(lock_file), own):
|
||||
Path(lock_file).unlink()
|
||||
|
||||
|
||||
def foreign_directory_owner(directory: os.stat_result, parent: os.stat_result) -> int | None:
|
||||
"""
|
||||
Return the uid that owns *directory* when another account controls it under a sticky parent.
|
||||
|
||||
A lock that keeps its ownership records in a directory relies on them staying in place. The sticky bit (a directory
|
||||
flag that lets only a file's owner remove it) on a parent such as ``/tmp`` stops other accounts from renaming the
|
||||
directory away. A directory that this account or the parent's owner owns is therefore safe there. One that another
|
||||
account owns is outside the lock's control. Without the sticky bit, any process that can create the directory can
|
||||
also rename this account's own one away, so a refusal there would change nothing. Linux applies the same rule for
|
||||
``fs.protected_regular``.
|
||||
|
||||
:param directory: ``lstat`` of the directory found in place.
|
||||
:param parent: ``stat`` of the directory that holds it.
|
||||
|
||||
:returns: the owning uid when another account controls the directory, otherwise ``None``.
|
||||
|
||||
"""
|
||||
if sys.platform != "win32" and parent.st_mode & stat.S_ISVTX: # pragma: needs foreign-sticky-tmp
|
||||
return None if directory.st_uid in {os.geteuid(), parent.st_uid} else directory.st_uid
|
||||
return None
|
||||
|
||||
|
||||
def break_lock_file(lock_file: str, mtime_before: float, ino_before: int) -> None:
|
||||
"""
|
||||
Remove the lock file a caller found stale with modification time *mtime_before* and inode *ino_before*.
|
||||
|
||||
The caller keeps the judged file open until we return, through :func:`pin_inode` or its own descriptor, so no
|
||||
other file can carry *ino_before* meanwhile. Right before breaking we ``lstat`` the lock again and leave it alone
|
||||
if a peer recreated it since the caller judged it stale, as Mercurial re-reads a lock under its break lock.
|
||||
|
||||
On Linux (``renameat2``) and macOS (``renamex_np``) the platform can exchange two names in one step. There we swap
|
||||
the marker with a placeholder of our own, so the lock path never stands empty, and a contender that polls meanwhile
|
||||
finds a file it cannot parse and waits. If the file we took is not the one judged stale, a peer recreated the lock,
|
||||
and a second exchange puts it back. Otherwise we remove it and then our placeholder. Of several processes racing to
|
||||
break the same lock, only one takes the stale file.
|
||||
|
||||
Windows cannot exchange names, but it refuses to rename a marker its live holder keeps open. There we rename the
|
||||
marker aside and, if it was recreated after all, link it back. ``link`` never replaces a name, so a marker a third
|
||||
process created meanwhile keeps the path. Elsewhere without an exchange (NFS, SMB and other filesystems that refuse
|
||||
it, platforms without the call) a rename would leave the path empty beside a possibly live holder. We then do not
|
||||
move the marker, and the caller waits for ``timeout`` or ``break_lock()``.
|
||||
|
||||
We compare inodes as well as modification times, because NFS and FAT store coarse modification times. The times are
|
||||
compared for equality, since a file's owner can set any time. We ``lstat`` so a symlink a peer swaps in is never
|
||||
followed. The break name carries a random token so no other process can guess it.
|
||||
|
||||
:param lock_file: path to the lock file to break.
|
||||
:param mtime_before: modification time the caller saw when it judged the lock stale.
|
||||
:param ino_before: inode number the caller saw when it judged the lock stale.
|
||||
|
||||
:raises OSError: if taking the marker fails, for example because the file vanished, another user owns it in a
|
||||
sticky directory, or on Windows its holder keeps it open.
|
||||
|
||||
"""
|
||||
if (current := os.lstat(lock_file)).st_mtime != mtime_before or current.st_ino != ino_before:
|
||||
return
|
||||
break_path: Final[str] = f"{lock_file}.break.{os.getpid()}.{secrets.token_hex(16)}"
|
||||
if exchange_supported(): # pragma: win32 no cover
|
||||
_swap_out(lock_file, break_path, mtime_before, ino_before)
|
||||
elif _MOVES_WITHOUT_EXCHANGE: # pragma: win32 cover
|
||||
_break_by_rename(lock_file, break_path, mtime_before, ino_before)
|
||||
|
||||
|
||||
def _swap_out(lock_file: str, break_path: str, mtime_before: float, ino_before: int) -> None: # pragma: win32 no cover
|
||||
# An atomic exchange with a placeholder takes the marker without the lock path ever being empty. No contender can
|
||||
# then create a marker while this one is judged, and a live marker goes back the same way.
|
||||
fd: Final = os.open(break_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600)
|
||||
try:
|
||||
try:
|
||||
# Not a record any lock writes, so a contender that reads it waits, and one left by a crash ages out. The
|
||||
# PID tells the contender that evicts it whose break names to search for a marker this process left.
|
||||
write_all(fd, f"filelock/break\n{os.getpid()}\n".encode())
|
||||
exchanged = exchange_names(break_path, lock_file)
|
||||
except OSError:
|
||||
exchanged = False
|
||||
if not exchanged:
|
||||
Path(break_path).unlink()
|
||||
return
|
||||
if (grabbed := os.lstat(break_path)).st_mtime == mtime_before and grabbed.st_ino == ino_before:
|
||||
Path(break_path).unlink()
|
||||
# Our descriptor keeps the placeholder's inode allocated, so a matching number at the path is still ours.
|
||||
if os.path.samestat(os.lstat(lock_file), os.fstat(fd)):
|
||||
Path(lock_file).unlink()
|
||||
else:
|
||||
exchange_names(break_path, lock_file)
|
||||
Path(break_path).unlink()
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _break_by_rename(
|
||||
lock_file: str, break_path: str, mtime_before: float, ino_before: int
|
||||
) -> None: # pragma: win32 cover
|
||||
# Renaming the marker aside empties the path until a recreated live marker is put back. link never replaces a name,
|
||||
# so a marker a third process created meanwhile keeps the path, and the live one waits under the break name.
|
||||
# Without os.link, or where the filesystem refuses hard links, the live one waits there too.
|
||||
with _pinned_for_rename(lock_file):
|
||||
Path(lock_file).rename(break_path)
|
||||
if (st_after := os.lstat(break_path)).st_mtime == mtime_before and st_after.st_ino == ino_before:
|
||||
unlink_path(break_path)
|
||||
elif stat.S_ISREG(st_after.st_mode) and hasattr(os, "link"): # pragma: needs hard-link
|
||||
with suppress(OSError):
|
||||
os.link(break_path, lock_file, follow_symlinks=False)
|
||||
unlink_path(break_path)
|
||||
|
||||
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
|
||||
def _pinned_for_rename(path: str) -> AbstractContextManager[object]:
|
||||
# A rename by path waits on a Level 1 or batch oplock holder until it answers or closes. Opening first with
|
||||
# FILE_COMPLETE_IF_OPLOCKED turns that wait into BlockingIOError, and the open handle shares delete, so the
|
||||
# rename then proceeds. Imported here because _windows imports this module.
|
||||
from filelock._windows import open_for_delete_without_oplock_wait # ruff:ignore[import-outside-top-level]
|
||||
|
||||
return os.fdopen(open_for_delete_without_oplock_wait(path), "rb")
|
||||
|
||||
def unlink_path(path: str) -> None:
|
||||
from filelock._windows import unlink_without_oplock_wait # ruff:ignore[import-outside-top-level]
|
||||
|
||||
unlink_without_oplock_wait(path)
|
||||
|
||||
else: # pragma: win32 no cover
|
||||
|
||||
def unlink_path(path: str) -> None:
|
||||
Path(path).unlink()
|
||||
|
||||
|
||||
def exchange_supported() -> bool:
|
||||
"""Whether this platform has a call that swaps two names in one step; a filesystem may still refuse it."""
|
||||
return _rename_call() is not None
|
||||
|
||||
|
||||
def exchange_names(source: str, target: str) -> bool: # pragma: win32 no cover
|
||||
"""
|
||||
Swap the names *source* and *target* in one step, so neither ever stands empty, where :func:`exchange_supported`.
|
||||
|
||||
:returns: ``False`` where the filesystem cannot, so the caller falls back
|
||||
|
||||
:raises OSError: for any other failure, such as a missing name or a denied rename
|
||||
|
||||
"""
|
||||
renamer, exchange_flag = cast("tuple[Callable[[bytes, bytes, int], int], int]", _rename_call())
|
||||
if not (error := renamer(os.fsencode(source), os.fsencode(target), exchange_flag)):
|
||||
return True
|
||||
# EXDEV: overlayfs and some FUSE filesystems refuse an exchange between layers.
|
||||
if error in {EINVAL, ENOSYS, EXDEV, getattr(errno, "ENOTSUP", EINVAL), getattr(errno, "EOPNOTSUPP", EINVAL)}:
|
||||
return False
|
||||
raise OSError(error, os.strerror(error), source, None, target)
|
||||
|
||||
|
||||
@cache
|
||||
def _rename_call() -> tuple[Callable[[bytes, bytes, int], int], int] | None:
|
||||
# The call returns 0 or the errno. Looked up on first use, since most processes never break a lock and ctypes is
|
||||
# slow to import. None where ctypes is missing (GraalPy) or the platform has no exchange.
|
||||
try:
|
||||
import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
|
||||
except ImportError: # pragma: no cover # GraalPy ships without ctypes
|
||||
return None
|
||||
if sys.platform in {"linux", "android"}: # pragma: linux cover
|
||||
if (renameat2 := _renameat2(ctypes.CDLL(None, use_errno=True))) is None:
|
||||
return None # pragma: no cover # an architecture the CI matrix does not run
|
||||
return renameat2, 2 # RENAME_EXCHANGE in linux/fs.h
|
||||
if sys.platform == "darwin": # pragma: darwin cover
|
||||
renamex_np: Final = ctypes.CDLL(None, use_errno=True).renamex_np
|
||||
renamex_np.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint]
|
||||
renamex_np.restype = ctypes.c_int
|
||||
|
||||
def call_renamex_np(source: bytes, target: bytes, flags: int) -> int:
|
||||
return 0 if renamex_np(source, target, flags) == 0 else ctypes.get_errno()
|
||||
|
||||
return call_renamex_np, 0x2 # RENAME_SWAP in sys/stdio.h
|
||||
return None # pragma: linux no cover # pragma: android no cover # pragma: darwin no cover
|
||||
|
||||
|
||||
def _renameat2(libc: ctypes.CDLL) -> Callable[[bytes, bytes, int], int] | None: # pragma: linux cover
|
||||
# Reached through syscall(2) rather than the C library's wrapper. glibc 2.28, musl 1.2.2 and bionic API 30 added
|
||||
# the wrapper, and Termux ships an older bionic; the kernel has had the call since 3.15. The number is the
|
||||
# architecture's own, so an architecture outside the table gets no exchange rather than another call.
|
||||
import ctypes # ruff:ignore[import-outside-top-level] # only a stale break needs it
|
||||
|
||||
numbers: Final = {
|
||||
"x86_64": 316,
|
||||
"i386": 353,
|
||||
"i686": 353,
|
||||
"armv7l": 382,
|
||||
"armv8l": 382,
|
||||
"aarch64": 276,
|
||||
"riscv64": 276,
|
||||
"loongarch64": 276,
|
||||
"ppc64le": 357,
|
||||
"s390x": 347,
|
||||
}
|
||||
if (number := numbers.get(platform.machine())) is None:
|
||||
return None # pragma: no cover # an architecture the CI matrix does not run
|
||||
syscall: Final = libc.syscall
|
||||
syscall.restype = ctypes.c_long
|
||||
at_fdcwd: Final = -100
|
||||
|
||||
def call_renameat2(source: bytes, target: bytes, flags: int) -> int:
|
||||
arguments: Final = (
|
||||
ctypes.c_long(number),
|
||||
ctypes.c_int(at_fdcwd),
|
||||
ctypes.c_char_p(source),
|
||||
ctypes.c_int(at_fdcwd),
|
||||
ctypes.c_char_p(target),
|
||||
ctypes.c_uint(flags),
|
||||
)
|
||||
return 0 if syscall(*arguments) == 0 else ctypes.get_errno()
|
||||
|
||||
return call_renameat2
|
||||
|
||||
|
||||
def touch(name: str, *, fd: int) -> None:
|
||||
# Prefer the already-open, already-verified fd so a peer that swaps a symlink or a different file in at the
|
||||
# path after our O_NOFOLLOW read cannot redirect the touch: utime then targets the inode behind the fd.
|
||||
# Where the platform cannot utime an fd, fall back to a path-based touch that still refuses to follow a
|
||||
# symlink where supported, matching the O_NOFOLLOW reads used elsewhere here.
|
||||
if _SUPPORTS_UTIME_FD: # pragma: needs utime-fd
|
||||
os.utime(fd, None)
|
||||
return
|
||||
os.utime(name, None, follow_symlinks=not _SUPPORTS_UTIME_NOFOLLOW) # pragma: lacks utime-fd
|
||||
|
||||
|
||||
# Windows refuses to rename a marker its live holder keeps open, so renaming aside only moves a marker nobody holds.
|
||||
# Elsewhere without an exchange a contender could take the emptied path beside a live holder, so the marker stays and
|
||||
# the lock counts as held.
|
||||
_MOVES_WITHOUT_EXCHANGE: Final[bool] = sys.platform == "win32"
|
||||
# Retargeting os.utime to an open fd lets a heartbeat refresh the exact inode it verified instead of whatever the
|
||||
# pathname now names.
|
||||
_SUPPORTS_UTIME_FD: Final[bool] = sys.platform != "win32" and os.utime in os.supports_fd
|
||||
# os.utime follows symlinks unless told not to; not every platform can refuse the follow, so probe support.
|
||||
_SUPPORTS_UTIME_NOFOLLOW: Final[bool] = os.utime in os.supports_follow_symlinks
|
||||
|
||||
|
||||
__all__ = [
|
||||
"break_lock_file",
|
||||
"ensure_directory_exists",
|
||||
"exchange_names",
|
||||
"exchange_supported",
|
||||
"foreign_directory_owner",
|
||||
"pin_inode",
|
||||
"raise_on_not_writable_file",
|
||||
"remove_own_file",
|
||||
"touch",
|
||||
"unlink_path",
|
||||
"write_all",
|
||||
]
|
||||
@@ -0,0 +1,780 @@
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = ["filelock._util", "msvcrt", "struct"]
|
||||
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager, suppress
|
||||
from errno import ENOENT, ENOTDIR, EWOULDBLOCK
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
from filelock._api import BaseFileLock
|
||||
from filelock._util import raise_on_not_writable_file
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
if sys.platform == "win32": # pragma: win32 cover
|
||||
import ctypes
|
||||
import msvcrt
|
||||
from ctypes import wintypes
|
||||
|
||||
_GENERIC_READ: Final[int] = 0x80000000
|
||||
_GENERIC_WRITE: Final[int] = 0x40000000
|
||||
_SYNCHRONIZE: Final[int] = 0x00100000
|
||||
_DELETE: Final[int] = 0x00010000
|
||||
_DESIRED_ACCESS: Final[int] = _GENERIC_READ | _GENERIC_WRITE | _SYNCHRONIZE
|
||||
_FILE_SHARE_READ_WRITE: Final[int] = (
|
||||
0x00000001 | 0x00000002
|
||||
) # read | write; matches os.open (_SH_DENYNO), no delete
|
||||
_FILE_OPEN: Final[int] = 1 # open the file only if it exists; the NtCreateFile OPEN_EXISTING
|
||||
_FILE_OPEN_IF: Final[int] = 3 # open the file if it exists, create it otherwise; the NtCreateFile OPEN_ALWAYS
|
||||
# Attribute-only access skips share-mode checks, so the probe sees a name's deletion state rather than a reader.
|
||||
_FILE_READ_ATTRIBUTES: Final[int] = 0x00000080
|
||||
_FILE_SHARE_ALL: Final[int] = 0x00000001 | 0x00000002 | 0x00000004 # read | write | delete
|
||||
_FILE_ATTRIBUTE_READONLY: Final[int] = 0x00000001
|
||||
_FILE_ATTRIBUTE_NORMAL: Final[int] = 0x00000080
|
||||
_FILE_ATTRIBUTE_REPARSE_POINT: Final[int] = 0x00000400
|
||||
# CreateOptions: keep the handle synchronous (the CRT and msvcrt.locking rely on a maintained file position),
|
||||
# refuse a directory, return at once instead of waiting for an oplock holder to answer the break, and open a
|
||||
# reparse point rather than following it so the check below acts on the link itself.
|
||||
_FILE_SYNCHRONOUS_IO_NONALERT: Final[int] = 0x00000020
|
||||
_FILE_NON_DIRECTORY_FILE: Final[int] = 0x00000040
|
||||
_FILE_DELETE_ON_CLOSE: Final[int] = 0x00001000
|
||||
_FILE_COMPLETE_IF_OPLOCKED: Final[int] = 0x00000100
|
||||
_FILE_OPEN_REPARSE_POINT: Final[int] = 0x00200000
|
||||
_CREATE_OPTIONS: Final[int] = (
|
||||
_FILE_SYNCHRONOUS_IO_NONALERT | _FILE_NON_DIRECTORY_FILE | _FILE_COMPLETE_IF_OPLOCKED | _FILE_OPEN_REPARSE_POINT
|
||||
)
|
||||
# Listing access takes part in share-mode checks, unlike attribute-only access, so a handle that holds it without
|
||||
# FILE_SHARE_DELETE blocks renames and removals of the directory until it closes.
|
||||
_FILE_LIST_DIRECTORY: Final[int] = 0x00000001
|
||||
_FILE_DIRECTORY_FILE: Final[int] = 0x00000001
|
||||
_DIRECTORY_OPTIONS: Final[int] = _FILE_SYNCHRONOUS_IO_NONALERT | _FILE_DIRECTORY_FILE | _FILE_OPEN_REPARSE_POINT
|
||||
_OBJ_CASE_INSENSITIVE: Final[int] = 0x00000040 # Win32 name lookups are case-insensitive
|
||||
_OWNER_WRITE: Final[int] = 0o200
|
||||
_GROUP_AND_OTHERS: Final[int] = 0o077
|
||||
# The DACL os.mkdir(mode=0o700) applies, less the inheritance flags a file has no use for: protected, so the
|
||||
# directory's inheritable entries stay out, with full access for SYSTEM, Administrators and the owner.
|
||||
_OWNER_ONLY_SDDL: Final[str] = "D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;OW)"
|
||||
_SDDL_REVISION_1: Final[int] = 1
|
||||
_INVALID_HANDLE_VALUE: Final[int | None] = ctypes.c_void_p(-1).value
|
||||
|
||||
# LockFileEx locks a byte range at an offset carried in OVERLAPPED, independent of the descriptor's file position.
|
||||
# msvcrt.locking starts at the current position instead, so a metadata write between lock and unlock could shift
|
||||
# the byte a later unlock targets; the explicit offset removes that hazard for both the path lock and #608's
|
||||
# descriptor lock.
|
||||
_LOCKFILE_FAIL_IMMEDIATELY: Final[int] = 0x00000001
|
||||
_LOCKFILE_EXCLUSIVE_LOCK: Final[int] = 0x00000002
|
||||
_ERROR_ACCESS_DENIED: Final[int] = 5
|
||||
_ERROR_LOCK_VIOLATION: Final[int] = 33 # another handle holds the byte range
|
||||
_ERROR_NOT_LOCKED: Final[int] = 158 # the byte range is not locked
|
||||
|
||||
# NtCreateFile returns the raw NTSTATUS as its value, where CreateFileW collapses several of these into one
|
||||
# ERROR_ACCESS_DENIED. Telling them apart is the point (#604): a name pending deletion or a share conflict is
|
||||
# transient and worth a retry, a real access denial is not.
|
||||
_STATUS_SUCCESS: Final[int] = 0x00000000
|
||||
# A success code that comes with a handle: another handle's oplock is breaking and its holder has not answered yet.
|
||||
_STATUS_OPLOCK_BREAK_IN_PROGRESS: Final[int] = 0x00000108
|
||||
_STATUS_ACCESS_DENIED: Final[int] = 0xC0000022
|
||||
_STATUS_OBJECT_NAME_NOT_FOUND: Final[int] = 0xC0000034
|
||||
_STATUS_OBJECT_PATH_NOT_FOUND: Final[int] = 0xC000003A
|
||||
_STATUS_SHARING_VIOLATION: Final[int] = 0xC0000043
|
||||
_STATUS_DELETE_PENDING: Final[int] = 0xC0000056
|
||||
#: Probes, 2 ms apart, for transient STATUS_ACCESS_DENIED while a peer unlinks the name (#604). A count ensures
|
||||
#: every probe runs even when the process stalls; oplock breaks use the caller's wait instead.
|
||||
_ACCESS_DENIED_PROBES: Final[int] = 250
|
||||
_ACCESS_DENIED_GRACE: Final[float] = 0.5
|
||||
_ACCESS_DENIED_RETRY: Final[float] = 0.002
|
||||
|
||||
_ntdll: Final[ctypes.WinDLL] = ctypes.WinDLL("ntdll")
|
||||
_kernel32: Final[ctypes.WinDLL] = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
_advapi32: Final[ctypes.WinDLL] = ctypes.WinDLL("advapi32", use_last_error=True)
|
||||
|
||||
class _UNICODE_STRING(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
|
||||
_fields_ = (
|
||||
("Length", wintypes.USHORT), # byte length, not character count
|
||||
("MaximumLength", wintypes.USHORT),
|
||||
("Buffer", wintypes.LPWSTR),
|
||||
)
|
||||
|
||||
class _OBJECT_ATTRIBUTES(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
|
||||
_fields_ = (
|
||||
("Length", wintypes.ULONG),
|
||||
("RootDirectory", wintypes.HANDLE),
|
||||
("ObjectName", ctypes.POINTER(_UNICODE_STRING)),
|
||||
("Attributes", wintypes.ULONG),
|
||||
("SecurityDescriptor", ctypes.c_void_p),
|
||||
("SecurityQualityOfService", ctypes.c_void_p),
|
||||
)
|
||||
|
||||
class _IO_STATUS_BLOCK(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
|
||||
_fields_ = (
|
||||
("Status", ctypes.c_void_p), # a union of NTSTATUS and PVOID, so it is pointer-sized
|
||||
("Information", ctypes.c_void_p),
|
||||
)
|
||||
|
||||
class _SECURITY_ATTRIBUTES(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
|
||||
_fields_ = (
|
||||
("nLength", wintypes.DWORD),
|
||||
("lpSecurityDescriptor", ctypes.c_void_p),
|
||||
("bInheritHandle", wintypes.BOOL),
|
||||
)
|
||||
|
||||
class _OVERLAPPED(ctypes.Structure): # mirrors the Win32 struct name
|
||||
_fields_ = (
|
||||
("Internal", ctypes.c_void_p), # ULONG_PTR: pointer-sized, not DWORD, or the x64 layout corrupts Offset
|
||||
("InternalHigh", ctypes.c_void_p),
|
||||
("Offset", wintypes.DWORD), # the DUMMYUNIONNAME struct, flattened: low 32 bits of the byte offset
|
||||
("OffsetHigh", wintypes.DWORD),
|
||||
("hEvent", wintypes.HANDLE),
|
||||
)
|
||||
|
||||
class _BY_HANDLE_FILE_INFORMATION(ctypes.Structure): # ruff:ignore[invalid-class-name] # mirrors the Win32 struct name
|
||||
_fields_ = (
|
||||
("dwFileAttributes", wintypes.DWORD),
|
||||
("ftCreationTime", wintypes.FILETIME),
|
||||
("ftLastAccessTime", wintypes.FILETIME),
|
||||
("ftLastWriteTime", wintypes.FILETIME),
|
||||
("dwVolumeSerialNumber", wintypes.DWORD),
|
||||
("nFileSizeHigh", wintypes.DWORD),
|
||||
("nFileSizeLow", wintypes.DWORD),
|
||||
("nNumberOfLinks", wintypes.DWORD),
|
||||
("nFileIndexHigh", wintypes.DWORD),
|
||||
("nFileIndexLow", wintypes.DWORD),
|
||||
)
|
||||
|
||||
_ntdll.NtCreateFile.restype = wintypes.LONG # NTSTATUS
|
||||
_ntdll.NtCreateFile.argtypes = [
|
||||
ctypes.POINTER(wintypes.HANDLE),
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(_OBJECT_ATTRIBUTES),
|
||||
ctypes.POINTER(_IO_STATUS_BLOCK),
|
||||
ctypes.POINTER(ctypes.c_longlong), # PLARGE_INTEGER AllocationSize
|
||||
wintypes.ULONG,
|
||||
wintypes.ULONG,
|
||||
wintypes.ULONG,
|
||||
wintypes.ULONG,
|
||||
ctypes.c_void_p,
|
||||
wintypes.ULONG,
|
||||
]
|
||||
_ntdll.RtlDosPathNameToNtPathName_U_WithStatus.restype = wintypes.LONG # NTSTATUS
|
||||
_ntdll.RtlDosPathNameToNtPathName_U_WithStatus.argtypes = [
|
||||
wintypes.LPCWSTR,
|
||||
ctypes.POINTER(_UNICODE_STRING),
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_void_p,
|
||||
]
|
||||
_ntdll.RtlFreeUnicodeString.restype = None
|
||||
_ntdll.RtlFreeUnicodeString.argtypes = [ctypes.POINTER(_UNICODE_STRING)]
|
||||
_ntdll.RtlNtStatusToDosError.restype = wintypes.ULONG
|
||||
_ntdll.RtlNtStatusToDosError.argtypes = [wintypes.LONG]
|
||||
|
||||
_kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
_kernel32.CloseHandle.restype = wintypes.BOOL
|
||||
_kernel32.GetFileInformationByHandle.argtypes = [wintypes.HANDLE, ctypes.POINTER(_BY_HANDLE_FILE_INFORMATION)]
|
||||
_kernel32.GetFileInformationByHandle.restype = wintypes.BOOL
|
||||
_kernel32.LockFileEx.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(_OVERLAPPED),
|
||||
]
|
||||
_kernel32.LockFileEx.restype = wintypes.BOOL
|
||||
_kernel32.UnlockFileEx.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(_OVERLAPPED),
|
||||
]
|
||||
_kernel32.UnlockFileEx.restype = wintypes.BOOL
|
||||
_advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.argtypes = [
|
||||
wintypes.LPCWSTR,
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(ctypes.c_void_p),
|
||||
ctypes.POINTER(wintypes.ULONG),
|
||||
]
|
||||
_advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW.restype = wintypes.BOOL
|
||||
_advapi32.GetNamedSecurityInfoW.argtypes = [
|
||||
wintypes.LPCWSTR,
|
||||
ctypes.c_int,
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(ctypes.c_void_p),
|
||||
ctypes.POINTER(ctypes.c_void_p),
|
||||
ctypes.POINTER(ctypes.c_void_p),
|
||||
ctypes.POINTER(ctypes.c_void_p),
|
||||
ctypes.POINTER(ctypes.c_void_p),
|
||||
]
|
||||
_advapi32.GetNamedSecurityInfoW.restype = wintypes.DWORD
|
||||
_advapi32.OpenThreadToken.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.DWORD,
|
||||
wintypes.BOOL,
|
||||
ctypes.POINTER(wintypes.HANDLE),
|
||||
]
|
||||
_advapi32.OpenThreadToken.restype = wintypes.BOOL
|
||||
_advapi32.OpenProcessToken.argtypes = [wintypes.HANDLE, wintypes.DWORD, ctypes.POINTER(wintypes.HANDLE)]
|
||||
_advapi32.OpenProcessToken.restype = wintypes.BOOL
|
||||
_advapi32.GetTokenInformation.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
ctypes.c_int,
|
||||
ctypes.c_void_p,
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(wintypes.DWORD),
|
||||
]
|
||||
_advapi32.GetTokenInformation.restype = wintypes.BOOL
|
||||
_kernel32.GetCurrentProcess.restype = wintypes.HANDLE
|
||||
_kernel32.GetCurrentThread.restype = wintypes.HANDLE
|
||||
_kernel32.LocalFree.argtypes = [ctypes.c_void_p]
|
||||
_kernel32.LocalFree.restype = ctypes.c_void_p
|
||||
_kernel32.CreateDirectoryW.argtypes = [wintypes.LPCWSTR, ctypes.POINTER(_SECURITY_ATTRIBUTES)]
|
||||
_kernel32.CreateDirectoryW.restype = wintypes.BOOL
|
||||
_kernel32.CreateFileW.argtypes = [
|
||||
wintypes.LPCWSTR,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
ctypes.c_void_p,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
wintypes.HANDLE,
|
||||
]
|
||||
_kernel32.CreateFileW.restype = ctypes.c_void_p
|
||||
|
||||
def _lock_fd_nonblocking(fd: int) -> bool:
|
||||
# One nonblocking exclusive LockFileEx attempt shared by WindowsFileLock and lock_descriptor, over the one-byte
|
||||
# range at offset 0. True on acquisition, False on contention, raise otherwise. The caller owns fd; the handle
|
||||
# from get_osfhandle belongs to the CRT descriptor and must not be closed here.
|
||||
overlapped = _OVERLAPPED() # zero-initialized, so Offset/OffsetHigh/hEvent are 0
|
||||
flags = _LOCKFILE_EXCLUSIVE_LOCK | _LOCKFILE_FAIL_IMMEDIATELY
|
||||
if _kernel32.LockFileEx(msvcrt.get_osfhandle(fd), flags, 0, 1, 0, ctypes.byref(overlapped)):
|
||||
return True
|
||||
err = ctypes.get_last_error()
|
||||
if err == _ERROR_LOCK_VIOLATION:
|
||||
return False
|
||||
# A non-contention LockFileEx failure is not reproducible in-process.
|
||||
raise ctypes.WinError(err) # pragma: no cover
|
||||
|
||||
def _unlock_fd(fd: int) -> None:
|
||||
overlapped = _OVERLAPPED() # the same offset 0 and one-byte length the lock used
|
||||
if ( # pragma: no cover # only an interrupted earlier release leaves our own range unlocked
|
||||
not _kernel32.UnlockFileEx(msvcrt.get_osfhandle(fd), 0, 1, 0, ctypes.byref(overlapped))
|
||||
and (err := ctypes.get_last_error()) != _ERROR_NOT_LOCKED
|
||||
):
|
||||
raise ctypes.WinError(err)
|
||||
|
||||
class WindowsFileLock(BaseFileLock):
|
||||
"""
|
||||
Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems.
|
||||
|
||||
Lock file cleanup: Windows attempts to delete the lock file after release, but deletion is
|
||||
not guaranteed in multi-threaded scenarios where another thread holds an open handle. The lock
|
||||
file may persist on disk, which does not affect lock correctness.
|
||||
"""
|
||||
|
||||
def _acquire(self) -> None:
|
||||
raise_on_not_writable_file(self._lock_path)
|
||||
|
||||
# The reparse test is bound to the opened handle, so a symlink or junction swapped in cannot defeat it
|
||||
# through a check-then-open TOCTOU race.
|
||||
try:
|
||||
fd = _open_non_reparse_fd(self._lock_path, self._open_mode())
|
||||
except BlockingIOError:
|
||||
return # an oplock holder has not answered its break; let the retry loop keep its deadline
|
||||
if fd is None:
|
||||
return # open contention (share conflict or a name pending deletion); let the retry loop try again
|
||||
try:
|
||||
locked = _lock_fd_nonblocking(fd)
|
||||
if locked:
|
||||
self._mark_descriptor_owned(fd)
|
||||
except BaseException:
|
||||
# Closing drops the byte-range lock, so a hold already recorded for this descriptor must go with it.
|
||||
self._mark_descriptor_released()
|
||||
os.close(fd)
|
||||
raise
|
||||
if not locked:
|
||||
os.close(fd) # another holder owns the byte-range lock; let the retry loop try again
|
||||
|
||||
def _release(self) -> None:
|
||||
fd = cast("int", self._context.lock_file_fd)
|
||||
# Retain the descriptor until the OS unlock succeeds: if UnlockFileEx raises, the byte-range lock is still
|
||||
# held, so is_locked must keep reporting held rather than losing the fd. Only after the unlock commits do
|
||||
# close and unlink run as post-unlock cleanup; their failure cannot make the lock held again.
|
||||
_unlock_fd(fd)
|
||||
self._mark_descriptor_released()
|
||||
self._close_released_fd(fd, default_suppresses=False)
|
||||
if not self._preserve_lock_file: # preserve_lock_file keeps a stable file identity for the caller (#605)
|
||||
with suppress(OSError):
|
||||
unlink_without_oplock_wait(self._lock_path)
|
||||
|
||||
def _open_non_reparse_fd(path: str, mode: int, *, oplock_wait: float = _ACCESS_DENIED_GRACE) -> int | None:
|
||||
"""
|
||||
Open *path* for locking while refusing reparse points, bound to the handle actually locked.
|
||||
|
||||
The file is opened through ``NtCreateFile`` with ``FILE_OPEN_REPARSE_POINT`` so a symlink or junction planted
|
||||
at the path is not followed, and the reparse decision is read from *that* handle via
|
||||
``GetFileInformationByHandle`` rather than from a prior pathname query. Reading the held handle closes the
|
||||
check-then-open race: an attacker cannot swap the path between validation and use because both act on the same
|
||||
handle. Share mode omits delete so a peer cannot unlink or rename the file out from under a live holder,
|
||||
matching ``os.open``'s ``_SH_DENYNO``.
|
||||
|
||||
``NtCreateFile`` is used instead of ``CreateFileW`` because its return value carries the raw ``NTSTATUS``.
|
||||
Windows collapses a transient delete-pending name and a permanent access denial into the same Win32
|
||||
``ERROR_ACCESS_DENIED``; the status keeps them apart, so a real denial fails fast instead of spinning until the
|
||||
caller's timeout (#604).
|
||||
|
||||
The reparse option only guards the final path component; Windows still follows reparse points in intermediate
|
||||
directories. This assumes the lock file sits in a lock directory untrusted users cannot modify. A path with
|
||||
attacker-controlled parent directories would need component-by-component handle validation.
|
||||
|
||||
:param path: the lock file path.
|
||||
:param mode: the permission mode; as ``os.open`` does on Windows, a cleared owner-write bit creates the file
|
||||
read-only. A mode without group and other bits creates it with an owner-only DACL (discretionary access
|
||||
control list, the Windows permission list); any other mode has no Windows counterpart and leaves the file
|
||||
the entries its directory hands down. Both only take effect when the file is created, not when an existing
|
||||
one is opened.
|
||||
|
||||
:param oplock_wait: seconds to wait for the holder of an oplock on the file to answer its break.
|
||||
|
||||
:returns: a file descriptor owning the opened handle, or ``None`` on a sharing violation or a delete-pending
|
||||
name, which the caller should treat as contention and retry.
|
||||
|
||||
:raises BlockingIOError: if an oplock break stays unanswered for *oplock_wait* seconds.
|
||||
:raises OSError: if the path resolves to a reparse point, or the open fails for any other reason, raised with
|
||||
the Win32 error the status maps to. An access denial is raised only once it outlasts a run of probes, since
|
||||
a peer unlinking the file as it releases can make the open answer ``STATUS_ACCESS_DENIED`` for a moment.
|
||||
|
||||
"""
|
||||
# Emit the audit event os.open would, so consumers watching "open" still see the path-level open and can veto.
|
||||
sys.audit("open", path, None, os.O_RDWR | os.O_CREAT)
|
||||
read_only = not mode & _OWNER_WRITE
|
||||
owner_only = not mode & 0o077
|
||||
handle, status = _nt_open(path, read_only=read_only, owner_only=owner_only)
|
||||
denied_probes = 0
|
||||
oplock_deadline = time.monotonic() + oplock_wait
|
||||
while True:
|
||||
if status == _STATUS_ACCESS_DENIED:
|
||||
if denied_probes >= _ACCESS_DENIED_PROBES:
|
||||
break
|
||||
denied_probes += 1
|
||||
elif status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
|
||||
if time.monotonic() >= oplock_deadline:
|
||||
break
|
||||
else:
|
||||
break
|
||||
time.sleep(_ACCESS_DENIED_RETRY)
|
||||
handle, status = _nt_open(path, read_only=read_only, owner_only=owner_only)
|
||||
if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
|
||||
msg = "oplock break in progress"
|
||||
raise BlockingIOError(EWOULDBLOCK, msg, path)
|
||||
if status != _STATUS_SUCCESS:
|
||||
if status in {_STATUS_SHARING_VIOLATION, _STATUS_DELETE_PENDING}:
|
||||
return None
|
||||
raise _status_error(status, path)
|
||||
|
||||
return _descriptor_for_non_reparse(handle, path, os.O_RDWR)
|
||||
|
||||
def open_read_without_oplock_wait(path: str) -> int:
|
||||
"""
|
||||
Open *path* read-only and return a descriptor, without waiting for an oplock holder to answer its break.
|
||||
|
||||
``os.open`` waits inside the system call until the holder of an oplock on the file answers or closes its handle,
|
||||
so no timeout or ``blocking=False`` ends the wait. A reparse point at *path* is refused rather than followed.
|
||||
|
||||
:param path: the file to read.
|
||||
|
||||
:returns: a read-only file descriptor.
|
||||
|
||||
:raises BlockingIOError: if an oplock break is in progress, which callers treat as contention.
|
||||
:raises FileNotFoundError: if *path* does not exist.
|
||||
:raises OSError: if the open fails for any other reason, or *path* is a reparse point.
|
||||
|
||||
"""
|
||||
sys.audit("open", path, None, os.O_RDONLY)
|
||||
return _open_existing_without_oplock_wait(path, _GENERIC_READ | _SYNCHRONIZE, os.O_RDONLY)
|
||||
|
||||
def open_for_delete_without_oplock_wait(path: str) -> int:
|
||||
"""
|
||||
Open *path* with delete access and return a descriptor, without waiting for an oplock holder to answer.
|
||||
|
||||
The open handle shares read, write and delete, so a rename of *path* can proceed while the caller keeps it.
|
||||
A holder that took an oplock through another handle is asked to answer by this open, which then raises, so the
|
||||
caller skips the rename instead of waiting for it.
|
||||
|
||||
:param path: the file to rename.
|
||||
|
||||
:returns: a file descriptor owning the handle; the caller closes it.
|
||||
|
||||
:raises BlockingIOError: if an oplock break is in progress, which callers treat as contention.
|
||||
:raises FileNotFoundError: if *path* does not exist.
|
||||
:raises OSError: if the open fails for any other reason, or *path* is a reparse point.
|
||||
|
||||
"""
|
||||
return _open_existing_without_oplock_wait(path, _DELETE | _FILE_READ_ATTRIBUTES | _SYNCHRONIZE, os.O_RDONLY)
|
||||
|
||||
def unlink_without_oplock_wait(path: str) -> None:
|
||||
"""
|
||||
Delete *path* through a handle that does not wait for an oplock holder to answer its break.
|
||||
|
||||
``os.unlink`` waits like ``os.open`` does. Here an unanswered break leaves the file in place, which callers
|
||||
accept since deleting a lock file is best effort. A symlink at *path* is deleted as a link.
|
||||
|
||||
:param path: the file to delete.
|
||||
|
||||
:raises FileNotFoundError: if *path* does not exist.
|
||||
:raises OSError: if the open fails for any other reason, such as a sharing violation.
|
||||
|
||||
"""
|
||||
sys.audit("os.remove", path, None)
|
||||
handle, status = _nt_create(
|
||||
path,
|
||||
_DELETE | _SYNCHRONIZE,
|
||||
_FILE_ATTRIBUTE_NORMAL,
|
||||
_FILE_SHARE_ALL,
|
||||
_FILE_OPEN,
|
||||
options=_CREATE_OPTIONS | _FILE_DELETE_ON_CLOSE,
|
||||
)
|
||||
if status == _STATUS_SUCCESS:
|
||||
_kernel32.CloseHandle(handle) # deletes the file
|
||||
elif status != _STATUS_OPLOCK_BREAK_IN_PROGRESS:
|
||||
raise _status_error(status, path)
|
||||
|
||||
def _open_existing_without_oplock_wait(path: str, access: int, flags: int) -> int:
|
||||
handle, status = _nt_create(path, access, _FILE_ATTRIBUTE_NORMAL, _FILE_SHARE_ALL, _FILE_OPEN)
|
||||
if status != _STATUS_SUCCESS:
|
||||
raise _status_error(status, path)
|
||||
return _descriptor_for_non_reparse(handle, path, flags)
|
||||
|
||||
def _status_error(status: int, path: str) -> OSError:
|
||||
if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
|
||||
return BlockingIOError(EWOULDBLOCK, "oplock break in progress", path)
|
||||
if status in {_STATUS_OBJECT_NAME_NOT_FOUND, _STATUS_OBJECT_PATH_NOT_FOUND}:
|
||||
return FileNotFoundError(ENOENT, os.strerror(ENOENT), path)
|
||||
winerror = _ntdll.RtlNtStatusToDosError(status)
|
||||
return OSError(None, ctypes.FormatError(winerror).strip(), path, winerror)
|
||||
|
||||
def _descriptor_for_non_reparse(handle: int, path: str, flags: int) -> int:
|
||||
info = _BY_HANDLE_FILE_INFORMATION()
|
||||
# Querying an open handle we just created does not fail.
|
||||
if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover
|
||||
err = ctypes.get_last_error()
|
||||
_kernel32.CloseHandle(handle)
|
||||
raise ctypes.WinError(err)
|
||||
if info.dwFileAttributes & _FILE_ATTRIBUTE_REPARSE_POINT:
|
||||
_kernel32.CloseHandle(handle)
|
||||
msg = f"Lock file is a reparse point (symlink/junction): {path}"
|
||||
raise OSError(msg)
|
||||
|
||||
try:
|
||||
# O_NOINHERIT mirrors os.open on Windows: the lock fd must not leak into child processes.
|
||||
return msvcrt.open_osfhandle(handle, flags | os.O_NOINHERIT)
|
||||
except BaseException: # pragma: no cover # open_osfhandle audits too; a hook raising must not leak the handle
|
||||
_kernel32.CloseHandle(handle)
|
||||
raise
|
||||
|
||||
def _open_deletable_for_reading(path: str) -> int:
|
||||
"""
|
||||
Open *path* read-only without blocking a concurrent unlink and without following a reparse point.
|
||||
|
||||
``os.open`` omits ``FILE_SHARE_DELETE``, so while a process has a file open that way, a delete of it fails with
|
||||
a sharing violation. Every contender reads a strict claim and its owner unlinks it, so a reader that keeps the
|
||||
claim open past the owner's retries makes the release fail.
|
||||
"""
|
||||
sys.audit("open", path, None, os.O_RDONLY)
|
||||
handle, status = _nt_create(path, _GENERIC_READ | _SYNCHRONIZE, 0, _FILE_SHARE_ALL, _FILE_OPEN)
|
||||
if status != _STATUS_SUCCESS:
|
||||
winerror = _ntdll.RtlNtStatusToDosError(status)
|
||||
raise OSError(None, ctypes.FormatError(winerror).strip(), path, winerror)
|
||||
try:
|
||||
return msvcrt.open_osfhandle(handle, os.O_RDONLY | os.O_NOINHERIT)
|
||||
except BaseException: # pragma: no cover # open_osfhandle audits too; a hook raising must not leak the handle
|
||||
_kernel32.CloseHandle(handle)
|
||||
raise
|
||||
|
||||
def _pin_directory(path: str) -> int:
|
||||
"""
|
||||
Open the directory at *path* so that no process can rename or remove it until :func:`_unpin_directory`.
|
||||
|
||||
Windows has no ``dir_fd``, so an operation resolves every path component by name again. A handle on each
|
||||
directory of a record path, opened without following a reparse point, keeps that name bound to the directory
|
||||
checked here until the operation ends.
|
||||
|
||||
:raises NotADirectoryError: when *path* is a junction, a symlink or anything other than a directory.
|
||||
"""
|
||||
sys.audit("open", path, None, os.O_RDONLY)
|
||||
# GetFileInformationByHandle needs FILE_READ_ATTRIBUTES; listing access is what takes part in share checks.
|
||||
access = _FILE_LIST_DIRECTORY | _FILE_READ_ATTRIBUTES | _SYNCHRONIZE
|
||||
handle, status = _nt_create(path, access, 0, _FILE_SHARE_READ_WRITE, _FILE_OPEN, _DIRECTORY_OPTIONS)
|
||||
if status != _STATUS_SUCCESS:
|
||||
winerror = _ntdll.RtlNtStatusToDosError(status)
|
||||
raise OSError(None, ctypes.FormatError(winerror).strip(), path, winerror)
|
||||
info = _BY_HANDLE_FILE_INFORMATION()
|
||||
# Querying an open handle we just created does not fail.
|
||||
if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover
|
||||
error = ctypes.get_last_error()
|
||||
_kernel32.CloseHandle(handle)
|
||||
raise ctypes.WinError(error)
|
||||
if info.dwFileAttributes & _FILE_ATTRIBUTE_REPARSE_POINT:
|
||||
_kernel32.CloseHandle(handle)
|
||||
raise NotADirectoryError(ENOTDIR, "is a junction or symlink, not a directory", path)
|
||||
return handle
|
||||
|
||||
def _unpin_directory(handle: int) -> None:
|
||||
_kernel32.CloseHandle(handle)
|
||||
|
||||
def _is_unlinked(path: str) -> bool:
|
||||
"""
|
||||
Report whether *path* is gone or pending deletion, which ``os.open`` reports as the same EACCES as a denial.
|
||||
|
||||
NTFS keeps an unlinked name listed, refusing every open with ``STATUS_DELETE_PENDING``, until the last handle
|
||||
to the file closes. Another process holding such a handle, such as an antivirus scan, can keep it that way for
|
||||
longer than any retry grace, yet the name's owner has already removed it.
|
||||
"""
|
||||
handle, status = _nt_create(path, _FILE_READ_ATTRIBUTES | _SYNCHRONIZE, 0, _FILE_SHARE_ALL, _FILE_OPEN)
|
||||
if handle:
|
||||
_kernel32.CloseHandle(handle)
|
||||
return status in {_STATUS_OBJECT_NAME_NOT_FOUND, _STATUS_DELETE_PENDING}
|
||||
|
||||
def _nt_open(path: str, *, read_only: bool, owner_only: bool) -> tuple[int, int]:
|
||||
"""
|
||||
Open or create *path* for locking through ``NtCreateFile`` and return ``(handle, status)``.
|
||||
|
||||
Without *owner_only* a file this call creates gets no DACL of its own and inherits its directory's ACL.
|
||||
"""
|
||||
attributes = _FILE_ATTRIBUTE_READONLY if read_only else _FILE_ATTRIBUTE_NORMAL
|
||||
descriptor = _owner_only_descriptor() if owner_only else None
|
||||
return _nt_create(
|
||||
path, _DESIRED_ACCESS, attributes, _FILE_SHARE_READ_WRITE, _FILE_OPEN_IF, descriptor=descriptor
|
||||
)
|
||||
|
||||
@cache
|
||||
def _owner_only_descriptor() -> ctypes.c_void_p:
|
||||
# The DACL os.mkdir(mode=0o700) applies, minus the directory-only inheritance flags: protected (P), so the
|
||||
# directory's inheritable entries stay out, with full access (FA) for SYSTEM, Administrators and the owner.
|
||||
# Converted once and never freed: the open retries every few milliseconds while it waits.
|
||||
descriptor = ctypes.c_void_p()
|
||||
if not _advapi32.ConvertStringSecurityDescriptorToSecurityDescriptorW(
|
||||
"D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;OW)", 1, ctypes.byref(descriptor), None
|
||||
): # pragma: no cover # a constant string converts
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
return descriptor
|
||||
|
||||
def _nt_create( # ruff:ignore[too-many-arguments, too-many-positional-arguments] # NtCreateFile's own knobs
|
||||
path: str,
|
||||
access: int,
|
||||
attributes: int,
|
||||
share: int,
|
||||
disposition: int,
|
||||
options: int = _CREATE_OPTIONS,
|
||||
*,
|
||||
descriptor: ctypes.c_void_p | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Open *path* through ``NtCreateFile`` and return ``(handle, status)``.
|
||||
|
||||
``RtlDosPathNameToNtPathName_U_WithStatus`` translates the Win32 path to the NT namespace, handling relative,
|
||||
drive, UNC and extended-length path forms as Win32 itself would, and allocates a buffer that
|
||||
``RtlFreeUnicodeString`` releases. The handle is ``0`` unless the status is ``STATUS_SUCCESS``: one that comes
|
||||
with ``STATUS_OPLOCK_BREAK_IN_PROGRESS`` is closed here, since the file is not ours until the oplock holder
|
||||
answers.
|
||||
"""
|
||||
nt_name = _UNICODE_STRING()
|
||||
status = _ntdll.RtlDosPathNameToNtPathName_U_WithStatus(path, ctypes.byref(nt_name), None, None) & 0xFFFFFFFF
|
||||
if status != _STATUS_SUCCESS:
|
||||
return 0, status
|
||||
try:
|
||||
object_attributes = _OBJECT_ATTRIBUTES()
|
||||
object_attributes.Length = ctypes.sizeof(_OBJECT_ATTRIBUTES)
|
||||
object_attributes.ObjectName = ctypes.pointer(nt_name)
|
||||
object_attributes.Attributes = _OBJ_CASE_INSENSITIVE
|
||||
object_attributes.SecurityDescriptor = descriptor
|
||||
handle = wintypes.HANDLE()
|
||||
io_status = _IO_STATUS_BLOCK()
|
||||
status = (
|
||||
_ntdll.NtCreateFile(
|
||||
ctypes.byref(handle),
|
||||
access,
|
||||
ctypes.byref(object_attributes),
|
||||
ctypes.byref(io_status),
|
||||
None,
|
||||
attributes,
|
||||
share,
|
||||
disposition,
|
||||
options,
|
||||
None,
|
||||
0,
|
||||
)
|
||||
& 0xFFFFFFFF
|
||||
)
|
||||
finally:
|
||||
_ntdll.RtlFreeUnicodeString(ctypes.byref(nt_name))
|
||||
if status == _STATUS_OPLOCK_BREAK_IN_PROGRESS:
|
||||
_kernel32.CloseHandle(handle)
|
||||
if status != _STATUS_SUCCESS:
|
||||
return 0, status
|
||||
return handle.value or 0, status
|
||||
|
||||
def _directory_is_private(path: str) -> bool:
|
||||
"""
|
||||
Report whether only the current user, ``SYSTEM`` and ``Administrators`` can create or remove names in *path*.
|
||||
|
||||
A directory whose owner is anyone else can have its ACL rewritten by that owner. Entries that only a child
|
||||
inherits count too, since the DACL says nothing more precise about who may write.
|
||||
"""
|
||||
owner, dacl, descriptor = ctypes.c_void_p(), ctypes.c_void_p(), ctypes.c_void_p()
|
||||
error = _advapi32.GetNamedSecurityInfoW(
|
||||
path,
|
||||
1, # SE_FILE_OBJECT
|
||||
0x1 | 0x4, # OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION
|
||||
ctypes.byref(owner),
|
||||
None,
|
||||
ctypes.byref(dacl),
|
||||
None,
|
||||
ctypes.byref(descriptor),
|
||||
)
|
||||
if error == _ERROR_ACCESS_DENIED:
|
||||
# C:\Windows\Temp is one. Anything that cannot be proven private is not.
|
||||
return False
|
||||
if error: # pragma: no cover # the directory was opened or listed an instant earlier
|
||||
raise ctypes.WinError(error)
|
||||
try:
|
||||
# SYSTEM, Administrators, CREATOR OWNER and OWNER RIGHTS; the last two stand for the owner.
|
||||
trusted = {_sid_from_text(text) for text in ("S-1-5-18", "S-1-5-32-544", "S-1-3-0", "S-1-3-4")}
|
||||
trusted.add(_user_sid())
|
||||
# A missing DACL grants everyone full access.
|
||||
return (
|
||||
bool(dacl.value)
|
||||
and _sid_at(cast("int", owner.value)) in trusted
|
||||
and _name_writers(cast("int", dacl.value)) <= trusted
|
||||
)
|
||||
finally:
|
||||
_kernel32.LocalFree(descriptor)
|
||||
|
||||
def _name_writers(dacl: int) -> set[bytes]:
|
||||
(count,) = struct.unpack("<4xH2x", ctypes.string_at(dacl, 8))
|
||||
writers = set()
|
||||
offset = 8
|
||||
for _ in range(count):
|
||||
ace_type, _flags, ace_size, mask = struct.unpack("<BBHI", ctypes.string_at(dacl + offset, 8))
|
||||
# Allowed and allowed-callback ACEs share a layout; object ACEs exist only in directory services. The check
|
||||
# is conservative: any right that can turn into creating a name counts, namely FILE_ADD_FILE,
|
||||
# FILE_ADD_SUBDIRECTORY, FILE_DELETE_CHILD, WRITE_DAC, WRITE_OWNER, GENERIC_WRITE and GENERIC_ALL.
|
||||
if ace_type in {0, 9} and mask & (0x2 | 0x4 | 0x40 | 0x40000 | 0x80000 | 0x40000000 | 0x10000000):
|
||||
writers.add(_sid_at(dacl + offset + 8))
|
||||
offset += ace_size
|
||||
return writers
|
||||
|
||||
def _user_sid() -> bytes:
|
||||
token = wintypes.HANDLE()
|
||||
token_query: Final = 0x0008
|
||||
open_as_self: Final = True
|
||||
# An impersonating thread acts as the account it impersonates, which is the one whose names the lock creates.
|
||||
if not (
|
||||
_advapi32.OpenThreadToken(_kernel32.GetCurrentThread(), token_query, open_as_self, ctypes.byref(token))
|
||||
or _advapi32.OpenProcessToken(_kernel32.GetCurrentProcess(), token_query, ctypes.byref(token))
|
||||
): # pragma: no cover # every process may query its own token
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
try:
|
||||
size = wintypes.DWORD()
|
||||
_advapi32.GetTokenInformation(token, 1, None, 0, ctypes.byref(size))
|
||||
buffer = ctypes.create_string_buffer(size.value)
|
||||
if not _advapi32.GetTokenInformation(
|
||||
token, 1, buffer, size, ctypes.byref(size)
|
||||
): # pragma: no cover # the buffer has the size the first call asked for
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
return _sid_at(cast("int", ctypes.c_void_p.from_buffer(buffer).value))
|
||||
finally:
|
||||
_kernel32.CloseHandle(token)
|
||||
|
||||
def _sid_from_text(text: str) -> bytes:
|
||||
revision, authority, *subauthorities = (int(part) for part in text.removeprefix("S-").split("-"))
|
||||
return (
|
||||
struct.pack("<BB", revision, len(subauthorities))
|
||||
+ authority.to_bytes(6, "big")
|
||||
+ struct.pack(f"<{len(subauthorities)}I", *subauthorities)
|
||||
)
|
||||
|
||||
def _sid_at(address: int) -> bytes:
|
||||
return ctypes.string_at(address, 8 + 4 * ctypes.string_at(address + 1, 1)[0])
|
||||
|
||||
def _make_private_directory(path: str) -> None:
|
||||
"""Create *path* with the owner-only descriptor, so that nobody else can add a name to it."""
|
||||
attributes = _SECURITY_ATTRIBUTES(
|
||||
nLength=ctypes.sizeof(_SECURITY_ATTRIBUTES),
|
||||
lpSecurityDescriptor=_owner_only_descriptor(),
|
||||
bInheritHandle=False,
|
||||
)
|
||||
if not _kernel32.CreateDirectoryW(path, ctypes.byref(attributes)): # pragma: no cover # the name is fresh
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
|
||||
@contextmanager
|
||||
def _pinned(path: str) -> Generator[tuple[int, int]]:
|
||||
"""Hold *path* open without delete sharing, which keeps its name from being renamed or removed."""
|
||||
with _opened(path, _FILE_SHARE_READ_WRITE) as handle:
|
||||
yield _identity_of(handle)
|
||||
|
||||
def _identity_at(path: str) -> tuple[int, int]:
|
||||
with _opened(path, _FILE_SHARE_ALL) as handle:
|
||||
return _identity_of(handle)
|
||||
|
||||
def _identity_of_fd(fd: int) -> tuple[int, int]:
|
||||
return _identity_of(msvcrt.get_osfhandle(fd))
|
||||
|
||||
@contextmanager
|
||||
def _opened(path: str, share: int) -> Generator[int]:
|
||||
# The final component is opened as itself, never through a reparse point.
|
||||
handle = _kernel32.CreateFileW(
|
||||
path,
|
||||
_FILE_READ_ATTRIBUTES,
|
||||
share,
|
||||
None,
|
||||
3, # OPEN_EXISTING
|
||||
0x02000000 | 0x00200000, # FILE_FLAG_BACKUP_SEMANTICS opens a directory; FILE_FLAG_OPEN_REPARSE_POINT
|
||||
None,
|
||||
)
|
||||
if handle in {None, _INVALID_HANDLE_VALUE}: # pragma: no cover # callers open a name they just made or listed
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
try:
|
||||
yield cast("int", handle)
|
||||
finally:
|
||||
_kernel32.CloseHandle(handle)
|
||||
|
||||
def _identity_of(handle: int) -> tuple[int, int]:
|
||||
info = _BY_HANDLE_FILE_INFORMATION()
|
||||
if not _kernel32.GetFileInformationByHandle(handle, ctypes.byref(info)): # pragma: no cover
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
return info.dwVolumeSerialNumber, info.nFileIndexHigh << 32 | info.nFileIndexLow
|
||||
|
||||
else: # pragma: win32 no cover
|
||||
|
||||
class WindowsFileLock(BaseFileLock):
|
||||
"""Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems."""
|
||||
|
||||
def _acquire(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def _release(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
__all__ = [
|
||||
"WindowsFileLock",
|
||||
"_directory_is_private",
|
||||
"_identity_at",
|
||||
"_identity_of_fd",
|
||||
"_is_unlinked",
|
||||
"_make_private_directory",
|
||||
"_open_deletable_for_reading",
|
||||
"_open_non_reparse_fd",
|
||||
"_pin_directory",
|
||||
"_pinned",
|
||||
"_unpin_directory",
|
||||
"open_for_delete_without_oplock_wait",
|
||||
"open_read_without_oplock_wait",
|
||||
"unlink_without_oplock_wait",
|
||||
]
|
||||
@@ -0,0 +1,925 @@
|
||||
"""An asyncio-based implementation of the file lock."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__lazy_modules__ = [
|
||||
"asyncio",
|
||||
"concurrent",
|
||||
"contextlib",
|
||||
"filelock._async",
|
||||
"filelock._error",
|
||||
"filelock._util",
|
||||
"functools",
|
||||
"inspect",
|
||||
"threading",
|
||||
]
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import functools
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from concurrent import futures
|
||||
from dataclasses import dataclass, field, fields
|
||||
from inspect import iscoroutinefunction
|
||||
from threading import RLock
|
||||
from typing import TYPE_CHECKING, Final, NoReturn, TypeVar, cast
|
||||
|
||||
from filelock._api import (
|
||||
_UNSET_FILE_MODE,
|
||||
BaseFileLock,
|
||||
CloseErrorPolicy,
|
||||
ContextErrorPolicy,
|
||||
FileLockContext,
|
||||
FileLockMeta,
|
||||
ThreadLocalFileContext,
|
||||
_append_exception_context,
|
||||
_canonical,
|
||||
_delegate_to_current,
|
||||
_ExtraValue,
|
||||
_fork_transition,
|
||||
_grouped_errors,
|
||||
_is_system_call_failure,
|
||||
_raise_body_and_release,
|
||||
_raise_chained_errors,
|
||||
_raise_cleanup_errors,
|
||||
_raise_grouped_errors,
|
||||
_register_fork_object,
|
||||
_resolve_poll_interval,
|
||||
_resolve_timeout,
|
||||
)
|
||||
from filelock._async import (
|
||||
_AsyncTransitionGate,
|
||||
_AsyncTransitionUnavailableError,
|
||||
_BackendOutcome,
|
||||
_capture_awaitable,
|
||||
_capture_call,
|
||||
_drain_future,
|
||||
_future_result,
|
||||
_run_in_executor,
|
||||
_wait_until_done,
|
||||
)
|
||||
from filelock._error import Timeout
|
||||
from filelock._lease import SoftFileLease
|
||||
from filelock._soft import SoftFileLock
|
||||
from filelock._strict import StrictSoftFileLock
|
||||
from filelock._unix import UnixFileLock
|
||||
from filelock._util import ensure_directory_exists
|
||||
from filelock._windows import WindowsFileLock
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Hashable, Mapping
|
||||
from types import TracebackType
|
||||
from typing import Self
|
||||
|
||||
|
||||
_LOGGER: Final[logging.Logger] = logging.getLogger("filelock")
|
||||
_ASYNC_RELEASE_CANCELLATION_ERRORS: Final[str] = "lock release cancellation and backend release both failed"
|
||||
_ASYNC_CONTEXT_RELEASE_ERRORS: Final[str] = "context body, release cancellation, and backend release failed"
|
||||
_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR: Final[str] = "_filelock_async_release_cancellation"
|
||||
_ASYNC_RELEASE_CANCELLATION_MARKER: Final[list[None]] = []
|
||||
_FINALIZER_RELEASES: Final[set[asyncio.Task[None] | futures.Future[None]]] = set()
|
||||
|
||||
_AT = TypeVar("_AT", bound="BaseAsyncFileLock")
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class AsyncFileLockMeta(FileLockMeta):
|
||||
def __call__( # ruff:ignore[too-many-arguments] # forwards the public constructor's documented parameters
|
||||
cls: type[_AT], # ruff:ignore[invalid-first-argument-name-for-method] # metaclass __call__ receives the class being constructed
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
mode: int = _UNSET_FILE_MODE,
|
||||
thread_local: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = False,
|
||||
poll_interval: float = 0.05,
|
||||
lifetime: float | None = None,
|
||||
context_error_policy: ContextErrorPolicy = "chain",
|
||||
close_error_policy: CloseErrorPolicy = "default",
|
||||
fallback_to_soft: bool = True,
|
||||
preserve_lock_file: bool = False,
|
||||
on_acquired: Callable[[int], None] | None = None,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
run_in_executor: bool = True,
|
||||
executor: futures.Executor | None = None,
|
||||
**kwargs: _ExtraValue,
|
||||
) -> _AT:
|
||||
if thread_local and run_in_executor:
|
||||
msg = "run_in_executor is not supported when thread_local is True"
|
||||
raise ValueError(msg)
|
||||
return super().__call__( # a subclass may add options of its own, as AsyncSoftFileLease does
|
||||
**kwargs,
|
||||
lock_file=lock_file,
|
||||
timeout=timeout,
|
||||
mode=mode,
|
||||
thread_local=thread_local,
|
||||
blocking=blocking,
|
||||
is_singleton=is_singleton,
|
||||
poll_interval=poll_interval,
|
||||
lifetime=lifetime,
|
||||
context_error_policy=context_error_policy,
|
||||
close_error_policy=close_error_policy,
|
||||
fallback_to_soft=fallback_to_soft,
|
||||
preserve_lock_file=preserve_lock_file,
|
||||
on_acquired=on_acquired,
|
||||
loop=loop,
|
||||
run_in_executor=run_in_executor,
|
||||
executor=executor,
|
||||
)
|
||||
|
||||
|
||||
class BaseAsyncFileLock(BaseFileLock, metaclass=AsyncFileLockMeta):
|
||||
"""
|
||||
Base class for asynchronous file locks.
|
||||
|
||||
.. versionadded:: 3.15.0
|
||||
|
||||
"""
|
||||
|
||||
_deadlock_holder_desc: str = "BaseAsyncFileLock instance in this task"
|
||||
_constructor_lifetime_warning_stacklevel: int = 4
|
||||
|
||||
@staticmethod
|
||||
def _deadlock_scope() -> Hashable | None:
|
||||
# One event loop thread runs every task, so a thread-scoped registry cannot tell a same-task reacquire
|
||||
# (a real deadlock: the polling task never reaches its own release) from another task queuing behind the
|
||||
# holder (no deadlock: each poll yields, so the holder runs on and releases). Only the first may fail
|
||||
# fast, so scope holders to the task.
|
||||
return asyncio.current_task()
|
||||
|
||||
def __init__( # ruff:ignore[too-many-arguments] # public constructor: one parameter per documented lock option
|
||||
self,
|
||||
lock_file: str | os.PathLike[str],
|
||||
timeout: float = -1,
|
||||
mode: int = _UNSET_FILE_MODE,
|
||||
thread_local: bool = False, # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility
|
||||
*,
|
||||
blocking: bool = True,
|
||||
is_singleton: bool = False,
|
||||
poll_interval: float = 0.05,
|
||||
lifetime: float | None = None,
|
||||
context_error_policy: ContextErrorPolicy = "chain",
|
||||
close_error_policy: CloseErrorPolicy = "default",
|
||||
fallback_to_soft: bool = True,
|
||||
preserve_lock_file: bool = False,
|
||||
on_acquired: Callable[[int], None] | None = None,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
run_in_executor: bool = True,
|
||||
executor: futures.Executor | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Create a new lock object.
|
||||
|
||||
:param lock_file: path to the file
|
||||
:param timeout: default timeout when acquiring the lock, in seconds. It will be used as fallback value in the
|
||||
acquire method, if no timeout value (``None``) is given. If you want to disable the timeout, set it to a
|
||||
negative value. A timeout of 0 means that there is exactly one attempt to acquire the file lock.
|
||||
:param mode: file permissions for the lockfile. When not specified, the OS controls permissions via umask and
|
||||
default ACLs, preserving POSIX default ACL inheritance in shared directories.
|
||||
:param thread_local: Whether this object's internal context should be thread local or not. If this is set to
|
||||
``False`` (the default) then the lock will be reentrant across threads. When ``True``, **all fields of the
|
||||
lock's internal context are per-thread**, including the configuration values ``poll_interval``, ``timeout``,
|
||||
``blocking``, ``mode``, and ``lifetime``. Setting one of these properties from one thread does not change
|
||||
the value seen by another thread; threads that did not perform the write continue to see the value supplied
|
||||
at construction time. ``mode`` has no setter, so construction is the only place it is ever set. If you need
|
||||
configuration values to be visible across threads, construct the lock with ``thread_local=False``.
|
||||
:param blocking: whether the lock should be blocking or not
|
||||
:param is_singleton: If this is set to ``True`` then only one instance of this class will be created per lock
|
||||
file. This is useful if you want to use the lock object for reentrant locking without needing to pass the
|
||||
same object around.
|
||||
:param poll_interval: default interval for polling the lock file, in seconds. It will be used as fallback value
|
||||
in the acquire method, if no poll_interval value (``None``) is given.
|
||||
:param lifetime: for :class:`AsyncSoftFileLock`, the age in seconds after which a waiting process may delete
|
||||
the marker, even while its holder remains alive. This legacy expiry mode does not provide strict mutual
|
||||
exclusion. ``None`` (the default) disables age-based expiry. Native OS locks (:class:`AsyncFileLock`)
|
||||
cannot be revoked by file age and ignore a non-``None`` ``lifetime`` with a warning.
|
||||
:param context_error_policy: how a context manager reconciles a failure in its body with a failure while
|
||||
releasing on exit. ``"chain"`` (the default) keeps Python's behavior: the release error propagates with the
|
||||
body error in its ``__context__``. ``"group"`` raises a :class:`BaseExceptionGroup` holding the body error
|
||||
first and the release error second, so neither hides the other.
|
||||
:param close_error_policy: for native locks (:class:`AsyncFileLock`), what to do with an ``os.close`` failure
|
||||
after the OS unlock has already committed. ``"default"`` keeps each platform's historical behavior,
|
||||
``"raise"`` always propagates the ``OSError``, and ``"suppress"`` always ignores it.
|
||||
:param fallback_to_soft: for :class:`AsyncFileLock`, whether to fall back to soft existence locking when
|
||||
``flock`` returns ``ENOSYS``. ``True`` (default) keeps the fallback; ``False`` propagates the error.
|
||||
:param preserve_lock_file: for native locks (:class:`AsyncFileLock`), whether filelock promises not to unlink
|
||||
the lock pathname on release. ``False`` (default) keeps each backend's cleanup; ``True`` keeps a stable file
|
||||
identity (Windows skips its unlink, Unix refuses the ``ENOSYS`` soft fallback). :class:`AsyncSoftFileLock`
|
||||
rejects ``True``.
|
||||
:param on_acquired: for native locks (:class:`AsyncFileLock`), a callable invoked with the borrowed lock
|
||||
descriptor once per physical acquisition, after the lock is held but before
|
||||
:meth:`~BaseAsyncFileLock.acquire` returns. With ``run_in_executor=True`` (the default) it runs in the
|
||||
backend executor. It must not close or unlock the descriptor; a raise rolls the acquisition back.
|
||||
:class:`AsyncSoftFileLock` rejects it.
|
||||
:param loop: the event loop a garbage-collected lock schedules its release on when no loop runs in the
|
||||
collecting thread. :meth:`~BaseAsyncFileLock.acquire` and :meth:`~BaseAsyncFileLock.release` always run
|
||||
on the loop that awaits them.
|
||||
:param run_in_executor: If this is set to ``True`` then the lock will be acquired in an executor.
|
||||
:param executor: The executor to use. If not specified, the default executor will be used.
|
||||
|
||||
"""
|
||||
self._creator_pid = os.getpid()
|
||||
self._holder_id = id(self)
|
||||
self._is_thread_local = thread_local
|
||||
self._is_singleton = is_singleton
|
||||
self._context_error_policy = context_error_policy # already validated by the metaclass
|
||||
self._close_error_policy = close_error_policy # already validated by the metaclass
|
||||
self._fallback_to_soft = fallback_to_soft
|
||||
self._preserve_lock_file = preserve_lock_file # already validated by the metaclass
|
||||
self._on_acquired = on_acquired # already validated by the metaclass
|
||||
# BaseFileLock.__init__ never runs here, and the finalizer releases thread-local holds through these.
|
||||
self._transition_lock = RLock()
|
||||
self._thread_holds = {}
|
||||
self._thread_pending = {}
|
||||
|
||||
self._async_context: Final[AsyncFileLockContext] = (
|
||||
AsyncThreadLocalFileContext if thread_local else AsyncFileLockContext
|
||||
)(
|
||||
lock_file=os.fspath(lock_file),
|
||||
timeout=timeout,
|
||||
mode=mode,
|
||||
blocking=blocking,
|
||||
poll_interval=poll_interval,
|
||||
lifetime=lifetime,
|
||||
loop=loop,
|
||||
run_in_executor=run_in_executor,
|
||||
executor=executor,
|
||||
)
|
||||
# BaseFileLock declares _context as FileLockContext; narrowing that writable attribute here is unsound.
|
||||
self._context = self._async_context
|
||||
_register_fork_object(self)
|
||||
|
||||
def _singleton_extra_mismatches(self, kwargs: Mapping[str, _ExtraValue], /) -> dict[str, tuple[str, str]]:
|
||||
mismatches = super()._singleton_extra_mismatches(kwargs)
|
||||
# Loops and executors compare by identity: a different one schedules the lock's work elsewhere.
|
||||
mismatches.update(
|
||||
(name, (str(kwargs[name]), str(value)))
|
||||
for name, value in (("loop", self.loop), ("executor", self.executor))
|
||||
if kwargs[name] is not value
|
||||
)
|
||||
if kwargs["run_in_executor"] != self.run_in_executor:
|
||||
mismatches["run_in_executor"] = (str(kwargs["run_in_executor"]), str(self.run_in_executor))
|
||||
return mismatches
|
||||
|
||||
@property
|
||||
def run_in_executor(self) -> bool:
|
||||
"""Whether run in executor."""
|
||||
return self._async_context.run_in_executor
|
||||
|
||||
@property
|
||||
def executor(self) -> futures.Executor | None:
|
||||
"""The executor."""
|
||||
return self._async_context.executor
|
||||
|
||||
@executor.setter
|
||||
def executor(self, value: futures.Executor | None) -> None: # pragma: no cover
|
||||
"""
|
||||
Change the executor.
|
||||
|
||||
:param futures.Executor | None value: the new executor or ``None``
|
||||
|
||||
"""
|
||||
self._async_context.executor = value
|
||||
|
||||
@property
|
||||
def loop(self) -> asyncio.AbstractEventLoop | None:
|
||||
"""The event loop."""
|
||||
return self._async_context.loop
|
||||
|
||||
async def acquire( # ty: ignore[invalid-method-override]
|
||||
self,
|
||||
timeout: float | None = None,
|
||||
poll_interval: float | None = None,
|
||||
*,
|
||||
blocking: bool | None = None,
|
||||
cancel_check: Callable[[], bool] | None = None,
|
||||
) -> AsyncAcquireReturnProxy:
|
||||
"""
|
||||
Try to acquire the file lock.
|
||||
|
||||
:param timeout: maximum wait time for acquiring the lock, ``None`` means use the default
|
||||
:attr:`~BaseFileLock.timeout` is and if ``timeout < 0``, there is no timeout and this method will block
|
||||
until the lock could be acquired
|
||||
:param poll_interval: interval of trying to acquire the lock file, ``None`` means use the default
|
||||
:attr:`~BaseFileLock.poll_interval`
|
||||
:param blocking: whether to retry until this call holds the lock or the timeout expires, ``None`` means use
|
||||
the default :attr:`~BaseFileLock.blocking`. ``False`` makes one attempt and raises :class:`~Timeout` on
|
||||
failure.
|
||||
:param cancel_check: a callable returning ``True`` when the acquisition should be canceled. Checked on each poll
|
||||
iteration. When triggered, raises :class:`~Timeout` just like an expired timeout.
|
||||
|
||||
:returns: a context object that will unlock the file when the context is exited
|
||||
|
||||
:raises Timeout: if fails to acquire lock within the timeout period
|
||||
:raises ValueError: if a blocking call gets a ``nan`` timeout, or *poll_interval* exceeds
|
||||
:data:`threading.TIMEOUT_MAX`
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# You can use this method in the context manager (recommended)
|
||||
async with await lock.acquire():
|
||||
pass
|
||||
|
||||
# Or use an equivalent try-finally construct:
|
||||
await lock.acquire()
|
||||
try:
|
||||
pass
|
||||
finally:
|
||||
await lock.release()
|
||||
|
||||
"""
|
||||
self._raise_if_inherited()
|
||||
if timeout is None:
|
||||
timeout = self._context.timeout
|
||||
|
||||
if blocking is None:
|
||||
blocking = self._context.blocking
|
||||
|
||||
if poll_interval is None:
|
||||
poll_interval = self._context.poll_interval
|
||||
poll_interval = _resolve_poll_interval(poll_interval)
|
||||
timeout = _resolve_timeout(timeout, blocking=blocking)
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
return await self._acquire_with_admission(
|
||||
blocking=blocking,
|
||||
cancel_check=cancel_check,
|
||||
timeout=timeout,
|
||||
poll_interval=poll_interval,
|
||||
start_time=start_time,
|
||||
)
|
||||
except _AsyncTransitionUnavailableError:
|
||||
raise Timeout(self.lock_file) from None
|
||||
|
||||
async def _acquire_with_admission(
|
||||
self,
|
||||
*,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
timeout: float,
|
||||
poll_interval: float,
|
||||
start_time: float,
|
||||
) -> AsyncAcquireReturnProxy:
|
||||
async with self._async_context.transition_gate.hold_for_acquire(
|
||||
blocking=blocking,
|
||||
cancel_check=cancel_check,
|
||||
deadline=None if timeout < 0 else start_time + timeout,
|
||||
poll_interval=poll_interval,
|
||||
):
|
||||
# Re-entry only counts, so it skips the executor round trip that resolving the path costs.
|
||||
if self.is_locked:
|
||||
self._context.lock_counter += 1
|
||||
return AsyncAcquireReturnProxy(lock=self)
|
||||
# A canceled provisional acquire must finish rollback before another caller can claim its descriptor.
|
||||
canonical = await self._off_loop(functools.partial(_canonical, self.lock_file))
|
||||
self._context.lock_counter += 1
|
||||
self._raise_if_would_deadlock(canonical, timeout=timeout, blocking=blocking)
|
||||
self._pin_lock_path(canonical)
|
||||
try:
|
||||
await self._async_poll_until_acquired(
|
||||
blocking=blocking,
|
||||
cancel_check=cancel_check,
|
||||
timeout=timeout,
|
||||
poll_interval=poll_interval,
|
||||
start_time=start_time,
|
||||
)
|
||||
except BaseException:
|
||||
self._reconcile_failed_acquire(canonical)
|
||||
raise
|
||||
self._commit_acquire(canonical)
|
||||
return AsyncAcquireReturnProxy(lock=self)
|
||||
|
||||
async def _off_loop(self, call: Callable[[], _T]) -> _T:
|
||||
# A slow mount stalls a filesystem call, and the loop with it; a coroutine backend runs on the loop anyway.
|
||||
if not self.run_in_executor or iscoroutinefunction(self._acquire):
|
||||
return call()
|
||||
future = _run_in_executor(asyncio.get_running_loop(), self.executor, call)
|
||||
await _wait_until_done(future)
|
||||
return _future_result(future)
|
||||
|
||||
async def _async_poll_until_acquired(
|
||||
self,
|
||||
*,
|
||||
blocking: bool,
|
||||
cancel_check: Callable[[], bool] | None,
|
||||
timeout: float,
|
||||
poll_interval: float,
|
||||
start_time: float,
|
||||
) -> None:
|
||||
lock_id = id(self)
|
||||
lock_filename = self.lock_file
|
||||
await self._off_loop(functools.partial(ensure_directory_exists, self._lock_path))
|
||||
while True:
|
||||
self._raise_if_inherited()
|
||||
_LOGGER.debug("Attempting to acquire lock %s on %s", lock_id, lock_filename)
|
||||
await self._run_acquire_attempt()
|
||||
self._raise_if_inherited()
|
||||
if self.is_locked:
|
||||
_LOGGER.debug("Lock %s acquired on %s", lock_id, lock_filename)
|
||||
return
|
||||
if self._check_give_up(
|
||||
blocking=blocking,
|
||||
cancel_check=cancel_check,
|
||||
timeout=timeout,
|
||||
start_time=start_time,
|
||||
):
|
||||
raise Timeout(lock_filename, await self._off_loop(self._unprobeable_holder))
|
||||
delay = poll_interval
|
||||
if timeout >= 0:
|
||||
delay = min(delay, max(start_time + timeout - time.perf_counter(), 0.0))
|
||||
_LOGGER.debug("Lock %s not acquired on %s, waiting %s seconds ...", lock_id, lock_filename, delay)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
async def _run_acquire_attempt(self) -> None:
|
||||
acquire_future = self._start_internal_method(
|
||||
self._acquire_with_fork_tracking_async
|
||||
if iscoroutinefunction(self._acquire)
|
||||
else self._break_expired_and_acquire
|
||||
)
|
||||
try:
|
||||
await _wait_until_done(acquire_future)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
acquire_error: BaseException | None = None
|
||||
try:
|
||||
await _drain_future(acquire_future)
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
acquire_error = error
|
||||
|
||||
rollback_error: BaseException | None = None
|
||||
if self.is_locked:
|
||||
try:
|
||||
await _drain_future(self._start_tracked_release())
|
||||
except BaseException as error: # ruff:ignore[blind-except] # reported with the cancellation below
|
||||
rollback_error = error
|
||||
|
||||
if acquire_error is not None:
|
||||
if rollback_error is not None: # pragma: needs fcntl
|
||||
self._raise_cancelled_errors(
|
||||
"lock acquisition cancellation, backend attempt, and rollback failed",
|
||||
cancellation,
|
||||
acquire_error,
|
||||
rollback_error,
|
||||
)
|
||||
self._raise_cancelled_errors(
|
||||
"lock acquisition cancellation and backend attempt both failed", cancellation, acquire_error
|
||||
)
|
||||
if rollback_error is not None:
|
||||
self._raise_cancelled_errors(
|
||||
"lock acquisition cancellation and rollback both failed", cancellation, rollback_error
|
||||
)
|
||||
raise
|
||||
try:
|
||||
_future_result(acquire_future)
|
||||
except asyncio.CancelledError as acquire_error:
|
||||
await self._rollback_backend_cancelled_acquire(acquire_error)
|
||||
|
||||
async def _rollback_backend_cancelled_acquire(self, acquire_error: asyncio.CancelledError) -> NoReturn:
|
||||
if self.is_locked:
|
||||
try:
|
||||
await _drain_future(self._start_tracked_release())
|
||||
except BaseException as rollback_error: # ruff:ignore[blind-except] # both backend errors must surface
|
||||
self._raise_acquire_rollback_errors(acquire_error, rollback_error)
|
||||
raise acquire_error
|
||||
|
||||
def _raise_acquire_rollback_errors(self, acquire_error: BaseException, rollback_error: BaseException) -> NoReturn:
|
||||
if self._context_error_policy == "group":
|
||||
_raise_grouped_errors("lock acquisition backend and rollback both failed", acquire_error, rollback_error)
|
||||
_raise_chained_errors(acquire_error, rollback_error)
|
||||
|
||||
def _raise_cancelled_errors(
|
||||
self,
|
||||
message: str,
|
||||
cancellation: asyncio.CancelledError,
|
||||
first_error: BaseException,
|
||||
second_error: BaseException | None = None,
|
||||
) -> NoReturn:
|
||||
if self._context_error_policy == "group":
|
||||
marker = (
|
||||
(_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER)
|
||||
if message == _ASYNC_RELEASE_CANCELLATION_ERRORS
|
||||
else None
|
||||
)
|
||||
if second_error is None:
|
||||
_raise_grouped_errors(message, cancellation, first_error, marker=marker)
|
||||
_raise_grouped_errors(message, cancellation, first_error, second_error, marker=marker)
|
||||
if (context := first_error.__context__) is not None and context is not cancellation:
|
||||
if (cancellation_context := cancellation.__context__) is not None:
|
||||
_append_exception_context(context, cancellation_context)
|
||||
cancellation.__context__ = context
|
||||
first_error.__context__ = cancellation
|
||||
_raise_chained_errors(first_error, second_error)
|
||||
|
||||
async def release(self, force: bool = False) -> None: # ty: ignore[invalid-method-override] # ruff:ignore[boolean-type-hint-positional-argument, boolean-default-value-positional-argument] # public API: positional bool kept for backwards compatibility
|
||||
"""
|
||||
Release the file lock. The lock is only completely released when the lock counter reaches 0. The lock file
|
||||
itself may be deleted automatically, the behavior is platform-specific.
|
||||
|
||||
:param force: If true, the lock counter is ignored and the lock is released in every case.
|
||||
|
||||
"""
|
||||
if self._creator_pid != os.getpid() or not self.is_locked:
|
||||
return
|
||||
async with self._async_context.transition_gate.hold():
|
||||
await self._release_serialized(force=force)
|
||||
|
||||
async def _release_serialized(self, *, force: bool) -> None:
|
||||
counter_before: Final = self._context.lock_counter
|
||||
if not self.is_locked:
|
||||
return
|
||||
if not force and self._context.lock_counter > 1:
|
||||
self._context.lock_counter -= 1
|
||||
return
|
||||
|
||||
try:
|
||||
await self._release_through_backend()
|
||||
except BaseException as release_error:
|
||||
if self._release_interrupted_before_commit(counter_before, release_error):
|
||||
with contextlib.suppress(Exception):
|
||||
self._release_with_fork_tracking()
|
||||
self._commit_release_if_released()
|
||||
raise
|
||||
|
||||
def _release_interrupted_before_commit(self, counter_before: int, release_error: BaseException) -> bool:
|
||||
# Same rule as the sync release: a handler that raised before anything committed leaves the counter as it was,
|
||||
# and only then does a second attempt release the hold. A worker thread never runs a handler, so a lock that
|
||||
# runs its backend in the executor has nothing to retry.
|
||||
return (
|
||||
counter_before == self._context.lock_counter
|
||||
and self.is_locked
|
||||
and not self.run_in_executor
|
||||
and not iscoroutinefunction(self._release)
|
||||
and not isinstance(release_error, asyncio.CancelledError)
|
||||
and not _is_system_call_failure(release_error)
|
||||
)
|
||||
|
||||
async def _release_through_backend(self) -> None:
|
||||
lock_id, lock_filename = id(self), self.lock_file
|
||||
_LOGGER.debug("Attempting to release lock %s on %s", lock_id, lock_filename)
|
||||
release_future = self._start_tracked_release()
|
||||
try:
|
||||
await _wait_until_done(release_future)
|
||||
except asyncio.CancelledError as cancellation:
|
||||
try:
|
||||
await _drain_future(release_future)
|
||||
except BaseException as release_error: # ruff:ignore[blind-except] # cancellation and backend failure must both surface
|
||||
self._commit_release_if_released()
|
||||
self._raise_cancelled_errors(_ASYNC_RELEASE_CANCELLATION_ERRORS, cancellation, release_error)
|
||||
self._commit_release()
|
||||
raise
|
||||
try:
|
||||
_future_result(release_future)
|
||||
except BaseException: # state follows the backend for control-flow exceptions too
|
||||
self._commit_release_if_released()
|
||||
raise
|
||||
self._commit_release()
|
||||
_LOGGER.debug("Lock %s released on %s", lock_id, lock_filename)
|
||||
|
||||
def _commit_release_if_released(self) -> None:
|
||||
# Commit only when the backend actually unlocked (close or unlink failed after the OS unlock). If the lock is
|
||||
# still held, keep the counter so a later release can retry.
|
||||
if not self.is_locked:
|
||||
self._commit_release()
|
||||
|
||||
def _start_internal_method(
|
||||
self, method: Callable[[], None] | Callable[[], Coroutine[None, None, None]]
|
||||
) -> asyncio.Future[_BackendOutcome[None]]:
|
||||
if iscoroutinefunction(method):
|
||||
return asyncio.create_task(_capture_awaitable(cast("Callable[[], Coroutine[None, None, None]]", method)()))
|
||||
loop = asyncio.get_running_loop()
|
||||
sync_method = cast("Callable[[], None]", method)
|
||||
if self.run_in_executor:
|
||||
return _run_in_executor(loop, self.executor, sync_method)
|
||||
future: asyncio.Future[_BackendOutcome[None]] = loop.create_future()
|
||||
future.set_result(_capture_call(sync_method))
|
||||
return future
|
||||
|
||||
def _start_tracked_release(self) -> asyncio.Future[_BackendOutcome[None]]:
|
||||
return self._start_internal_method(
|
||||
self._release_with_fork_tracking_async
|
||||
if iscoroutinefunction(self._release)
|
||||
else self._release_with_fork_tracking
|
||||
)
|
||||
|
||||
def _break_expired_and_acquire(self) -> None:
|
||||
# Runs as one backend call so the lifetime check's lstat and unlink stay off the event loop with the acquire.
|
||||
self._try_break_expired_lock()
|
||||
self._acquire_with_fork_tracking()
|
||||
|
||||
async def _acquire_with_fork_tracking_async(self) -> None:
|
||||
self._try_break_expired_lock()
|
||||
with _fork_transition(self):
|
||||
try:
|
||||
await cast("Callable[[], Awaitable[None]]", self._acquire)()
|
||||
except BaseException as acquisition_error:
|
||||
await self._rollback_failed_acquire_async(acquisition_error)
|
||||
raise
|
||||
try:
|
||||
self._register_context_descriptor()
|
||||
except BaseException as registration_error: # cancellation must roll back the descriptor
|
||||
await self._rollback_failed_registration_async(registration_error)
|
||||
raise
|
||||
if self.is_locked:
|
||||
await self._invoke_on_acquired_async()
|
||||
|
||||
async def _rollback_failed_acquire_async(self, acquisition_error: BaseException) -> None:
|
||||
if not self.is_locked:
|
||||
return
|
||||
registration_error: BaseException | None = None
|
||||
tracking_error: BaseException | None = None
|
||||
try:
|
||||
self._register_context_descriptor()
|
||||
except BaseException as error: # ruff:ignore[blind-except] # preserve registration and acquisition failures
|
||||
registration_error = error
|
||||
try:
|
||||
# Rollback may fail too; retain the fd so a child can close it without another identity probe.
|
||||
self._register_unverified_context_descriptor()
|
||||
except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback
|
||||
tracking_error = error
|
||||
try:
|
||||
await _drain_future(self._start_tracked_release())
|
||||
except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and acquisition failures
|
||||
if registration_error is None and tracking_error is None:
|
||||
self._raise_acquire_rollback_errors(acquisition_error, rollback_error)
|
||||
_raise_cleanup_errors(
|
||||
"lock acquisition cleanup failed",
|
||||
acquisition_error,
|
||||
registration_error,
|
||||
tracking_error,
|
||||
rollback_error,
|
||||
)
|
||||
if registration_error is not None:
|
||||
_raise_cleanup_errors(
|
||||
"lock acquisition cleanup failed", acquisition_error, registration_error, tracking_error
|
||||
)
|
||||
|
||||
async def _rollback_failed_registration_async(self, registration_error: BaseException) -> None:
|
||||
tracking_error: BaseException | None = None
|
||||
try:
|
||||
# Rollback may fail too; retain the fd so a child can close it without another identity probe.
|
||||
self._register_unverified_context_descriptor()
|
||||
except BaseException as error: # ruff:ignore[blind-except] # pragma: no cover - allocation/control-flow during fallback
|
||||
tracking_error = error
|
||||
try:
|
||||
await _drain_future(self._start_tracked_release())
|
||||
except BaseException as rollback_error: # ruff:ignore[blind-except] # preserve rollback and registration failures
|
||||
_raise_cleanup_errors(
|
||||
"descriptor registration cleanup failed", registration_error, tracking_error, rollback_error
|
||||
)
|
||||
if tracking_error is not None: # pragma: no cover - requires failed in-memory fallback
|
||||
_raise_cleanup_errors("descriptor registration cleanup failed", registration_error, tracking_error)
|
||||
|
||||
async def _invoke_on_acquired_async(self) -> None:
|
||||
if self._on_acquired is None or self._context.lock_counter != 1:
|
||||
return
|
||||
try:
|
||||
self._on_acquired(cast("int", self._context.lock_file_fd))
|
||||
except BaseException as callback_error: # caller control-flow errors must release the lock
|
||||
callback_context = callback_error.__context__
|
||||
try:
|
||||
await _drain_future(self._start_tracked_release())
|
||||
except BaseException as release_error: # ruff:ignore[blind-except] # both errors surface via the group below
|
||||
_raise_body_and_release(callback_error, release_error)
|
||||
callback_error.__context__ = callback_context
|
||||
raise
|
||||
|
||||
async def _release_with_fork_tracking_async(self) -> None:
|
||||
with _fork_transition(self):
|
||||
try:
|
||||
await cast("Callable[[], Awaitable[None]]", self._release)()
|
||||
finally:
|
||||
self._unregister_released_descriptor()
|
||||
|
||||
def __enter__(self) -> NoReturn:
|
||||
"""Sync context manager entry is not supported because lock acquisition is a coroutine."""
|
||||
msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager."
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
"""Sync context manager exit is not supported because lock release is a coroutine."""
|
||||
msg = "Use `async with`: acquire/release are coroutines and cannot be awaited in a sync context manager."
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
"""
|
||||
Acquire the lock.
|
||||
|
||||
:returns: the lock object
|
||||
|
||||
"""
|
||||
await self.acquire()
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
"""
|
||||
Release the lock, reconciling a release failure with any body failure per :attr:`context_error_policy`.
|
||||
|
||||
:param exc_type: the exception type if raised
|
||||
:param exc_value: the exception value if raised
|
||||
:param traceback: the exception traceback if raised
|
||||
|
||||
"""
|
||||
await self._release_in_context(exc_value)
|
||||
|
||||
async def _release_in_context( # ty: ignore[invalid-method-override]
|
||||
self, body_error: BaseException | None
|
||||
) -> None:
|
||||
# The async counterpart of BaseFileLock._release_in_context: await release, then apply the same policy.
|
||||
try:
|
||||
await self.release()
|
||||
except BaseException as release_error:
|
||||
if body_error is None:
|
||||
raise
|
||||
if self._context_error_policy == "chain":
|
||||
_append_exception_context(release_error, body_error)
|
||||
raise
|
||||
if (
|
||||
errors := _grouped_errors(
|
||||
release_error,
|
||||
_ASYNC_RELEASE_CANCELLATION_ERRORS,
|
||||
(_ASYNC_RELEASE_CANCELLATION_MARKER_ATTR, _ASYNC_RELEASE_CANCELLATION_MARKER),
|
||||
)
|
||||
) is not None:
|
||||
match errors:
|
||||
# The marker is only ever set on a (CancelledError, backend_error) pair, so this always matches.
|
||||
case (asyncio.CancelledError() as cancellation, backend_error): # pragma: no branch
|
||||
_raise_grouped_errors(_ASYNC_CONTEXT_RELEASE_ERRORS, body_error, cancellation, backend_error)
|
||||
_raise_body_and_release(body_error, release_error)
|
||||
|
||||
def _release_abandoned_holds(self) -> None:
|
||||
# Runs from garbage collection, where no event loop may be running.
|
||||
with contextlib.suppress(Exception):
|
||||
# A thread-local lock never uses an executor, so an in-place release matches the loop's and also reaches
|
||||
# threads whose loops it cannot schedule on.
|
||||
if self._is_thread_local and not iscoroutinefunction(self._release):
|
||||
self._release_without_loop()
|
||||
return
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
# At interpreter exit daemon threads no longer run, so a loop running in one never runs the release.
|
||||
if (loop := self._async_context.loop) is None or loop.is_closed() or sys.is_finalizing():
|
||||
self._release_without_loop()
|
||||
return
|
||||
if not loop.is_running(): # pragma: no cover
|
||||
loop.run_until_complete(self.release(force=True))
|
||||
return
|
||||
# Off the deadlock registry, a new lock in this task waits for the scheduled release instead of failing.
|
||||
self._drop_registry_entry()
|
||||
# The stored loop runs in another thread, and only its own thread may create tasks on it.
|
||||
pending = asyncio.run_coroutine_threadsafe(self.release(force=True), loop)
|
||||
else:
|
||||
self._drop_registry_entry()
|
||||
pending = loop.create_task(self.release(force=True))
|
||||
# The loop keeps only a weak reference to a task, so hold one until the release finishes.
|
||||
_FINALIZER_RELEASES.add(pending)
|
||||
pending.add_done_callback(self._settle_finalizer_release)
|
||||
|
||||
def _settle_finalizer_release(self, pending: asyncio.Task[None] | futures.Future[None]) -> None:
|
||||
_FINALIZER_RELEASES.discard(pending)
|
||||
# Nothing awaits this release, so a failure or cancellation falls back to the sync release here.
|
||||
with contextlib.suppress(asyncio.CancelledError, futures.CancelledError):
|
||||
if pending.exception() is None:
|
||||
return
|
||||
with contextlib.suppress(Exception):
|
||||
self._release_without_loop()
|
||||
|
||||
def _release_without_loop(self) -> None:
|
||||
# No loop can run a release here, so call a sync backend directly as BaseFileLock.__del__ does; a coroutine
|
||||
# backend cannot run and keeps its lock.
|
||||
if iscoroutinefunction(self._release):
|
||||
return
|
||||
if self.is_locked:
|
||||
try:
|
||||
self._release_with_fork_tracking()
|
||||
finally:
|
||||
self._commit_release_if_released()
|
||||
# A thread-local lock also holds for the threads that ran its loops, and nothing else releases those now.
|
||||
self._release_thread_holds(exited_only=False)
|
||||
|
||||
|
||||
def _new_transition_gate() -> _AsyncTransitionGate:
|
||||
# Called per context rather than passed as the factory, so the lazy import resolves on first use, not at import.
|
||||
return _AsyncTransitionGate()
|
||||
|
||||
|
||||
@dataclass
|
||||
class AsyncFileLockContext(FileLockContext):
|
||||
"""A dataclass which holds the context for a ``BaseAsyncFileLock`` object."""
|
||||
|
||||
#: Whether run in executor
|
||||
run_in_executor: bool = True
|
||||
|
||||
#: The executor
|
||||
executor: futures.Executor | None = None
|
||||
|
||||
#: The loop
|
||||
loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
#: Serializes the tasks that transition this context. An acquire keeps it while it polls, so a thread-local lock
|
||||
#: needs one per thread. A thread waiting for the lock must not keep the thread that holds it from releasing.
|
||||
transition_gate: _AsyncTransitionGate = field(default_factory=_new_transition_gate)
|
||||
|
||||
|
||||
class AsyncThreadLocalFileContext(AsyncFileLockContext, ThreadLocalFileContext):
|
||||
"""A thread local version of the ``FileLockContext`` class."""
|
||||
|
||||
# threading.local runs __init__ again with these arguments the first time each thread touches the context.
|
||||
def __init__( # ruff:ignore[too-many-arguments] # mirrors the AsyncFileLockContext fields set at construction
|
||||
self,
|
||||
*,
|
||||
lock_file: str,
|
||||
timeout: float,
|
||||
mode: int,
|
||||
blocking: bool,
|
||||
poll_interval: float,
|
||||
lifetime: float | None = None,
|
||||
run_in_executor: bool = True,
|
||||
executor: futures.Executor | None = None,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
) -> None:
|
||||
self.current = AsyncFileLockContext(
|
||||
lock_file=lock_file,
|
||||
timeout=timeout,
|
||||
mode=mode,
|
||||
blocking=blocking,
|
||||
poll_interval=poll_interval,
|
||||
lifetime=lifetime,
|
||||
run_in_executor=run_in_executor,
|
||||
executor=executor,
|
||||
loop=loop,
|
||||
)
|
||||
|
||||
|
||||
for _field in fields(AsyncFileLockContext):
|
||||
setattr(AsyncThreadLocalFileContext, _field.name, _delegate_to_current(_field.name))
|
||||
|
||||
|
||||
class AsyncAcquireReturnProxy:
|
||||
"""A context-aware object that will release the lock file when exiting."""
|
||||
|
||||
def __init__(self, lock: BaseAsyncFileLock) -> None: # ruff:ignore[undocumented-public-init] # trivial release-on-exit proxy
|
||||
self.lock = lock
|
||||
|
||||
async def __aenter__(self) -> BaseAsyncFileLock: # ruff:ignore[undocumented-magic-method] # returns the wrapped lock
|
||||
return self.lock
|
||||
|
||||
async def __aexit__( # ruff:ignore[undocumented-magic-method] # releases the wrapped lock on exit
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
await self.lock._release_in_context(exc_value) # ruff:ignore[private-member-access] # releases the wrapped lock's context
|
||||
|
||||
|
||||
class AsyncSoftFileLock(SoftFileLock, BaseAsyncFileLock):
|
||||
"""Simply watches the existence of the lock file."""
|
||||
|
||||
_lifetime_replacements: tuple[str, str] | None = ("AsyncStrictSoftFileLock", "AsyncSoftFileLease")
|
||||
|
||||
|
||||
class AsyncStrictSoftFileLock(StrictSoftFileLock, BaseAsyncFileLock):
|
||||
"""Run strict owner-claim locking without blocking the event loop."""
|
||||
|
||||
|
||||
class AsyncSoftFileLease(SoftFileLease, BaseAsyncFileLock):
|
||||
"""Existence lock whose claim expires, so a peer may take it while the previous holder still runs."""
|
||||
|
||||
|
||||
class AsyncUnixFileLock(UnixFileLock, BaseAsyncFileLock):
|
||||
"""Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems."""
|
||||
|
||||
|
||||
class AsyncWindowsFileLock(WindowsFileLock, BaseAsyncFileLock):
|
||||
"""Uses ``LockFileEx`` to hard lock a byte range of the lock file on Windows systems."""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AsyncAcquireReturnProxy",
|
||||
"AsyncSoftFileLease",
|
||||
"AsyncSoftFileLock",
|
||||
"AsyncStrictSoftFileLock",
|
||||
"AsyncUnixFileLock",
|
||||
"AsyncWindowsFileLock",
|
||||
"BaseAsyncFileLock",
|
||||
]
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
# Hide the sync bases and helpers imported at top level.
|
||||
return list(__all__)
|
||||
Whitespace-only changes.
@@ -0,0 +1,24 @@
|
||||
# file generated by vcs-versioning
|
||||
# don't change, don't track in version control
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"__version_tuple__",
|
||||
"version",
|
||||
"version_tuple",
|
||||
"__commit_id__",
|
||||
"commit_id",
|
||||
]
|
||||
|
||||
version: str
|
||||
__version__: str
|
||||
__version_tuple__: tuple[int | str, ...]
|
||||
version_tuple: tuple[int | str, ...]
|
||||
commit_id: str | None
|
||||
__commit_id__: str | None
|
||||
|
||||
__version__ = version = '4.2.4'
|
||||
__version_tuple__ = version_tuple = (4, 2, 4)
|
||||
|
||||
__commit_id__ = commit_id = None
|
||||
Reference in new issue
Block a user