Database — the one SQLite file behind the index, opened once per scratch handle.
Every process sharing the scratch writes through the same file, one writer at a
time, so a write transaction never walks the filesystem: a waiter queues behind
it for at most BUSY_TIMEOUT_SECONDS. On a local disk the file is in WAL
mode, where reads take a snapshot and never wait. On a network filesystem
(fs.network_filesystem) it is in rollback-journal mode, which needs only
file locks, so machines sharing the mount cannot corrupt it; there a read
waits out a commit and a commit waits out the reads.
The file carries a layout version. An older layout with a migration to the
next one is upgraded in place, step by step, and the cache survives; any other
version is rebuilt empty, and the scratch that opened it discards its objects.
MIGRATIONS = {10: ('DROP TABLE running',), 11: ('DROP TABLE IF EXISTS steps',), 12: ('DROP TABLE file_hashes',)}
module-attribute
The statements taking a layout to the next version, keyed by the version they start from.
Database
The index connection; every write happens inside one transaction.
A transaction opened while the same thread already has one open joins it as
a savepoint, so an explicit call made mid-transaction never nests a BEGIN.
Source code in capturegraph-lib/capturegraph/scratch/index/database.py
| class Database:
"""The index connection; every write happens inside one ``transaction``.
A transaction opened while the same thread already has one open joins it as
a savepoint, so an explicit call made mid-transaction never nests a ``BEGIN``.
"""
def __init__(self, path: Path) -> None:
"""Open or create the index at ``path``, migrating or rebuilding an older layout.
``rebuilt`` says whether the index was emptied because its layout could
not be migrated; the objects it indexed are then orphans to discard.
"""
self.path = path
self._lock = threading.RLock()
self._connection = connect(path, BUSY_TIMEOUT_SECONDS)
self.rebuilt = self._initialize()
def _initialize(self) -> bool:
with self.read() as db:
(version,) = db.execute("PRAGMA user_version").fetchone()
if version == SCHEMA_VERSION:
return False
with self.transaction() as db:
(version,) = db.execute("PRAGMA user_version").fetchone()
if version == SCHEMA_VERSION:
return False
tables = _tables(db)
if version == 0 and not tables:
_create(db)
return False
while version in MIGRATIONS and version < SCHEMA_VERSION:
for statement in MIGRATIONS[version]:
db.execute(statement)
version += 1
db.execute(f"PRAGMA user_version = {version}")
if version == SCHEMA_VERSION:
logger.info(f"{self.path}: index migrated to layout {SCHEMA_VERSION}")
return False
logger.warning(
f"{self.path}: layout {version} cannot be migrated to {SCHEMA_VERSION}; "
f"rebuilding the index empty"
)
for table in tables:
db.execute(f"DROP TABLE {table}")
_create(db)
return True
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
"""One immediate (write) transaction, serialized against every other writer.
Raises:
ScratchError: If another writer still holds the index after the busy timeout.
"""
with self._lock:
if self._connection.in_transaction:
yield from self._savepoint()
return
try:
self._connection.execute("BEGIN IMMEDIATE")
except sqlite3.OperationalError as error:
raise ScratchError(
f"{self.path}: still locked by another writer after "
f"{BUSY_TIMEOUT_SECONDS:.0f} s ({error})"
) from error
yield from self._committed()
@contextmanager
def read(self) -> Iterator[sqlite3.Connection]:
"""One consistent snapshot of the index, taken without waiting on any writer."""
with self._lock:
if self._connection.in_transaction:
yield self._connection
return
self._connection.execute("BEGIN DEFERRED")
yield from self._committed()
def _savepoint(self) -> Iterator[sqlite3.Connection]:
self._connection.execute("SAVEPOINT nested")
try:
yield self._connection
except BaseException:
self._connection.execute("ROLLBACK TO nested")
self._connection.execute("RELEASE nested")
raise
self._connection.execute("RELEASE nested")
def _committed(self) -> Iterator[sqlite3.Connection]:
try:
yield self._connection
except BaseException:
self._connection.execute("ROLLBACK")
raise
self._connection.execute("COMMIT")
def close(self) -> None:
"""Close the connection; the index file stays."""
with self._lock:
self._connection.close()
|
__init__(path)
Open or create the index at path, migrating or rebuilding an older layout.
rebuilt says whether the index was emptied because its layout could
not be migrated; the objects it indexed are then orphans to discard.
Source code in capturegraph-lib/capturegraph/scratch/index/database.py
| def __init__(self, path: Path) -> None:
"""Open or create the index at ``path``, migrating or rebuilding an older layout.
``rebuilt`` says whether the index was emptied because its layout could
not be migrated; the objects it indexed are then orphans to discard.
"""
self.path = path
self._lock = threading.RLock()
self._connection = connect(path, BUSY_TIMEOUT_SECONDS)
self.rebuilt = self._initialize()
|
close()
Close the connection; the index file stays.
Source code in capturegraph-lib/capturegraph/scratch/index/database.py
| def close(self) -> None:
"""Close the connection; the index file stays."""
with self._lock:
self._connection.close()
|
read()
One consistent snapshot of the index, taken without waiting on any writer.
Source code in capturegraph-lib/capturegraph/scratch/index/database.py
| @contextmanager
def read(self) -> Iterator[sqlite3.Connection]:
"""One consistent snapshot of the index, taken without waiting on any writer."""
with self._lock:
if self._connection.in_transaction:
yield self._connection
return
self._connection.execute("BEGIN DEFERRED")
yield from self._committed()
|
transaction()
One immediate (write) transaction, serialized against every other writer.
Raises:
| Type |
Description |
ScratchError
|
If another writer still holds the index after the busy timeout.
|
Source code in capturegraph-lib/capturegraph/scratch/index/database.py
| @contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
"""One immediate (write) transaction, serialized against every other writer.
Raises:
ScratchError: If another writer still holds the index after the busy timeout.
"""
with self._lock:
if self._connection.in_transaction:
yield from self._savepoint()
return
try:
self._connection.execute("BEGIN IMMEDIATE")
except sqlite3.OperationalError as error:
raise ScratchError(
f"{self.path}: still locked by another writer after "
f"{BUSY_TIMEOUT_SECONDS:.0f} s ({error})"
) from error
yield from self._committed()
|
connect(path, busy_timeout_seconds)
A connection to the SQLite file at path, usable from any thread.
The file is put in WAL mode on a local disk and in rollback-journal mode
on a network filesystem, converting one left in the other mode.
Parameters:
| Name |
Type |
Description |
Default |
path
|
Path
|
The database file, created if absent.
|
required
|
busy_timeout_seconds
|
float
|
How long a statement waits on another writer
before failing with database is locked.
|
required
|
Raises:
| Type |
Description |
ScratchError
|
If the file is in the other mode and another process
has it open, so it cannot be converted.
|
Source code in capturegraph-lib/capturegraph/scratch/index/database.py
| def connect(
path: Path,
busy_timeout_seconds: float,
) -> sqlite3.Connection:
"""A connection to the SQLite file at ``path``, usable from any thread.
The file is put in WAL mode on a local disk and in rollback-journal mode
on a network filesystem, converting one left in the other mode.
Args:
path: The database file, created if absent.
busy_timeout_seconds: How long a statement waits on another writer
before failing with ``database is locked``.
Raises:
ScratchError: If the file is in the other mode and another process
has it open, so it cannot be converted.
"""
connection = sqlite3.connect(
path,
timeout=busy_timeout_seconds,
isolation_level=None,
check_same_thread=False,
)
network = network_filesystem(path.parent)
wanted = "wal" if network is None else "delete"
try:
(mode,) = connection.execute(f"PRAGMA journal_mode={wanted}").fetchone()
except sqlite3.OperationalError as error:
mode = str(error)
if mode != wanted:
connection.close()
raise ScratchError(
f"{path}: cannot switch to {wanted} journal mode, the safe one on "
f"{network or 'a local disk'} ({mode}); it converts once no other process "
f"has the scratch open"
)
connection.execute(f"PRAGMA synchronous={'NORMAL' if network is None else 'FULL'}")
if network is not None:
logger.info(f"{path}: on {network}, opened in rollback-journal mode")
return connection
|