Skip to content

database

database #

Database — the one SQLite file behind the index, opened once per scratch handle.

Every process sharing the scratch writes through the same file, one writer at a time, so a write transaction never walks the filesystem: a waiter queues behind it for at most BUSY_TIMEOUT_SECONDS. On a local disk the file is in WAL mode, where reads take a snapshot and never wait. On a network filesystem (fs.network_filesystem) it is in rollback-journal mode, which needs only file locks, so machines sharing the mount cannot corrupt it; there a read waits out a commit and a commit waits out the reads.

The file carries a layout version. An older layout with a migration to the next one is upgraded in place, step by step, and the cache survives; any other version is rebuilt empty, and the scratch that opened it discards its objects.

MIGRATIONS = {10: ('DROP TABLE running',), 11: ('DROP TABLE IF EXISTS steps',), 12: ('DROP TABLE file_hashes',)} module-attribute #

The statements taking a layout to the next version, keyed by the version they start from.

Database #

The index connection; every write happens inside one transaction.

A transaction opened while the same thread already has one open joins it as a savepoint, so an explicit call made mid-transaction never nests a BEGIN.

Source code in capturegraph-lib/capturegraph/scratch/index/database.py
class Database:
    """The index connection; every write happens inside one ``transaction``.

    A transaction opened while the same thread already has one open joins it as
    a savepoint, so an explicit call made mid-transaction never nests a ``BEGIN``.
    """

    def __init__(self, path: Path) -> None:
        """Open or create the index at ``path``, migrating or rebuilding an older layout.

        ``rebuilt`` says whether the index was emptied because its layout could
        not be migrated; the objects it indexed are then orphans to discard.
        """
        self.path = path
        self._lock = threading.RLock()
        self._connection = connect(path, BUSY_TIMEOUT_SECONDS)
        self.rebuilt = self._initialize()

    def _initialize(self) -> bool:
        with self.read() as db:
            (version,) = db.execute("PRAGMA user_version").fetchone()
        if version == SCHEMA_VERSION:
            return False
        with self.transaction() as db:
            (version,) = db.execute("PRAGMA user_version").fetchone()
            if version == SCHEMA_VERSION:
                return False
            tables = _tables(db)
            if version == 0 and not tables:
                _create(db)
                return False
            while version in MIGRATIONS and version < SCHEMA_VERSION:
                for statement in MIGRATIONS[version]:
                    db.execute(statement)
                version += 1
                db.execute(f"PRAGMA user_version = {version}")
            if version == SCHEMA_VERSION:
                logger.info(f"{self.path}: index migrated to layout {SCHEMA_VERSION}")
                return False
            logger.warning(
                f"{self.path}: layout {version} cannot be migrated to {SCHEMA_VERSION}; "
                f"rebuilding the index empty"
            )
            for table in tables:
                db.execute(f"DROP TABLE {table}")
            _create(db)
            return True

    @contextmanager
    def transaction(self) -> Iterator[sqlite3.Connection]:
        """One immediate (write) transaction, serialized against every other writer.

        Raises:
            ScratchError: If another writer still holds the index after the busy timeout.
        """
        with self._lock:
            if self._connection.in_transaction:
                yield from self._savepoint()
                return
            try:
                self._connection.execute("BEGIN IMMEDIATE")
            except sqlite3.OperationalError as error:
                raise ScratchError(
                    f"{self.path}: still locked by another writer after "
                    f"{BUSY_TIMEOUT_SECONDS:.0f} s ({error})"
                ) from error
            yield from self._committed()

    @contextmanager
    def read(self) -> Iterator[sqlite3.Connection]:
        """One consistent snapshot of the index, taken without waiting on any writer."""
        with self._lock:
            if self._connection.in_transaction:
                yield self._connection
                return
            self._connection.execute("BEGIN DEFERRED")
            yield from self._committed()

    def _savepoint(self) -> Iterator[sqlite3.Connection]:
        self._connection.execute("SAVEPOINT nested")
        try:
            yield self._connection
        except BaseException:
            self._connection.execute("ROLLBACK TO nested")
            self._connection.execute("RELEASE nested")
            raise
        self._connection.execute("RELEASE nested")

    def _committed(self) -> Iterator[sqlite3.Connection]:
        try:
            yield self._connection
        except BaseException:
            self._connection.execute("ROLLBACK")
            raise
        self._connection.execute("COMMIT")

    def close(self) -> None:
        """Close the connection; the index file stays."""
        with self._lock:
            self._connection.close()

__init__(path) #

Open or create the index at path, migrating or rebuilding an older layout.

rebuilt says whether the index was emptied because its layout could not be migrated; the objects it indexed are then orphans to discard.

Source code in capturegraph-lib/capturegraph/scratch/index/database.py
def __init__(self, path: Path) -> None:
    """Open or create the index at ``path``, migrating or rebuilding an older layout.

    ``rebuilt`` says whether the index was emptied because its layout could
    not be migrated; the objects it indexed are then orphans to discard.
    """
    self.path = path
    self._lock = threading.RLock()
    self._connection = connect(path, BUSY_TIMEOUT_SECONDS)
    self.rebuilt = self._initialize()

close() #

Close the connection; the index file stays.

Source code in capturegraph-lib/capturegraph/scratch/index/database.py
def close(self) -> None:
    """Close the connection; the index file stays."""
    with self._lock:
        self._connection.close()

read() #

One consistent snapshot of the index, taken without waiting on any writer.

Source code in capturegraph-lib/capturegraph/scratch/index/database.py
@contextmanager
def read(self) -> Iterator[sqlite3.Connection]:
    """One consistent snapshot of the index, taken without waiting on any writer."""
    with self._lock:
        if self._connection.in_transaction:
            yield self._connection
            return
        self._connection.execute("BEGIN DEFERRED")
        yield from self._committed()

transaction() #

One immediate (write) transaction, serialized against every other writer.

Raises:

Type Description
ScratchError

If another writer still holds the index after the busy timeout.

Source code in capturegraph-lib/capturegraph/scratch/index/database.py
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
    """One immediate (write) transaction, serialized against every other writer.

    Raises:
        ScratchError: If another writer still holds the index after the busy timeout.
    """
    with self._lock:
        if self._connection.in_transaction:
            yield from self._savepoint()
            return
        try:
            self._connection.execute("BEGIN IMMEDIATE")
        except sqlite3.OperationalError as error:
            raise ScratchError(
                f"{self.path}: still locked by another writer after "
                f"{BUSY_TIMEOUT_SECONDS:.0f} s ({error})"
            ) from error
        yield from self._committed()

connect(path, busy_timeout_seconds) #

A connection to the SQLite file at path, usable from any thread.

The file is put in WAL mode on a local disk and in rollback-journal mode on a network filesystem, converting one left in the other mode.

Parameters:

Name Type Description Default
path Path

The database file, created if absent.

required
busy_timeout_seconds float

How long a statement waits on another writer before failing with database is locked.

required

Raises:

Type Description
ScratchError

If the file is in the other mode and another process has it open, so it cannot be converted.

Source code in capturegraph-lib/capturegraph/scratch/index/database.py
def connect(
    path: Path,
    busy_timeout_seconds: float,
) -> sqlite3.Connection:
    """A connection to the SQLite file at ``path``, usable from any thread.

    The file is put in WAL mode on a local disk and in rollback-journal mode
    on a network filesystem, converting one left in the other mode.

    Args:
        path: The database file, created if absent.
        busy_timeout_seconds: How long a statement waits on another writer
            before failing with ``database is locked``.

    Raises:
        ScratchError: If the file is in the other mode and another process
            has it open, so it cannot be converted.
    """
    connection = sqlite3.connect(
        path,
        timeout=busy_timeout_seconds,
        isolation_level=None,
        check_same_thread=False,
    )
    network = network_filesystem(path.parent)
    wanted = "wal" if network is None else "delete"
    try:
        (mode,) = connection.execute(f"PRAGMA journal_mode={wanted}").fetchone()
    except sqlite3.OperationalError as error:
        mode = str(error)
    if mode != wanted:
        connection.close()
        raise ScratchError(
            f"{path}: cannot switch to {wanted} journal mode, the safe one on "
            f"{network or 'a local disk'} ({mode}); it converts once no other process "
            f"has the scratch open"
        )
    connection.execute(f"PRAGMA synchronous={'NORMAL' if network is None else 'FULL'}")
    if network is not None:
        logger.info(f"{path}: on {network}, opened in rollback-journal mode")
    return connection