Skip to content

database

database #

Database — the one SQLite file behind the index, opened once per scratch handle.

The file carries a layout version. An older layout with a migration to the next one is upgraded in place, step by step, and the cache survives; any other version is rebuilt empty, and the scratch that opened it discards its objects.

MIGRATIONS = {10: ('DROP TABLE running', steps.CREATE)} module-attribute #

The statements taking a layout to the next version, keyed by the version they start from.

Database #

The index connection; every write happens inside one transaction.

A transaction opened while the same thread already has one open joins it as a savepoint, so an explicit call made mid-transaction never nests a BEGIN.

Source code in capturegraph-lib/capturegraph/recipes/scratch/index/database.py 
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
class Database:
    """The index connection; every write happens inside one ``transaction``.

    A transaction opened while the same thread already has one open joins it as
    a savepoint, so an explicit call made mid-transaction never nests a ``BEGIN``.
    """

    def __init__(self, path: Path) -> None:
        """Open or create the index at ``path``, migrating or rebuilding an older layout.

        ``rebuilt`` says whether the index was emptied because its layout could
        not be migrated; the objects it indexed are then orphans to discard.
        """
        self.path = path
        self._lock = threading.RLock()
        self._connection = connect(path)
        self.rebuilt = self._initialize()

    def _initialize(self) -> bool:
        with self.transaction() as db:
            (version,) = db.execute("PRAGMA user_version").fetchone()
            if version == SCHEMA_VERSION:
                return False
            tables = _tables(db)
            if version == 0 and not tables:
                _create(db)
                return False
            while version in MIGRATIONS and version < SCHEMA_VERSION:
                for statement in MIGRATIONS[version]:
                    db.execute(statement)
                version += 1
                db.execute(f"PRAGMA user_version = {version}")
            if version == SCHEMA_VERSION:
                logger.info(f"{self.path}: index migrated to layout {SCHEMA_VERSION}")
                return False
            logger.warning(
                f"{self.path}: layout {version} cannot be migrated to {SCHEMA_VERSION}; "
                f"rebuilding the index empty"
            )
            for table in tables:
                db.execute(f"DROP TABLE {table}")
            _create(db)
            return True

    @contextmanager
    def transaction(self) -> Iterator[sqlite3.Connection]:
        """One immediate transaction, serialized against this process's other writers."""
        with self._lock:
            if self._connection.in_transaction:
                self._connection.execute("SAVEPOINT nested")
                try:
                    yield self._connection
                except BaseException:
                    self._connection.execute("ROLLBACK TO nested")
                    self._connection.execute("RELEASE nested")
                    raise
                self._connection.execute("RELEASE nested")
                return
            self._connection.execute("BEGIN IMMEDIATE")
            try:
                yield self._connection
            except BaseException:
                self._connection.execute("ROLLBACK")
                raise
            self._connection.execute("COMMIT")

    def close(self) -> None:
        """Close the connection; the index file stays."""
        with self._lock:
            self._connection.close()

__init__(path) #

Open or create the index at path, migrating or rebuilding an older layout.

rebuilt says whether the index was emptied because its layout could not be migrated; the objects it indexed are then orphans to discard.

Source code in capturegraph-lib/capturegraph/recipes/scratch/index/database.py 
52
53
54
55
56
57
58
59
60
61
def __init__(self, path: Path) -> None:
    """Open or create the index at ``path``, migrating or rebuilding an older layout.

    ``rebuilt`` says whether the index was emptied because its layout could
    not be migrated; the objects it indexed are then orphans to discard.
    """
    self.path = path
    self._lock = threading.RLock()
    self._connection = connect(path)
    self.rebuilt = self._initialize()

close() #

Close the connection; the index file stays.

Source code in capturegraph-lib/capturegraph/recipes/scratch/index/database.py 
111
112
113
114
def close(self) -> None:
    """Close the connection; the index file stays."""
    with self._lock:
        self._connection.close()

transaction() #

One immediate transaction, serialized against this process's other writers.

Source code in capturegraph-lib/capturegraph/recipes/scratch/index/database.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
@contextmanager
def transaction(self) -> Iterator[sqlite3.Connection]:
    """One immediate transaction, serialized against this process's other writers."""
    with self._lock:
        if self._connection.in_transaction:
            self._connection.execute("SAVEPOINT nested")
            try:
                yield self._connection
            except BaseException:
                self._connection.execute("ROLLBACK TO nested")
                self._connection.execute("RELEASE nested")
                raise
            self._connection.execute("RELEASE nested")
            return
        self._connection.execute("BEGIN IMMEDIATE")
        try:
            yield self._connection
        except BaseException:
            self._connection.execute("ROLLBACK")
            raise
        self._connection.execute("COMMIT")

connect(path) #

A WAL-mode connection to the index file at path, usable from any thread.

Source code in capturegraph-lib/capturegraph/recipes/scratch/index/database.py 
32
33
34
35
36
37
38
39
40
41
42
def connect(path: Path) -> sqlite3.Connection:
    """A WAL-mode connection to the index file at ``path``, usable from any thread."""
    connection = sqlite3.connect(
        path,
        timeout=BUSY_TIMEOUT_SECONDS,
        isolation_level=None,
        check_same_thread=False,
    )
    connection.execute("PRAGMA journal_mode=WAL")
    connection.execute("PRAGMA synchronous=NORMAL")
    return connection