Skip to content

file_hashes

file_hashes #

The hash cache: leaf hashes remembered by path and stat, in their own SQLite file.

The cache lives beside the index rather than in it, so hashing a large tree never holds the index's write lock. It only saves re-reads: a file of another layout is emptied and recreated, and a write still locked out after the busy timeout is skipped.

BUSY_TIMEOUT_SECONDS = 5.0 module-attribute #

How long a cache write waits on another writer before it is skipped.

FileHashes #

A HashCache over the file_hashes table of its own file.

Source code in capturegraph-lib/capturegraph/scratch/index/file_hashes.py
class FileHashes:
    """A ``HashCache`` over the ``file_hashes`` table of its own file."""

    def __init__(self, path: Path) -> None:
        """Open or create the cache file at ``path``, emptying one of another layout."""
        self.path = path
        self._lock = threading.Lock()
        self._connection = connect(path, BUSY_TIMEOUT_SECONDS)
        (version,) = self._connection.execute("PRAGMA user_version").fetchone()
        if version != LAYOUT_VERSION:
            self._connection.execute("DROP TABLE IF EXISTS file_hashes")
            self._connection.execute(CREATE)
            self._connection.execute(f"PRAGMA user_version = {LAYOUT_VERSION}")

    def lookup(self, path: Path, stat: Stat) -> ContentHash | None:
        """The remembered hash of ``path`` at ``stat``, if any."""
        with self._lock:
            row = self._connection.execute(
                "SELECT hash FROM file_hashes WHERE path = ? AND mtime_ns = ? AND size_bytes = ? "
                "AND inode = ? AND is_dir = ?",
                (str(path), stat.mtime_ns, stat.size, stat.inode, int(stat.is_dir)),
            ).fetchone()
        return ContentHash.from_hex(row[0]) if row else None

    def store_all(self, learned: list[Learned]) -> None:
        """Remember every ``(path, stat, hash)`` in one transaction."""
        self._write(
            "INSERT OR REPLACE INTO file_hashes "
            "(path, mtime_ns, size_bytes, inode, is_dir, hash) VALUES (?, ?, ?, ?, ?, ?)",
            [
                (str(path), stat.mtime_ns, stat.size, stat.inode, int(stat.is_dir), hash.hex)
                for path, stat, hash in learned
            ],
        )

    def move(self, source: Path, destination: Path) -> None:
        """Re-key every remembered path under ``source`` to live under ``destination``."""
        start, end = _subtree(source)
        self._write(
            "UPDATE OR REPLACE file_hashes SET path = ? || substr(path, ?) "
            "WHERE path >= ? AND path < ?",
            [(f"{destination}/", len(start) + 1, start, end)],
        )

    def forget(self, source: Path) -> None:
        """Drop every remembered path under ``source``."""
        start, end = _subtree(source)
        self._write("DELETE FROM file_hashes WHERE path >= ? AND path < ?", [(start, end)])

    def close(self) -> None:
        """Close the connection."""
        with self._lock:
            self._connection.close()

    def _write(self, statement: str, rows: list[Row]) -> None:
        with self._lock:
            try:
                self._connection.execute("BEGIN IMMEDIATE")
            except sqlite3.OperationalError as error:
                logger.warning(f"{self.path}: {len(rows)} hash cache writes skipped: {error}")
                return
            try:
                self._connection.executemany(statement, rows)
            except BaseException:
                self._connection.execute("ROLLBACK")
                raise
            self._connection.execute("COMMIT")

__init__(path) #

Open or create the cache file at path, emptying one of another layout.

Source code in capturegraph-lib/capturegraph/scratch/index/file_hashes.py
def __init__(self, path: Path) -> None:
    """Open or create the cache file at ``path``, emptying one of another layout."""
    self.path = path
    self._lock = threading.Lock()
    self._connection = connect(path, BUSY_TIMEOUT_SECONDS)
    (version,) = self._connection.execute("PRAGMA user_version").fetchone()
    if version != LAYOUT_VERSION:
        self._connection.execute("DROP TABLE IF EXISTS file_hashes")
        self._connection.execute(CREATE)
        self._connection.execute(f"PRAGMA user_version = {LAYOUT_VERSION}")

close() #

Close the connection.

Source code in capturegraph-lib/capturegraph/scratch/index/file_hashes.py
def close(self) -> None:
    """Close the connection."""
    with self._lock:
        self._connection.close()

forget(source) #

Drop every remembered path under source.

Source code in capturegraph-lib/capturegraph/scratch/index/file_hashes.py
def forget(self, source: Path) -> None:
    """Drop every remembered path under ``source``."""
    start, end = _subtree(source)
    self._write("DELETE FROM file_hashes WHERE path >= ? AND path < ?", [(start, end)])

lookup(path, stat) #

The remembered hash of path at stat, if any.

Source code in capturegraph-lib/capturegraph/scratch/index/file_hashes.py
def lookup(self, path: Path, stat: Stat) -> ContentHash | None:
    """The remembered hash of ``path`` at ``stat``, if any."""
    with self._lock:
        row = self._connection.execute(
            "SELECT hash FROM file_hashes WHERE path = ? AND mtime_ns = ? AND size_bytes = ? "
            "AND inode = ? AND is_dir = ?",
            (str(path), stat.mtime_ns, stat.size, stat.inode, int(stat.is_dir)),
        ).fetchone()
    return ContentHash.from_hex(row[0]) if row else None

move(source, destination) #

Re-key every remembered path under source to live under destination.

Source code in capturegraph-lib/capturegraph/scratch/index/file_hashes.py
def move(self, source: Path, destination: Path) -> None:
    """Re-key every remembered path under ``source`` to live under ``destination``."""
    start, end = _subtree(source)
    self._write(
        "UPDATE OR REPLACE file_hashes SET path = ? || substr(path, ?) "
        "WHERE path >= ? AND path < ?",
        [(f"{destination}/", len(start) + 1, start, end)],
    )

store_all(learned) #

Remember every (path, stat, hash) in one transaction.

Source code in capturegraph-lib/capturegraph/scratch/index/file_hashes.py
def store_all(self, learned: list[Learned]) -> None:
    """Remember every ``(path, stat, hash)`` in one transaction."""
    self._write(
        "INSERT OR REPLACE INTO file_hashes "
        "(path, mtime_ns, size_bytes, inode, is_dir, hash) VALUES (?, ?, ?, ?, ?, ?)",
        [
            (str(path), stat.mtime_ns, stat.size, stat.inode, int(stat.is_dir), hash.hex)
            for path, stat, hash in learned
        ],
    )