Skip to content

trainer

trainer #

The splat trainer, run by cgr.splats.train inside the GPU interpreter.

Only torch, nerfstudio and numpy are imported here, never capturegraph: the GPU environment is a separate installation, so the frames arrive as an explicit manifest rather than as a stored array. The run writes progress.json twice a second and the trained gaussians as the standard 3DGS PLY cgr.Splat documents, each colour as its degree-zero harmonic, which the SplatViewer reads. The gaussians are written in the frame of the poses given: when the cameras were refined, carried back by the similarity realigning the refined cameras onto the given ones.

python trainer.py --frames FRAMES.json --output O --iterations N \\
    --seed S.ply --splat OUT.ply --progress P.json \\
    --depth-weight W [--optimize-cameras]

FRAMES.json is a list, in training order, of {"image", "pose", "depth", "confidence"}: the image's path, the pose as capturegraph's Pose JSON (position, quaternion and intrinsics at the image's size, in the OpenGL camera-to-world convention), and the paths of the frame's measured depth in metres and its per-pixel confidence as .npy planes at any resolution, or null where it measured none. With a positive --depth-weight the depth-supervised splatfacto depth_model defines trains, its depth term the confidence-weighted LogL1 distance between the rendered and the measured depth.

CONFIDENCE_OF = {} module-attribute #

Each staged depth plane's confidence plane, filled by stage_dataset.

DEPTH_CONFIDENCE_FLOOR = 0.1 module-attribute #

Loss weight of a measured zero-confidence pixel; full confidence weighs 1.

DEPTH_TOLERANCE_METERS = 0.05 module-attribute #

The least measured depth treated as a return.

HARMONIC_DEGREE_ZERO = 0.28209479177387814 module-attribute #

The degree-zero spherical harmonic: a PLY stores colour c as (c - 0.5) / this.

Gaussians dataclass #

N trained gaussians, as the standard 3DGS PLY stores them.

Attributes:

Name Type Description
positions ndarray

Their centres, N × 3.

colours ndarray

Their RGB in [0, 1], N × 3.

opacities ndarray

Their opacity logits, N.

scales ndarray

Their log-scales, N × 3.

quaternions ndarray

Their rotations, w first, N × 4.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
@dataclasses.dataclass(frozen=True)
class Gaussians:
    """N trained gaussians, as the standard 3DGS PLY stores them.

    Attributes:
        positions: Their centres, N × 3.
        colours: Their RGB in [0, 1], N × 3.
        opacities: Their opacity logits, N.
        scales: Their log-scales, N × 3.
        quaternions: Their rotations, w first, N × 4.
    """

    positions: np.ndarray
    colours: np.ndarray
    opacities: np.ndarray
    scales: np.ndarray
    quaternions: np.ndarray

    def carried(self, similarity: Similarity) -> Gaussians:
        """These gaussians moved, turned and grown by ``similarity``."""
        return dataclasses.replace(
            self,
            positions=similarity.scale * self.positions @ similarity.rotation.T
            + similarity.translation,
            scales=self.scales + np.log(similarity.scale),
            quaternions=hamilton(quaternion(similarity.rotation), self.quaternions),
        )

carried(similarity) #

These gaussians moved, turned and grown by similarity.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def carried(self, similarity: Similarity) -> Gaussians:
    """These gaussians moved, turned and grown by ``similarity``."""
    return dataclasses.replace(
        self,
        positions=similarity.scale * self.positions @ similarity.rotation.T
        + similarity.translation,
        scales=self.scales + np.log(similarity.scale),
        quaternions=hamilton(quaternion(similarity.rotation), self.quaternions),
    )

Similarity dataclass #

The map x -> scale * rotation @ x + translation.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
@dataclasses.dataclass(frozen=True)
class Similarity:
    """The map ``x -> scale * rotation @ x + translation``."""

    scale: float
    rotation: np.ndarray
    translation: np.ndarray

    @classmethod
    def realigning(cls, given: np.ndarray, refined: np.ndarray) -> Similarity:
        """The similarity carrying ``refined`` cameras back onto ``given`` ones.

        Its rotation best turns the refined orientations into the given ones,
        so it holds where the centres lie along a line, as a corridor walk's
        do; its scale and translation then best carry the refined centres
        onto the given ones.

        Args:
            given: The cameras as trained from, N × 4 × 4 camera-to-world.
            refined: The same cameras after refinement, N × 4 × 4.
        """
        rotation = nearest_rotation(
            np.einsum("nij,nkj->ik", given[:, :3, :3], refined[:, :3, :3]),
        )
        source = refined[:, :3, 3] @ rotation.T
        target = given[:, :3, 3]
        source_offsets = source - source.mean(axis=0)
        target_offsets = target - target.mean(axis=0)
        scale = float(np.sum(source_offsets * target_offsets) / np.sum(source_offsets**2))
        return cls(
            scale=scale,
            rotation=rotation,
            translation=target.mean(axis=0) - scale * source.mean(axis=0),
        )

    def cameras(self, matrices: np.ndarray) -> np.ndarray:
        """N × 4 × 4 camera-to-world ``matrices`` carried by this similarity."""
        carried = matrices.copy()
        carried[:, :3, :3] = self.rotation @ matrices[:, :3, :3]
        carried[:, :3, 3] = self.scale * matrices[:, :3, 3] @ self.rotation.T + self.translation
        return carried

cameras(matrices) #

N × 4 × 4 camera-to-world matrices carried by this similarity.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def cameras(self, matrices: np.ndarray) -> np.ndarray:
    """N × 4 × 4 camera-to-world ``matrices`` carried by this similarity."""
    carried = matrices.copy()
    carried[:, :3, :3] = self.rotation @ matrices[:, :3, :3]
    carried[:, :3, 3] = self.scale * matrices[:, :3, 3] @ self.rotation.T + self.translation
    return carried

realigning(given, refined) classmethod #

The similarity carrying refined cameras back onto given ones.

Its rotation best turns the refined orientations into the given ones, so it holds where the centres lie along a line, as a corridor walk's do; its scale and translation then best carry the refined centres onto the given ones.

Parameters:

Name Type Description Default
given ndarray

The cameras as trained from, N × 4 × 4 camera-to-world.

required
refined ndarray

The same cameras after refinement, N × 4 × 4.

required
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
@classmethod
def realigning(cls, given: np.ndarray, refined: np.ndarray) -> Similarity:
    """The similarity carrying ``refined`` cameras back onto ``given`` ones.

    Its rotation best turns the refined orientations into the given ones,
    so it holds where the centres lie along a line, as a corridor walk's
    do; its scale and translation then best carry the refined centres
    onto the given ones.

    Args:
        given: The cameras as trained from, N × 4 × 4 camera-to-world.
        refined: The same cameras after refinement, N × 4 × 4.
    """
    rotation = nearest_rotation(
        np.einsum("nij,nkj->ik", given[:, :3, :3], refined[:, :3, :3]),
    )
    source = refined[:, :3, 3] @ rotation.T
    target = given[:, :3, 3]
    source_offsets = source - source.mean(axis=0)
    target_offsets = target - target.mean(axis=0)
    scale = float(np.sum(source_offsets * target_offsets) / np.sum(source_offsets**2))
    return cls(
        scale=scale,
        rotation=rotation,
        translation=target.mean(axis=0) - scale * source.mean(axis=0),
    )

reporting #

Writes {"step", "total", "loss", "gaussians"} to progress while the block trains.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
class reporting:
    """Writes ``{"step", "total", "loss", "gaussians"}`` to ``progress`` while the block trains."""

    def __init__(self, trainer: Trainer, total: int, progress: Path) -> None:
        """Report ``trainer``'s run of ``total`` steps into ``progress``."""
        self._trainer = trainer
        self._total = total
        self._progress = progress
        self._loss: float | None = None
        self._stop = threading.Event()
        self._thread = threading.Thread(target=self._run, daemon=True)

    def __enter__(self) -> reporting:
        """Start reporting; every training iteration records its loss."""
        iterate = self._trainer.train_iteration

        def recorded(step: int) -> tuple[object, dict[str, object], dict[str, object]]:
            loss, loss_dict, metrics = iterate(step)
            self._loss = float(loss)
            return loss, loss_dict, metrics

        self._trainer.train_iteration = recorded
        self._thread.start()
        return self

    def __exit__(self, *_: object) -> None:
        """Stop reporting and write the final count."""
        self._stop.set()
        self._thread.join()
        self._write(self._total)

    def _run(self) -> None:
        while not self._stop.wait(REPORT_SECONDS):
            self._write(int(getattr(self._trainer, "step", 0)))

    def _write(self, step: int) -> None:
        payload = {
            "step": min(step, self._total),
            "total": self._total,
            "loss": self._loss,
            "gaussians": int(self._trainer.pipeline.model.num_points),
            "written_at": time.time(),
        }
        staging = self._progress.with_suffix(".tmp")
        staging.write_text(json.dumps(payload))
        os.replace(staging, self._progress)

__enter__() #

Start reporting; every training iteration records its loss.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def __enter__(self) -> reporting:
    """Start reporting; every training iteration records its loss."""
    iterate = self._trainer.train_iteration

    def recorded(step: int) -> tuple[object, dict[str, object], dict[str, object]]:
        loss, loss_dict, metrics = iterate(step)
        self._loss = float(loss)
        return loss, loss_dict, metrics

    self._trainer.train_iteration = recorded
    self._thread.start()
    return self

__exit__(*_) #

Stop reporting and write the final count.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def __exit__(self, *_: object) -> None:
    """Stop reporting and write the final count."""
    self._stop.set()
    self._thread.join()
    self._write(self._total)

__init__(trainer, total, progress) #

Report trainer's run of total steps into progress.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def __init__(self, trainer: Trainer, total: int, progress: Path) -> None:
    """Report ``trainer``'s run of ``total`` steps into ``progress``."""
    self._trainer = trainer
    self._total = total
    self._progress = progress
    self._loss: float | None = None
    self._stop = threading.Event()
    self._thread = threading.Thread(target=self._run, daemon=True)

build_trainer(data, output, iterations, *, optimize_cameras, depth_weight) #

A splatfacto trainer over data in the capture's own metric frame, degree-0 colour.

A positive depth_weight swaps in the depth-supervised model and the datamanager that loads each frame's depth and confidence; otherwise the run is stock splatfacto.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def build_trainer(
    data: Path,
    output: Path,
    iterations: int,
    *,
    optimize_cameras: bool,
    depth_weight: float,
) -> Trainer:
    """A splatfacto trainer over ``data`` in the capture's own metric frame, degree-0 colour.

    A positive ``depth_weight`` swaps in the depth-supervised model and the
    datamanager that loads each frame's depth and confidence; otherwise the
    run is stock splatfacto.
    """
    from nerfstudio.configs.method_configs import (  # pyright: ignore[reportMissingImports]
        all_methods,
    )

    config = copy.deepcopy(all_methods["splatfacto"])
    model = config.pipeline.model
    model.sh_degree = 0
    model.stop_split_at = iterations // 2
    model.rasterize_mode = "classic"
    model.camera_optimizer.mode = "SO3xR3" if optimize_cameras else "off"

    datamanager = config.pipeline.datamanager
    datamanager.cache_images = "cpu"
    datamanager.cache_images_type = "uint8"
    dataparser = datamanager.dataparser
    dataparser.data = data
    dataparser.load_3D_points = True
    dataparser.orientation_method = "none"
    dataparser.center_method = "none"
    dataparser.auto_scale_poses = False
    dataparser.downscale_factor = 1
    dataparser.eval_mode = "all"

    if depth_weight > 0:
        model_config, datamanager_target = depth_model()
        fields = {
            field.name: getattr(model, field.name)
            for field in dataclasses.fields(model)
            if field.name != "_target"
        }
        config.pipeline.model = model_config(
            **{
                **fields,
                "depth_loss_mult": depth_weight,
                "output_depth_during_training": True,
            }
        )
        datamanager._target = datamanager_target
        dataparser.depth_unit_scale_factor = 1.0

    never = iterations + 1
    config.output_dir = output
    config.max_num_iterations = iterations
    config.steps_per_eval_image = never
    config.steps_per_eval_all_images = never
    config.steps_per_eval_batch = never
    config.steps_per_save = iterations
    config.logging.steps_per_log = STEPS_PER_LOG
    config.vis = "tensorboard"
    config.set_timestamp()
    return config.setup(local_rank=0, world_size=1)

camera_to_world(pose) #

The 4×4 camera-to-world matrix of a stored Pose.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def camera_to_world(pose: dict) -> np.ndarray:
    """The 4×4 camera-to-world matrix of a stored ``Pose``."""
    w, x, y, z = (pose[f"quaternion_{axis}"] for axis in "wxyz")
    norm = math.sqrt(w * w + x * x + y * y + z * z) or 1.0
    w, x, y, z = w / norm, x / norm, y / norm, z / norm
    matrix = np.eye(4)
    matrix[:3, :3] = [
        [1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)],
        [2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)],
        [2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)],
    ]
    matrix[:3, 3] = [pose["position_x"], pose["position_y"], pose["position_z"]]
    return matrix

depth_model() #

The depth-supervised splatfacto: its model config and the datamanager feeding it.

Photometric-only splatfacto lets textureless walls and floors drift, the image loss having no gradient where colour is flat, exactly where measured depth is most informative. The model adds a depth term: the rendered expected depth pulled toward the frame's measured depth, weighted per pixel by the measurement's confidence rather than masked, so good but underconfident returns still contribute. Defined against nerfstudio on first use, so the module imports without it.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def depth_model() -> tuple[type[SplatfactoModelConfig], type[FullImageDatamanager]]:
    """The depth-supervised splatfacto: its model config and the datamanager feeding it.

    Photometric-only splatfacto lets textureless walls and floors drift, the
    image loss having no gradient where colour is flat, exactly where measured
    depth is most informative. The model adds a depth term: the rendered
    expected depth pulled toward the frame's measured depth, weighted per
    pixel by the measurement's confidence rather than masked, so good but
    underconfident returns still contribute. Defined against nerfstudio on
    first use, so the module imports without it.
    """
    import torch  # pyright: ignore[reportMissingImports]
    from nerfstudio.data.datamanagers.full_images_datamanager import (  # pyright: ignore[reportMissingImports]
        FullImageDatamanager,
    )
    from nerfstudio.data.datasets.depth_dataset import (  # pyright: ignore[reportMissingImports]
        DepthDataset,
    )
    from nerfstudio.data.utils.data_utils import (  # pyright: ignore[reportMissingImports]
        get_depth_image_from_path,
    )
    from nerfstudio.models.splatfacto import (  # pyright: ignore[reportMissingImports]
        SplatfactoModel,
        SplatfactoModelConfig,
    )

    class ConfidenceDepthDataset(DepthDataset):
        """A ``DepthDataset`` that also serves the frame's confidence, resized like the depth."""

        exclude_batch_keys_from_device = DepthDataset.exclude_batch_keys_from_device + [
            CONFIDENCE_KEY
        ]

        def get_metadata(self, data: dict) -> dict:
            metadata = super().get_metadata(data)
            depth_path = self._dataparser_outputs.metadata["depth_filenames"][data["image_idx"]]
            metadata[CONFIDENCE_KEY] = get_depth_image_from_path(
                filepath=CONFIDENCE_OF[Path(depth_path).resolve()],
                height=int(self._dataparser_outputs.cameras.height[data["image_idx"]]),
                width=int(self._dataparser_outputs.cameras.width[data["image_idx"]]),
                scale_factor=1.0,
            )
            return metadata

    class ConfidenceDepthDatamanager(FullImageDatamanager[ConfidenceDepthDataset]):
        """Splatfacto's full-image datamanager, typed to load depth and confidence."""

    @dataclasses.dataclass
    class DepthSplatfactoModelConfig(SplatfactoModelConfig):
        """Splatfacto's config plus the weight of the depth term."""

        _target: type = dataclasses.field(default_factory=lambda: DepthSplatfactoModel)
        depth_loss_mult: float = 0.2

    class DepthSplatfactoModel(SplatfactoModel):
        """Splatfacto with the confidence-weighted LogL1 depth term."""

        config: DepthSplatfactoModelConfig

        def get_loss_dict(
            self,
            outputs: dict,
            batch: dict,
            metrics_dict: dict | None = None,
        ) -> dict:
            loss_dict = super().get_loss_dict(outputs, batch, metrics_dict)
            if self.training and "depth_image" in batch and "depth" in outputs:
                loss_dict["depth_loss"] = self._depth_loss(outputs, batch)
            return loss_dict

        def _depth_loss(self, outputs: dict, batch: dict) -> torch.Tensor:
            rendered = outputs["depth"]
            measured = batch["depth_image"].to(rendered.device)
            confidence = batch[CONFIDENCE_KEY].to(rendered.device)
            if measured.shape[:2] != rendered.shape[:2]:
                measured = _resize_to(measured, rendered.shape[:2])
                confidence = _resize_to(confidence, rendered.shape[:2])
            weight = torch.where(
                measured > DEPTH_TOLERANCE_METERS,
                DEPTH_CONFIDENCE_FLOOR
                + (1.0 - DEPTH_CONFIDENCE_FLOOR) * confidence.clamp(0.0, 1.0),
                torch.zeros_like(confidence),
            )
            logl1 = torch.log1p(torch.abs(rendered - measured))
            return (
                self.config.depth_loss_mult * (weight * logl1).sum() / weight.sum().clamp_min(1.0)
            )

    return DepthSplatfactoModelConfig, ConfidenceDepthDatamanager

drift(given, moved) #

How far moved cameras lie from given ones: the median and 95th percentile.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def drift(given: np.ndarray, moved: np.ndarray) -> str:
    """How far ``moved`` cameras lie from ``given`` ones: the median and 95th percentile."""
    metres = np.linalg.norm(moved[:, :3, 3] - given[:, :3, 3], axis=1)
    cosines = (np.einsum("nij,nij->n", moved[:, :3, :3], given[:, :3, :3]) - 1.0) / 2.0
    degrees = np.degrees(np.arccos(np.clip(cosines, -1.0, 1.0)))
    return (
        f"{np.median(metres) * 100:.1f} cm / {np.median(degrees):.2f}° median, "
        f"{np.percentile(metres, 95) * 100:.1f} cm / {np.percentile(degrees, 95):.2f}° 95th"
    )

export(model, cameras, splat, progress) #

The model's gaussians written to splat in the frame of the cameras it trained from.

Camera refinement may carry the whole scene with the cameras, so the gaussians are carried back by the similarity realigning the refined cameras onto the given ones, the identity when nothing was refined. How far the cameras moved, before and after, is printed and noted in progress.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def export(
    model: SplatfactoModel,
    cameras: Cameras,
    splat: Path,
    progress: Path,
) -> None:
    """The model's gaussians written to ``splat`` in the frame of the cameras it trained from.

    Camera refinement may carry the whole scene with the cameras, so the
    gaussians are carried back by the similarity realigning the refined
    cameras onto the given ones, the identity when nothing was refined. How
    far the cameras moved, before and after, is printed and noted in
    ``progress``.
    """
    given = homogeneous(cameras.camera_to_worlds.cpu().numpy())
    refined = refined_cameras(model, cameras)
    similarity = Similarity.realigning(given, refined)
    note = (
        f"cameras moved {drift(given, refined)}; "
        f"{drift(given, similarity.cameras(refined))} after realigning "
        f"(scale {similarity.scale:.4f})"
    )
    print(note, flush=True)
    noted(progress, note)
    write_splat(
        splat,
        Gaussians(
            positions=model.means.detach().cpu().numpy(),
            colours=model.colors.detach().cpu().numpy(),
            opacities=model.opacities.detach().cpu().numpy().reshape(-1),
            scales=model.scales.detach().cpu().numpy(),
            quaternions=model.quats.detach().cpu().numpy(),
        ).carried(similarity),
    )

hamilton(left, right) #

The Hamilton product of one quaternion left with N quaternions right, w first.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def hamilton(left: np.ndarray, right: np.ndarray) -> np.ndarray:
    """The Hamilton product of one quaternion ``left`` with N quaternions ``right``, w first."""
    a0, a1, a2, a3 = left
    b0, b1, b2, b3 = right.T
    return np.stack(
        [
            a0 * b0 - a1 * b1 - a2 * b2 - a3 * b3,
            a0 * b1 + a1 * b0 + a2 * b3 - a3 * b2,
            a0 * b2 - a1 * b3 + a2 * b0 + a3 * b1,
            a0 * b3 + a1 * b2 - a2 * b1 + a3 * b0,
        ],
        axis=1,
    )

homogeneous(matrices) #

N × 3 × 4 camera-to-world matrices as N × 4 × 4.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def homogeneous(matrices: np.ndarray) -> np.ndarray:
    """N × 3 × 4 camera-to-world matrices as N × 4 × 4."""
    full = np.tile(np.eye(4), (len(matrices), 1, 1))
    full[:, :3, :] = matrices
    return full

main() #

Train on the dataset the arguments name and write the splat.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def main() -> None:
    """Train on the dataset the arguments name and write the splat."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--frames", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--iterations", type=int, required=True)
    parser.add_argument("--seed", type=Path, required=True)
    parser.add_argument("--splat", type=Path, required=True)
    parser.add_argument("--progress", type=Path, required=True)
    parser.add_argument("--depth-weight", type=float, required=True)
    parser.add_argument("--optimize-cameras", action="store_true")
    arguments = parser.parse_args()

    data = stage_dataset(arguments.frames, arguments.output / "data", arguments.seed)
    trainer = build_trainer(
        data,
        arguments.output / "runs",
        arguments.iterations,
        optimize_cameras=arguments.optimize_cameras,
        depth_weight=arguments.depth_weight,
    )
    trainer.setup()
    with reporting(trainer, arguments.iterations, arguments.progress):
        trainer.train()
    export(
        trainer.pipeline.model,
        trainer.pipeline.datamanager.train_dataset.cameras,
        arguments.splat,
        arguments.progress,
    )

nearest_rotation(matrix) #

The rotation nearest matrix in the Frobenius norm.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def nearest_rotation(matrix: np.ndarray) -> np.ndarray:
    """The rotation nearest ``matrix`` in the Frobenius norm."""
    left, _, right = np.linalg.svd(matrix)
    return left @ np.diag([1.0, 1.0, np.linalg.det(left @ right)]) @ right

noted(progress, note) #

progress with note added under "cameras".

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def noted(progress: Path, note: str) -> None:
    """``progress`` with ``note`` added under ``"cameras"``."""
    payload = json.loads(progress.read_text())
    payload["cameras"] = note
    staging = progress.with_suffix(".tmp")
    staging.write_text(json.dumps(payload))
    os.replace(staging, progress)

quaternion(rotation) #

The unit quaternion, w first, of a 3 × 3 rotation.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def quaternion(rotation: np.ndarray) -> np.ndarray:
    """The unit quaternion, w first, of a 3 × 3 rotation."""
    trace = np.trace(rotation)
    w = np.sqrt(max(0.0, 1.0 + trace)) / 2.0
    x = np.sqrt(max(0.0, 1.0 + 2.0 * rotation[0, 0] - trace)) / 2.0
    y = np.sqrt(max(0.0, 1.0 + 2.0 * rotation[1, 1] - trace)) / 2.0
    z = np.sqrt(max(0.0, 1.0 + 2.0 * rotation[2, 2] - trace)) / 2.0
    return np.array(
        [
            w,
            np.copysign(x, rotation[2, 1] - rotation[1, 2]),
            np.copysign(y, rotation[0, 2] - rotation[2, 0]),
            np.copysign(z, rotation[1, 0] - rotation[0, 1]),
        ],
    )

refined_cameras(model, cameras) #

Every training camera's camera-to-world matrix as the camera optimizer corrected it.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def refined_cameras(model: SplatfactoModel, cameras: Cameras) -> np.ndarray:
    """Every training camera's camera-to-world matrix as the camera optimizer corrected it."""
    import torch  # pyright: ignore[reportMissingImports]

    given = cameras.camera_to_worlds.cpu()
    with torch.no_grad():
        corrections = model.camera_optimizer(torch.arange(len(given))).cpu()
    rotations = corrections[:, :3, :3] @ given[:, :3, :3]
    centres = given[:, :3, 3:] + corrections[:, :3, 3:]
    return homogeneous(torch.cat([rotations, centres], dim=-1).cpu().numpy())

stage_dataset(manifest, data, seed) #

Nerfstudio's transforms.json under data for the frames manifest lists.

Every frame's image is referenced in place; seed is the PLY the gaussians start from. When any frame carries a depth, every frame records a depth_file_path — its own, or an empty return written under data for a frame that measured none — and CONFIDENCE_OF maps each depth plane to its confidence plane for the dataset to read.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def stage_dataset(manifest: Path, data: Path, seed: Path) -> Path:
    """Nerfstudio's ``transforms.json`` under ``data`` for the frames ``manifest`` lists.

    Every frame's image is referenced in place; ``seed`` is the PLY the
    gaussians start from. When any frame carries a depth, every frame records
    a ``depth_file_path`` — its own, or an empty return written under ``data``
    for a frame that measured none — and ``CONFIDENCE_OF`` maps each depth
    plane to its confidence plane for the dataset to read.
    """
    frames = json.loads(manifest.read_text())
    supervised = any(frame["depth"] is not None for frame in frames)
    records = []
    for index, frame in enumerate(frames):
        pose = frame["pose"]
        record = {
            "file_path": frame["image"],
            "w": int(pose["reference_width"]),
            "h": int(pose["reference_height"]),
            "fl_x": pose["focal_x"],
            "fl_y": pose["focal_y"],
            "cx": pose["center_x"],
            "cy": pose["center_y"],
            "transform_matrix": camera_to_world(pose).tolist(),
        }
        if supervised:
            depth, confidence = staged_depth(data, index, frame["depth"], frame["confidence"])
            CONFIDENCE_OF[depth] = confidence
            record["depth_file_path"] = str(depth)
        records.append(record)
    data.mkdir(parents=True, exist_ok=True)
    (data / "transforms.json").write_text(
        json.dumps(
            {
                "camera_model": "OPENCV",
                "ply_file_path": str(seed.resolve()),
                "frames": records,
            },
            indent=1,
        )
    )
    return data

staged_depth(data, index, depth, confidence) #

Frame index's (depth, confidence) planes, an empty return written when it has none.

Nerfstudio's dataparser takes a depth for every frame or for none, so a frame without one gets a single zero pixel at zero confidence, which the tolerance masks out of the loss.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def staged_depth(
    data: Path,
    index: int,
    depth: str | None,
    confidence: str | None,
) -> tuple[Path, Path]:
    """Frame ``index``'s ``(depth, confidence)`` planes, an empty return written when it has none.

    Nerfstudio's dataparser takes a depth for every frame or for none, so a
    frame without one gets a single zero pixel at zero confidence, which the
    tolerance masks out of the loss.
    """
    if depth is not None and confidence is not None:
        return Path(depth).resolve(), Path(confidence).resolve()
    empty = data / "empty_returns"
    empty.mkdir(parents=True, exist_ok=True)
    planes = empty / f"{index}.npy", empty / f"{index}_confidence.npy"
    for plane in planes:
        np.save(plane, np.zeros((1, 1), np.float32))
    return planes[0].resolve(), planes[1].resolve()

write_splat(splat, gaussians) #

gaussians as a binary PLY of the standard 3DGS properties, non-finite ones left out.

A colour is stored as its degree-zero harmonic, (colour - 0.5) / HARMONIC_DEGREE_ZERO, in f_dc_0..2.

Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
def write_splat(splat: Path, gaussians: Gaussians) -> None:
    """``gaussians`` as a binary PLY of the standard 3DGS properties, non-finite ones left out.

    A colour is stored as its degree-zero harmonic,
    ``(colour - 0.5) / HARMONIC_DEGREE_ZERO``, in ``f_dc_0..2``.
    """
    columns = np.concatenate(
        [
            gaussians.positions,
            np.zeros_like(gaussians.positions),
            (gaussians.colours - 0.5) / HARMONIC_DEGREE_ZERO,
            gaussians.opacities.reshape(-1, 1),
            gaussians.scales,
            gaussians.quaternions,
        ],
        axis=1,
    ).astype(np.float32)
    columns = columns[np.isfinite(columns).all(axis=1)]
    record = np.zeros(len(columns), dtype=[(name, "<f4") for name in PLY_PROPERTIES])
    for index, name in enumerate(PLY_PROPERTIES):
        record[name] = columns[:, index]
    header = (
        "ply\nformat binary_little_endian 1.0\n"
        f"element vertex {len(columns)}\n"
        + "".join(f"property float {name}\n" for name in PLY_PROPERTIES)
        + "end_header\n"
    )
    splat.write_bytes(header.encode("ascii") + record.tobytes())