The splat trainer, run by cgr.splats.train inside the GPU interpreter.
Only torch, nerfstudio and numpy are imported here, never capturegraph: the
GPU environment is a separate installation, so the frames arrive as an
explicit manifest rather than as a stored array. The run writes
progress.json twice a second and the trained gaussians as the standard
3DGS PLY cgr.Splat documents, each colour as its degree-zero harmonic,
which the SplatViewer reads. The gaussians are written in the frame of the
poses given: when the cameras were refined, carried back by the similarity
realigning the refined cameras onto the given ones.
python trainer.py --frames FRAMES.json --output O --iterations N \\
--seed S.ply --splat OUT.ply --progress P.json \\
--depth-weight W [--optimize-cameras]
FRAMES.json is a list, in training order, of {"image", "pose", "depth",
"confidence"}: the image's path, the pose as capturegraph's Pose JSON
(position, quaternion and intrinsics at the image's size, in the OpenGL
camera-to-world convention), and the paths of the frame's measured depth in
metres and its per-pixel confidence as .npy planes at any resolution, or
null where it measured none. With a positive --depth-weight the
depth-supervised splatfacto depth_model defines trains, its depth term
the confidence-weighted LogL1 distance between the rendered and the
measured depth.
CONFIDENCE_OF = {}
module-attribute
Each staged depth plane's confidence plane, filled by stage_dataset.
DEPTH_CONFIDENCE_FLOOR = 0.1
module-attribute
Loss weight of a measured zero-confidence pixel; full confidence weighs 1.
DEPTH_TOLERANCE_METERS = 0.05
module-attribute
The least measured depth treated as a return.
HARMONIC_DEGREE_ZERO = 0.28209479177387814
module-attribute
The degree-zero spherical harmonic: a PLY stores colour c as (c - 0.5) / this.
Gaussians
dataclass
N trained gaussians, as the standard 3DGS PLY stores them.
Attributes:
| Name |
Type |
Description |
positions |
ndarray
|
|
colours |
ndarray
|
Their RGB in [0, 1], N × 3.
|
opacities |
ndarray
|
|
scales |
ndarray
|
|
quaternions |
ndarray
|
Their rotations, w first, N × 4.
|
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| @dataclasses.dataclass(frozen=True)
class Gaussians:
"""N trained gaussians, as the standard 3DGS PLY stores them.
Attributes:
positions: Their centres, N × 3.
colours: Their RGB in [0, 1], N × 3.
opacities: Their opacity logits, N.
scales: Their log-scales, N × 3.
quaternions: Their rotations, w first, N × 4.
"""
positions: np.ndarray
colours: np.ndarray
opacities: np.ndarray
scales: np.ndarray
quaternions: np.ndarray
def carried(self, similarity: Similarity) -> Gaussians:
"""These gaussians moved, turned and grown by ``similarity``."""
return dataclasses.replace(
self,
positions=similarity.scale * self.positions @ similarity.rotation.T
+ similarity.translation,
scales=self.scales + np.log(similarity.scale),
quaternions=hamilton(quaternion(similarity.rotation), self.quaternions),
)
|
carried(similarity)
These gaussians moved, turned and grown by similarity.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def carried(self, similarity: Similarity) -> Gaussians:
"""These gaussians moved, turned and grown by ``similarity``."""
return dataclasses.replace(
self,
positions=similarity.scale * self.positions @ similarity.rotation.T
+ similarity.translation,
scales=self.scales + np.log(similarity.scale),
quaternions=hamilton(quaternion(similarity.rotation), self.quaternions),
)
|
Similarity
dataclass
The map x -> scale * rotation @ x + translation.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| @dataclasses.dataclass(frozen=True)
class Similarity:
"""The map ``x -> scale * rotation @ x + translation``."""
scale: float
rotation: np.ndarray
translation: np.ndarray
@classmethod
def realigning(cls, given: np.ndarray, refined: np.ndarray) -> Similarity:
"""The similarity carrying ``refined`` cameras back onto ``given`` ones.
Its rotation best turns the refined orientations into the given ones,
so it holds where the centres lie along a line, as a corridor walk's
do; its scale and translation then best carry the refined centres
onto the given ones.
Args:
given: The cameras as trained from, N × 4 × 4 camera-to-world.
refined: The same cameras after refinement, N × 4 × 4.
"""
rotation = nearest_rotation(
np.einsum("nij,nkj->ik", given[:, :3, :3], refined[:, :3, :3]),
)
source = refined[:, :3, 3] @ rotation.T
target = given[:, :3, 3]
source_offsets = source - source.mean(axis=0)
target_offsets = target - target.mean(axis=0)
scale = float(np.sum(source_offsets * target_offsets) / np.sum(source_offsets**2))
return cls(
scale=scale,
rotation=rotation,
translation=target.mean(axis=0) - scale * source.mean(axis=0),
)
def cameras(self, matrices: np.ndarray) -> np.ndarray:
"""N × 4 × 4 camera-to-world ``matrices`` carried by this similarity."""
carried = matrices.copy()
carried[:, :3, :3] = self.rotation @ matrices[:, :3, :3]
carried[:, :3, 3] = self.scale * matrices[:, :3, 3] @ self.rotation.T + self.translation
return carried
|
cameras(matrices)
N × 4 × 4 camera-to-world matrices carried by this similarity.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def cameras(self, matrices: np.ndarray) -> np.ndarray:
"""N × 4 × 4 camera-to-world ``matrices`` carried by this similarity."""
carried = matrices.copy()
carried[:, :3, :3] = self.rotation @ matrices[:, :3, :3]
carried[:, :3, 3] = self.scale * matrices[:, :3, 3] @ self.rotation.T + self.translation
return carried
|
realigning(given, refined)
classmethod
The similarity carrying refined cameras back onto given ones.
Its rotation best turns the refined orientations into the given ones,
so it holds where the centres lie along a line, as a corridor walk's
do; its scale and translation then best carry the refined centres
onto the given ones.
Parameters:
| Name |
Type |
Description |
Default |
given
|
ndarray
|
The cameras as trained from, N × 4 × 4 camera-to-world.
|
required
|
refined
|
ndarray
|
The same cameras after refinement, N × 4 × 4.
|
required
|
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| @classmethod
def realigning(cls, given: np.ndarray, refined: np.ndarray) -> Similarity:
"""The similarity carrying ``refined`` cameras back onto ``given`` ones.
Its rotation best turns the refined orientations into the given ones,
so it holds where the centres lie along a line, as a corridor walk's
do; its scale and translation then best carry the refined centres
onto the given ones.
Args:
given: The cameras as trained from, N × 4 × 4 camera-to-world.
refined: The same cameras after refinement, N × 4 × 4.
"""
rotation = nearest_rotation(
np.einsum("nij,nkj->ik", given[:, :3, :3], refined[:, :3, :3]),
)
source = refined[:, :3, 3] @ rotation.T
target = given[:, :3, 3]
source_offsets = source - source.mean(axis=0)
target_offsets = target - target.mean(axis=0)
scale = float(np.sum(source_offsets * target_offsets) / np.sum(source_offsets**2))
return cls(
scale=scale,
rotation=rotation,
translation=target.mean(axis=0) - scale * source.mean(axis=0),
)
|
reporting
Writes {"step", "total", "loss", "gaussians"} to progress while the block trains.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| class reporting:
"""Writes ``{"step", "total", "loss", "gaussians"}`` to ``progress`` while the block trains."""
def __init__(self, trainer: Trainer, total: int, progress: Path) -> None:
"""Report ``trainer``'s run of ``total`` steps into ``progress``."""
self._trainer = trainer
self._total = total
self._progress = progress
self._loss: float | None = None
self._stop = threading.Event()
self._thread = threading.Thread(target=self._run, daemon=True)
def __enter__(self) -> reporting:
"""Start reporting; every training iteration records its loss."""
iterate = self._trainer.train_iteration
def recorded(step: int) -> tuple[object, dict[str, object], dict[str, object]]:
loss, loss_dict, metrics = iterate(step)
self._loss = float(loss)
return loss, loss_dict, metrics
self._trainer.train_iteration = recorded
self._thread.start()
return self
def __exit__(self, *_: object) -> None:
"""Stop reporting and write the final count."""
self._stop.set()
self._thread.join()
self._write(self._total)
def _run(self) -> None:
while not self._stop.wait(REPORT_SECONDS):
self._write(int(getattr(self._trainer, "step", 0)))
def _write(self, step: int) -> None:
payload = {
"step": min(step, self._total),
"total": self._total,
"loss": self._loss,
"gaussians": int(self._trainer.pipeline.model.num_points),
"written_at": time.time(),
}
staging = self._progress.with_suffix(".tmp")
staging.write_text(json.dumps(payload))
os.replace(staging, self._progress)
|
__enter__()
Start reporting; every training iteration records its loss.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def __enter__(self) -> reporting:
"""Start reporting; every training iteration records its loss."""
iterate = self._trainer.train_iteration
def recorded(step: int) -> tuple[object, dict[str, object], dict[str, object]]:
loss, loss_dict, metrics = iterate(step)
self._loss = float(loss)
return loss, loss_dict, metrics
self._trainer.train_iteration = recorded
self._thread.start()
return self
|
__exit__(*_)
Stop reporting and write the final count.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def __exit__(self, *_: object) -> None:
"""Stop reporting and write the final count."""
self._stop.set()
self._thread.join()
self._write(self._total)
|
__init__(trainer, total, progress)
Report trainer's run of total steps into progress.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def __init__(self, trainer: Trainer, total: int, progress: Path) -> None:
"""Report ``trainer``'s run of ``total`` steps into ``progress``."""
self._trainer = trainer
self._total = total
self._progress = progress
self._loss: float | None = None
self._stop = threading.Event()
self._thread = threading.Thread(target=self._run, daemon=True)
|
build_trainer(data, output, iterations, *, optimize_cameras, depth_weight)
A splatfacto trainer over data in the capture's own metric frame, degree-0 colour.
A positive depth_weight swaps in the depth-supervised model and the
datamanager that loads each frame's depth and confidence; otherwise the
run is stock splatfacto.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def build_trainer(
data: Path,
output: Path,
iterations: int,
*,
optimize_cameras: bool,
depth_weight: float,
) -> Trainer:
"""A splatfacto trainer over ``data`` in the capture's own metric frame, degree-0 colour.
A positive ``depth_weight`` swaps in the depth-supervised model and the
datamanager that loads each frame's depth and confidence; otherwise the
run is stock splatfacto.
"""
from nerfstudio.configs.method_configs import ( # pyright: ignore[reportMissingImports]
all_methods,
)
config = copy.deepcopy(all_methods["splatfacto"])
model = config.pipeline.model
model.sh_degree = 0
model.stop_split_at = iterations // 2
model.rasterize_mode = "classic"
model.camera_optimizer.mode = "SO3xR3" if optimize_cameras else "off"
datamanager = config.pipeline.datamanager
datamanager.cache_images = "cpu"
datamanager.cache_images_type = "uint8"
dataparser = datamanager.dataparser
dataparser.data = data
dataparser.load_3D_points = True
dataparser.orientation_method = "none"
dataparser.center_method = "none"
dataparser.auto_scale_poses = False
dataparser.downscale_factor = 1
dataparser.eval_mode = "all"
if depth_weight > 0:
model_config, datamanager_target = depth_model()
fields = {
field.name: getattr(model, field.name)
for field in dataclasses.fields(model)
if field.name != "_target"
}
config.pipeline.model = model_config(
**{
**fields,
"depth_loss_mult": depth_weight,
"output_depth_during_training": True,
}
)
datamanager._target = datamanager_target
dataparser.depth_unit_scale_factor = 1.0
never = iterations + 1
config.output_dir = output
config.max_num_iterations = iterations
config.steps_per_eval_image = never
config.steps_per_eval_all_images = never
config.steps_per_eval_batch = never
config.steps_per_save = iterations
config.logging.steps_per_log = STEPS_PER_LOG
config.vis = "tensorboard"
config.set_timestamp()
return config.setup(local_rank=0, world_size=1)
|
camera_to_world(pose)
The 4×4 camera-to-world matrix of a stored Pose.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def camera_to_world(pose: dict) -> np.ndarray:
"""The 4×4 camera-to-world matrix of a stored ``Pose``."""
w, x, y, z = (pose[f"quaternion_{axis}"] for axis in "wxyz")
norm = math.sqrt(w * w + x * x + y * y + z * z) or 1.0
w, x, y, z = w / norm, x / norm, y / norm, z / norm
matrix = np.eye(4)
matrix[:3, :3] = [
[1 - 2 * (y * y + z * z), 2 * (x * y - z * w), 2 * (x * z + y * w)],
[2 * (x * y + z * w), 1 - 2 * (x * x + z * z), 2 * (y * z - x * w)],
[2 * (x * z - y * w), 2 * (y * z + x * w), 1 - 2 * (x * x + y * y)],
]
matrix[:3, 3] = [pose["position_x"], pose["position_y"], pose["position_z"]]
return matrix
|
depth_model()
The depth-supervised splatfacto: its model config and the datamanager feeding it.
Photometric-only splatfacto lets textureless walls and floors drift, the
image loss having no gradient where colour is flat, exactly where measured
depth is most informative. The model adds a depth term: the rendered
expected depth pulled toward the frame's measured depth, weighted per
pixel by the measurement's confidence rather than masked, so good but
underconfident returns still contribute. Defined against nerfstudio on
first use, so the module imports without it.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def depth_model() -> tuple[type[SplatfactoModelConfig], type[FullImageDatamanager]]:
"""The depth-supervised splatfacto: its model config and the datamanager feeding it.
Photometric-only splatfacto lets textureless walls and floors drift, the
image loss having no gradient where colour is flat, exactly where measured
depth is most informative. The model adds a depth term: the rendered
expected depth pulled toward the frame's measured depth, weighted per
pixel by the measurement's confidence rather than masked, so good but
underconfident returns still contribute. Defined against nerfstudio on
first use, so the module imports without it.
"""
import torch # pyright: ignore[reportMissingImports]
from nerfstudio.data.datamanagers.full_images_datamanager import ( # pyright: ignore[reportMissingImports]
FullImageDatamanager,
)
from nerfstudio.data.datasets.depth_dataset import ( # pyright: ignore[reportMissingImports]
DepthDataset,
)
from nerfstudio.data.utils.data_utils import ( # pyright: ignore[reportMissingImports]
get_depth_image_from_path,
)
from nerfstudio.models.splatfacto import ( # pyright: ignore[reportMissingImports]
SplatfactoModel,
SplatfactoModelConfig,
)
class ConfidenceDepthDataset(DepthDataset):
"""A ``DepthDataset`` that also serves the frame's confidence, resized like the depth."""
exclude_batch_keys_from_device = DepthDataset.exclude_batch_keys_from_device + [
CONFIDENCE_KEY
]
def get_metadata(self, data: dict) -> dict:
metadata = super().get_metadata(data)
depth_path = self._dataparser_outputs.metadata["depth_filenames"][data["image_idx"]]
metadata[CONFIDENCE_KEY] = get_depth_image_from_path(
filepath=CONFIDENCE_OF[Path(depth_path).resolve()],
height=int(self._dataparser_outputs.cameras.height[data["image_idx"]]),
width=int(self._dataparser_outputs.cameras.width[data["image_idx"]]),
scale_factor=1.0,
)
return metadata
class ConfidenceDepthDatamanager(FullImageDatamanager[ConfidenceDepthDataset]):
"""Splatfacto's full-image datamanager, typed to load depth and confidence."""
@dataclasses.dataclass
class DepthSplatfactoModelConfig(SplatfactoModelConfig):
"""Splatfacto's config plus the weight of the depth term."""
_target: type = dataclasses.field(default_factory=lambda: DepthSplatfactoModel)
depth_loss_mult: float = 0.2
class DepthSplatfactoModel(SplatfactoModel):
"""Splatfacto with the confidence-weighted LogL1 depth term."""
config: DepthSplatfactoModelConfig
def get_loss_dict(
self,
outputs: dict,
batch: dict,
metrics_dict: dict | None = None,
) -> dict:
loss_dict = super().get_loss_dict(outputs, batch, metrics_dict)
if self.training and "depth_image" in batch and "depth" in outputs:
loss_dict["depth_loss"] = self._depth_loss(outputs, batch)
return loss_dict
def _depth_loss(self, outputs: dict, batch: dict) -> torch.Tensor:
rendered = outputs["depth"]
measured = batch["depth_image"].to(rendered.device)
confidence = batch[CONFIDENCE_KEY].to(rendered.device)
if measured.shape[:2] != rendered.shape[:2]:
measured = _resize_to(measured, rendered.shape[:2])
confidence = _resize_to(confidence, rendered.shape[:2])
weight = torch.where(
measured > DEPTH_TOLERANCE_METERS,
DEPTH_CONFIDENCE_FLOOR
+ (1.0 - DEPTH_CONFIDENCE_FLOOR) * confidence.clamp(0.0, 1.0),
torch.zeros_like(confidence),
)
logl1 = torch.log1p(torch.abs(rendered - measured))
return (
self.config.depth_loss_mult * (weight * logl1).sum() / weight.sum().clamp_min(1.0)
)
return DepthSplatfactoModelConfig, ConfidenceDepthDatamanager
|
drift(given, moved)
How far moved cameras lie from given ones: the median and 95th percentile.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def drift(given: np.ndarray, moved: np.ndarray) -> str:
"""How far ``moved`` cameras lie from ``given`` ones: the median and 95th percentile."""
metres = np.linalg.norm(moved[:, :3, 3] - given[:, :3, 3], axis=1)
cosines = (np.einsum("nij,nij->n", moved[:, :3, :3], given[:, :3, :3]) - 1.0) / 2.0
degrees = np.degrees(np.arccos(np.clip(cosines, -1.0, 1.0)))
return (
f"{np.median(metres) * 100:.1f} cm / {np.median(degrees):.2f}° median, "
f"{np.percentile(metres, 95) * 100:.1f} cm / {np.percentile(degrees, 95):.2f}° 95th"
)
|
export(model, cameras, splat, progress)
The model's gaussians written to splat in the frame of the cameras it trained from.
Camera refinement may carry the whole scene with the cameras, so the
gaussians are carried back by the similarity realigning the refined
cameras onto the given ones, the identity when nothing was refined. How
far the cameras moved, before and after, is printed and noted in
progress.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def export(
model: SplatfactoModel,
cameras: Cameras,
splat: Path,
progress: Path,
) -> None:
"""The model's gaussians written to ``splat`` in the frame of the cameras it trained from.
Camera refinement may carry the whole scene with the cameras, so the
gaussians are carried back by the similarity realigning the refined
cameras onto the given ones, the identity when nothing was refined. How
far the cameras moved, before and after, is printed and noted in
``progress``.
"""
given = homogeneous(cameras.camera_to_worlds.cpu().numpy())
refined = refined_cameras(model, cameras)
similarity = Similarity.realigning(given, refined)
note = (
f"cameras moved {drift(given, refined)}; "
f"{drift(given, similarity.cameras(refined))} after realigning "
f"(scale {similarity.scale:.4f})"
)
print(note, flush=True)
noted(progress, note)
write_splat(
splat,
Gaussians(
positions=model.means.detach().cpu().numpy(),
colours=model.colors.detach().cpu().numpy(),
opacities=model.opacities.detach().cpu().numpy().reshape(-1),
scales=model.scales.detach().cpu().numpy(),
quaternions=model.quats.detach().cpu().numpy(),
).carried(similarity),
)
|
hamilton(left, right)
The Hamilton product of one quaternion left with N quaternions right, w first.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def hamilton(left: np.ndarray, right: np.ndarray) -> np.ndarray:
"""The Hamilton product of one quaternion ``left`` with N quaternions ``right``, w first."""
a0, a1, a2, a3 = left
b0, b1, b2, b3 = right.T
return np.stack(
[
a0 * b0 - a1 * b1 - a2 * b2 - a3 * b3,
a0 * b1 + a1 * b0 + a2 * b3 - a3 * b2,
a0 * b2 - a1 * b3 + a2 * b0 + a3 * b1,
a0 * b3 + a1 * b2 - a2 * b1 + a3 * b0,
],
axis=1,
)
|
homogeneous(matrices)
N × 3 × 4 camera-to-world matrices as N × 4 × 4.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def homogeneous(matrices: np.ndarray) -> np.ndarray:
"""N × 3 × 4 camera-to-world matrices as N × 4 × 4."""
full = np.tile(np.eye(4), (len(matrices), 1, 1))
full[:, :3, :] = matrices
return full
|
main()
Train on the dataset the arguments name and write the splat.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def main() -> None:
"""Train on the dataset the arguments name and write the splat."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--frames", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--iterations", type=int, required=True)
parser.add_argument("--seed", type=Path, required=True)
parser.add_argument("--splat", type=Path, required=True)
parser.add_argument("--progress", type=Path, required=True)
parser.add_argument("--depth-weight", type=float, required=True)
parser.add_argument("--optimize-cameras", action="store_true")
arguments = parser.parse_args()
data = stage_dataset(arguments.frames, arguments.output / "data", arguments.seed)
trainer = build_trainer(
data,
arguments.output / "runs",
arguments.iterations,
optimize_cameras=arguments.optimize_cameras,
depth_weight=arguments.depth_weight,
)
trainer.setup()
with reporting(trainer, arguments.iterations, arguments.progress):
trainer.train()
export(
trainer.pipeline.model,
trainer.pipeline.datamanager.train_dataset.cameras,
arguments.splat,
arguments.progress,
)
|
nearest_rotation(matrix)
The rotation nearest matrix in the Frobenius norm.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def nearest_rotation(matrix: np.ndarray) -> np.ndarray:
"""The rotation nearest ``matrix`` in the Frobenius norm."""
left, _, right = np.linalg.svd(matrix)
return left @ np.diag([1.0, 1.0, np.linalg.det(left @ right)]) @ right
|
noted(progress, note)
progress with note added under "cameras".
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def noted(progress: Path, note: str) -> None:
"""``progress`` with ``note`` added under ``"cameras"``."""
payload = json.loads(progress.read_text())
payload["cameras"] = note
staging = progress.with_suffix(".tmp")
staging.write_text(json.dumps(payload))
os.replace(staging, progress)
|
quaternion(rotation)
The unit quaternion, w first, of a 3 × 3 rotation.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def quaternion(rotation: np.ndarray) -> np.ndarray:
"""The unit quaternion, w first, of a 3 × 3 rotation."""
trace = np.trace(rotation)
w = np.sqrt(max(0.0, 1.0 + trace)) / 2.0
x = np.sqrt(max(0.0, 1.0 + 2.0 * rotation[0, 0] - trace)) / 2.0
y = np.sqrt(max(0.0, 1.0 + 2.0 * rotation[1, 1] - trace)) / 2.0
z = np.sqrt(max(0.0, 1.0 + 2.0 * rotation[2, 2] - trace)) / 2.0
return np.array(
[
w,
np.copysign(x, rotation[2, 1] - rotation[1, 2]),
np.copysign(y, rotation[0, 2] - rotation[2, 0]),
np.copysign(z, rotation[1, 0] - rotation[0, 1]),
],
)
|
refined_cameras(model, cameras)
Every training camera's camera-to-world matrix as the camera optimizer corrected it.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def refined_cameras(model: SplatfactoModel, cameras: Cameras) -> np.ndarray:
"""Every training camera's camera-to-world matrix as the camera optimizer corrected it."""
import torch # pyright: ignore[reportMissingImports]
given = cameras.camera_to_worlds.cpu()
with torch.no_grad():
corrections = model.camera_optimizer(torch.arange(len(given))).cpu()
rotations = corrections[:, :3, :3] @ given[:, :3, :3]
centres = given[:, :3, 3:] + corrections[:, :3, 3:]
return homogeneous(torch.cat([rotations, centres], dim=-1).cpu().numpy())
|
stage_dataset(manifest, data, seed)
Nerfstudio's transforms.json under data for the frames manifest lists.
Every frame's image is referenced in place; seed is the PLY the
gaussians start from. When any frame carries a depth, every frame records
a depth_file_path — its own, or an empty return written under data
for a frame that measured none — and CONFIDENCE_OF maps each depth
plane to its confidence plane for the dataset to read.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def stage_dataset(manifest: Path, data: Path, seed: Path) -> Path:
"""Nerfstudio's ``transforms.json`` under ``data`` for the frames ``manifest`` lists.
Every frame's image is referenced in place; ``seed`` is the PLY the
gaussians start from. When any frame carries a depth, every frame records
a ``depth_file_path`` — its own, or an empty return written under ``data``
for a frame that measured none — and ``CONFIDENCE_OF`` maps each depth
plane to its confidence plane for the dataset to read.
"""
frames = json.loads(manifest.read_text())
supervised = any(frame["depth"] is not None for frame in frames)
records = []
for index, frame in enumerate(frames):
pose = frame["pose"]
record = {
"file_path": frame["image"],
"w": int(pose["reference_width"]),
"h": int(pose["reference_height"]),
"fl_x": pose["focal_x"],
"fl_y": pose["focal_y"],
"cx": pose["center_x"],
"cy": pose["center_y"],
"transform_matrix": camera_to_world(pose).tolist(),
}
if supervised:
depth, confidence = staged_depth(data, index, frame["depth"], frame["confidence"])
CONFIDENCE_OF[depth] = confidence
record["depth_file_path"] = str(depth)
records.append(record)
data.mkdir(parents=True, exist_ok=True)
(data / "transforms.json").write_text(
json.dumps(
{
"camera_model": "OPENCV",
"ply_file_path": str(seed.resolve()),
"frames": records,
},
indent=1,
)
)
return data
|
staged_depth(data, index, depth, confidence)
Frame index's (depth, confidence) planes, an empty return written when it has none.
Nerfstudio's dataparser takes a depth for every frame or for none, so a
frame without one gets a single zero pixel at zero confidence, which the
tolerance masks out of the loss.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def staged_depth(
data: Path,
index: int,
depth: str | None,
confidence: str | None,
) -> tuple[Path, Path]:
"""Frame ``index``'s ``(depth, confidence)`` planes, an empty return written when it has none.
Nerfstudio's dataparser takes a depth for every frame or for none, so a
frame without one gets a single zero pixel at zero confidence, which the
tolerance masks out of the loss.
"""
if depth is not None and confidence is not None:
return Path(depth).resolve(), Path(confidence).resolve()
empty = data / "empty_returns"
empty.mkdir(parents=True, exist_ok=True)
planes = empty / f"{index}.npy", empty / f"{index}_confidence.npy"
for plane in planes:
np.save(plane, np.zeros((1, 1), np.float32))
return planes[0].resolve(), planes[1].resolve()
|
write_splat(splat, gaussians)
gaussians as a binary PLY of the standard 3DGS properties, non-finite ones left out.
A colour is stored as its degree-zero harmonic,
(colour - 0.5) / HARMONIC_DEGREE_ZERO, in f_dc_0..2.
Source code in capturegraph-recipes/capturegraph_recipes/splats/training/trainer.py
| def write_splat(splat: Path, gaussians: Gaussians) -> None:
"""``gaussians`` as a binary PLY of the standard 3DGS properties, non-finite ones left out.
A colour is stored as its degree-zero harmonic,
``(colour - 0.5) / HARMONIC_DEGREE_ZERO``, in ``f_dc_0..2``.
"""
columns = np.concatenate(
[
gaussians.positions,
np.zeros_like(gaussians.positions),
(gaussians.colours - 0.5) / HARMONIC_DEGREE_ZERO,
gaussians.opacities.reshape(-1, 1),
gaussians.scales,
gaussians.quaternions,
],
axis=1,
).astype(np.float32)
columns = columns[np.isfinite(columns).all(axis=1)]
record = np.zeros(len(columns), dtype=[(name, "<f4") for name in PLY_PROPERTIES])
for index, name in enumerate(PLY_PROPERTIES):
record[name] = columns[:, index]
header = (
"ply\nformat binary_little_endian 1.0\n"
f"element vertex {len(columns)}\n"
+ "".join(f"property float {name}\n" for name in PLY_PROPERTIES)
+ "end_header\n"
)
splat.write_bytes(header.encode("ascii") + record.tobytes())
|