adding sailrecon

.pre-commit-config.yaml

@@ -4,12 +4,14 @@ repos:
     hooks:
       - id: black
         args: ["--line-length=120"]
+        exclude: ^third_party/
   - repo: https://github.com/pycqa/isort
     rev: 6.0.1
     hooks:
       - id: isort
         name: isort (python)
         args: ["--line-length=120"]
+        exclude: ^third_party/
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: "v5.0.0"
     hooks:
@@ -25,11 +27,14 @@ repos:
         exclude: ^.devcontainer/devcontainer.json
         args: [--autofix]
       - id: end-of-file-fixer
+        exclude: ^third_party/
       - id: trailing-whitespace
+        exclude: ^third_party/
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.11.5"
     hooks:
       - id: ruff
         args: [--exit-non-zero-on-fix]
+        exclude: ^third_party/
       # - id: ruff-format

pyproject.toml

@@ -117,6 +117,14 @@ testpaths = ["tests"]
 target-version = "py39"
 line-length = 120
 fix = true
+extend-exclude = [
+    # Imported SAIL-Recon operators intentionally retain upstream arithmetic.
+    "tinyexp/sail_recon/dependency",
+    "tinyexp/sail_recon/heads",
+    "tinyexp/sail_recon/layers",
+    "tinyexp/sail_recon/models/aggregator.py",
+    "tinyexp/sail_recon/utils",
+]
 
 [tool.ruff.lint]
 select = [

tests/examples/sail_recon/test_atoms.py

@@ -0,0 +1,105 @@
+import importlib
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+import torch
+from PIL import Image
+from plyfile import PlyData
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "third_party" / "sail-recon"))
+
+from sailrecon.layers.attention import Attention as ReferenceAttention
+from sailrecon.models.aggregator import build_allow_block as reference_mask
+from sailrecon.utils.load_fn import load_and_preprocess_images as reference_preprocess
+
+from tinyexp.sail_recon.io import cpu_predictions, save_pointcloud, uniform_sample
+from tinyexp.sail_recon.layers.attention import Attention
+from tinyexp.sail_recon.models.aggregator import build_allow_block
+from tinyexp.sail_recon.models.model import SailRecon
+from tinyexp.sail_recon.utils.load_fn import load_and_preprocess_images
+
+
+@pytest.mark.parametrize("mode", ["crop", "pad"])
+def test_preprocess_mixed_shapes_rgba_grayscale(tmp_path, mode):
+    paths = []
+    rng = np.random.default_rng(10)
+    for i, shape in enumerate(((121, 80, 4), (83, 191, 3), (77, 101))):
+        path = tmp_path / f"{i}.png"
+        Image.fromarray(rng.integers(0, 256, shape, dtype=np.uint8)).save(path)
+        paths.append(str(path))
+    actual = load_and_preprocess_images(paths, mode)
+    expected = reference_preprocess(paths, mode)
+    torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+    assert actual.shape[0:2] == (3, 3)
+    assert all(size % 14 == 0 for size in actual.shape[-2:])
+
+
+def test_selection_and_mask():
+    assert uniform_sample(7, 3) == [0, 2, 4]
+    for total, select in ((0, 0), (3, 4), (3, 0)):
+        with pytest.raises(ValueError):
+            uniform_sample(total, select)
+    torch.testing.assert_close(build_allow_block(4, [0, 1], [2, 3]), reference_mask(4, [0, 1], [2, 3]))
+
+
+@pytest.mark.parametrize("fused", [True, False])
+def test_attention_full_tensor(fused):
+    torch.manual_seed(8)
+    reference = ReferenceAttention(32, num_heads=4, qk_norm=True, fused_attn=fused).eval()
+    candidate = Attention(32, num_heads=4, qk_norm=True, fused_attn=fused).eval()
+    candidate.load_state_dict(reference.state_dict(), strict=True)
+    x = torch.randn(2, 17, 32)
+    torch.testing.assert_close(candidate(x), reference(x), rtol=0, atol=0)
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="KV cache upstream explicitly requires CUDA")
+def test_attention_cache_full_tensor():
+    torch.manual_seed(9)
+    reference = ReferenceAttention(32, num_heads=4, kv_cache=True).eval().cuda()
+    candidate = Attention(32, num_heads=4, kv_cache=True).eval().cuda()
+    candidate.load_state_dict(reference.state_dict())
+    with torch.no_grad():
+        for count in (11, 5, 3):
+            x = torch.randn(1, count, 32, device="cuda")
+            torch.testing.assert_close(candidate(x), reference(x), rtol=0, atol=0)
+        torch.testing.assert_close(candidate.k_cache, reference.k_cache, rtol=0, atol=0)
+    candidate.clear_kv_cache()
+    assert candidate.k_cache is None and candidate.v_cache is None
+
+
+def test_geometry_pose_roundtrip():
+    original = importlib.import_module("sailrecon.utils.pose_enc")
+    replica = importlib.import_module("tinyexp.sail_recon.utils.pose_enc")
+    torch.manual_seed(11)
+    encoding = torch.randn(1, 3, 9)
+    encoding[..., -2:] = 1.0
+    expected = original.pose_encoding_to_extri_intri(encoding, (392, 518))
+    actual = replica.pose_encoding_to_extri_intri(encoding, (392, 518))
+    for x, y in zip(actual, expected):
+        torch.testing.assert_close(x, y, rtol=0, atol=0)
+
+
+def test_empty_cloud_writes_valid_ply(tmp_path):
+    view = {
+        "point_map_by_unprojection": torch.zeros(1, 14, 14, 3),
+        "rgbs": torch.zeros(1, 3, 14, 14),
+        "dpt_cnf": torch.ones(1, 14, 14),
+    }
+    path = tmp_path / "empty.ply"
+    save_pointcloud([view], path)
+    assert len(PlyData.read(path)["vertex"]) == 0
+
+
+def test_numpy_unprojection_boundary():
+    source = [{"points": np.ones((1, 3, 3), dtype=np.float64), "depth": torch.ones(1, 3, 3)}]
+    result = cpu_predictions(source)
+    assert result[0]["points"].dtype == torch.float64
+    assert result[0]["depth"].device.type == "cpu"
+
+
+@pytest.mark.parametrize("shape", [(0, 3, 14, 14), (1, 1, 14, 14), (1, 3, 15, 14)])
+def test_invalid_image_shapes(shape):
+    with pytest.raises(ValueError):
+        SailRecon.validate_images(torch.zeros(shape))

third_party/sail-recon/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 HKUST SAIL-LAB and Horizon Robotics.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

third_party/sail-recon/README.md

@@ -0,0 +1,94 @@
+<div align="center">
+<h1>SAIL-Recon: Large SfM by Augmenting Scene Regression with Localization</h1>
+<h2>2026 3DV Oral</h2>
+<a href="https://arxiv.org/pdf/2508.17972"><img src="https://img.shields.io/badge/arXiv-2508.17972-b31b1b" alt="arXiv"></a>
+<a href="https://hkust-sail.github.io/sail-recon/"><img src="https://img.shields.io/badge/Project_Page-green" alt="Project Page"></a>
+<a href='https://huggingface.co/spaces/HKUST-SAIL/SAIL-Recon'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Demo-blue'></a>
+
+
+**[HKUST Spatial Artificial Intelligence Lab](https://github.com/HKUST-SAIL)**; **[Horizon Robotics](https://en.horizon.auto/)**
+
+
+[Junyuan Deng](https://scholar.google.com/citations?user=KTCPC5IAAAAJ&hl=en), [Heng Li](https://hengli.me/), [Tao Xie](https://github.com/xbillowy), [Weiqiang Ren](https://cn.linkedin.com/in/weiqiang-ren-b2798636), [Qian Zhang](https://cn.linkedin.com/in/qian-zhang-10234b73), [Ping Tan](https://facultyprofiles.hkust.edu.hk/profiles.php?profile=ping-tan-pingtan), [Xiaoyang Guo](https://xy-guo.github.io/)
+</div>
+
+![pic1](docs/traj_ply.png)
+
+
+
+## Overview
+
+Sail-Recon is a feed-forward Transformer that scales neural scene regression to large-scale Structure-from-Motion by augmenting it with visual localization. From a few anchor views, it constructs a global latent scene representation that encodes both geometry and appearance. Conditioned on this representation, the network directly regresses camera poses, intrinsics, depth maps, and scene coordinate maps for thousands of images in minutes, enabling precise and robust reconstruction without iterative optimization.
+
+
+## TODO
+- [x] Inference Code Release
+- [x] Gradio Demo
+- [x] Evaluation Script
+
+## Quick Start
+
+First, clone this repository to your local machine, and install the dependencies (torch, torchvision, numpy, Pillow, and huggingface_hub) following VGGT.
+
+```bash
+git clone https://github.com/HKUST-SAIL/sail-recon.git
+cd sail-recon
+pip install -e .
+```
+
+You can download the demo image (e.g., [Barn](https://drive.google.com/file/d/0B-ePgl6HF260NzQySklGdXZyQzA/view?resourcekey=0-luQ7Jaym5BQL6IjxsgXY9A) from [Tanks & Temples](https://www.tanksandtemples.org/)) and put the images in `examples/demo_image`.
+
+Now, you can try the model demo:
+```bash
+# Images
+python demo.py --img_dir path/to/your/images --out_dir outputs
+# Video
+python demo.py --vid_dir path/to/your/images --out_dir outputs
+```
+
+You can find the ply file and camera pose under `outputs`.
+
+We also provide a Gradio demo for easier usage. You can run the demo by:
+```bash
+python demo_gradio.py
+```
+Please note that the Gradio demo is slower than `demo.py` due to the visualization part.
+
+
+## Evaluation
+
+Please refer to [this](eval/readme.md) for more details.
+
+## Acknowledgements
+
+Thanks to these great repositories:
+
+[ACE0](https://github.com/nianticlabs/acezero) for the PSNR evaluation;
+
+[VGGT](https://github.com/facebookresearch/vggt) for the template of github, gradio and visualization;
+
+[Fast3R](https://github.com/facebookresearch/fast3r) for the training data processing and some utility functions;
+
+And many other inspiring works in the community.
+
+If you find this project useful in your research, please consider citing:
+```bibtex
+@article{dengli2025sail,
+  title={SAIL-Recon: Large SfM by Augmenting Scene Regression with Localization},
+  author={Deng, Junyuan and Li, Heng and Xie, Tao and Ren, Weiqiang and Zhang, Qian and Tan, Ping and Guo, Xiaoyang},
+  journal       = {2026 International Conference on 3D Vision (3DV).},
+  year          = {2026},
+  archivePrefix = {IEEE},
+  primaryClass  = {cs.CV}
+}
+```
+
+## License
+
+See the [LICENSE](./LICENSE.txt) file for details about the license under which this code is made available.
+
+Please see the license of [VGGT](https://github.com/facebookresearch/vggt) about the other code used in this project.
+
+Please see the license of [ACE0](https://github.com/nianticlabs/acezero) about the evaluation used in this project.
+
+Please see the license of [Fast3R](https://github.com/facebookresearch/fast3r) about the utility functions used in this project.

third_party/sail-recon/REVISION

@@ -0,0 +1 @@
+9045ec5ebe3f5b16af62376ee0782907532b5120

third_party/sail-recon/SHA256SUMS.json

@@ -0,0 +1,73 @@
+{
+  "LICENSE.txt": "0480e776e19f1af4c0fb5ac25b2f739741143325b83bce4dfd4a155379bf5ef3",
+  "README.md": "e528a59bbb0cb721c2fa3b71c8388f1e4785c541179f4f07e1c0aa970b4e23f1",
+  "REVISION": "e26779530583d6b9413eebeccb283bf1f5811a1222f0ef9f50ae11c5878ac597",
+  "demo.py": "bf2f2b2058cd1fb2b58cf305192b7f2024f63c7c8d5afefb96ffe63fef5abdb8",
+  "eval/datasets/mip_360.py": "f3c5c1c7c2d47c7ac54a900433461b9ff391928b31ed0732ec6748f408f97106",
+  "eval/datasets/seven_scenes.py": "a4493088cd249f96305f42d0db0dc287d7d7b73eb92eec574a340c7933b3f8f6",
+  "eval/datasets/tnt.py": "d6564f973c5938a8fda483ce550a22deb0bacabeec33d46bf58411a6ae7c65ee",
+  "eval/datasets/tum.py": "c5078b32d666ba909fbbcd7f8df12ee156f442da5da23d72d535df2f9f2bf0eb",
+  "eval/readme.md": "0a880fa57f7b51caa156a893dcd3ec1274e0edfed266a62dc5355f7d0d354edf",
+  "eval/reloc_7scenes.py": "30010b36fe2e5dba6dbf09c27575b7e154f9dcc708a3bb93f7388f6d40921bf6",
+  "eval/reloc_mip.py": "67feaaae3b5233ef4717005df3a07d134bdd8c8923ffbeffa99386ea249102ce",
+  "eval/reloc_tnt.py": "153d9b1744466a160a1b9c3fef221332d05d48ac4a856045faeab03e2d1fe963",
+  "eval/reloc_tum.py": "0a8df25af8569e2cca8420acbcd7064881b259e3a019880d1f9731f68003302d",
+  "eval/run_7scenes.sh": "de2e027e1ad1acc10442a8e743a0a8c05d8d128e947f6af6e5807001cb1b22e5",
+  "eval/run_co3d.sh": "26477aeede026c6052f7772c8bc0fe859e0509af371ca7c8027bffd9bee798ef",
+  "eval/run_mip.sh": "3af542fa734d3c426ccd842420390eaf1d95446ba686ec9a4d4dfd8854620724",
+  "eval/run_tnt.sh": "ffb2367ed0cfe7e356ffd439ac3976070f5dd224fba38d21bcfbebeb5827bd30",
+  "eval/run_tum.sh": "d8444169c0d6b97abaaed60318ebdb8d84ca0f73ff2e214239095dc512382754",
+  "eval/test_co3d.py": "4db2c8c293b121ea630722257f038d15ff864252dd11c58c7663542da2918da0",
+  "eval/utils/cropping.py": "70e639e62244eab3dfa635ed95f74b3ed7e62e8589d4f5642c08a1112746d1ee",
+  "eval/utils/device.py": "50696db36907c8d4d32ec44825426f7346bbd1966a47218ed5e2e868841c8056",
+  "eval/utils/eval_pose_ransac.py": "740e2e6b6116b6105a66b662ddb47f6933ff62a1b7b31e7ee0641b06bc8d8cb7",
+  "eval/utils/eval_utils.py": "16724c2660e483f398dcafd50df292fcbee1bbb262c2d2931442bea2baa822a8",
+  "eval/utils/geometry.py": "689537c8ccdbab1035b143d2b3fe4c23191c37419e859ee61fb1e39b2067e2a1",
+  "eval/utils/image.py": "9b1800eba5f26cf4fc366980fec1bb1754c0f0a179a4330fb95341eaec4994b3",
+  "eval/utils/load_fn.py": "2d99c66c4ac5fdb8127f3149e2d1ca118aebdca3c2e50ef358e0d468e244c002",
+  "eval/utils/misc.py": "3d66a8c4de69bcef79036cd41a3f2c02eb104a080254c6c7c593b6f5b628c024",
+  "eval/utils/pose_enc.py": "90fccb0e33380993c762e937564bb023e8bf0f38ff79e19e6ccaa1c5b83303cd",
+  "eval/utils/rotation.py": "ba95e2b1279361affe60425aadb6665c93089a4251496cd801ce3299f7b93736",
+  "eval/utils/visual_track.py": "538cc598384600fcc703b0e09b0ac203a2b7792c520fa9cfa543538c71b0f87a",
+  "sailrecon/dependency/__init__.py": "6d3e7ebac44553057b13feed1fb768892a593481f53953028d80574646f8762d",
+  "sailrecon/dependency/distortion.py": "3fd881e28fbd233cb7d9279f441d5993abfb7fb983da7c86ad6f16c57a524fde",
+  "sailrecon/dependency/np_to_pycolmap.py": "55d6c6d8df73c2bc9e1a0eb4ddd04eefceee87fc48a0a551d0a4b98e899714c7",
+  "sailrecon/dependency/projection.py": "f00c5d35c8fc25c5f4a78d14efb64a2d9467ef662e46631ccf1dc435bfb94388",
+  "sailrecon/dependency/track_modules/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+  "sailrecon/dependency/track_modules/base_track_predictor.py": "4d6089f0e22547367bc8e56fddab5e7e7b8424c035b0c275f8866aadbc19fc66",
+  "sailrecon/dependency/track_modules/blocks.py": "2e36d29d74cd805df0579bb0410c1d9daff954ba2d2bb8b3f9db8f0d2c309a8e",
+  "sailrecon/dependency/track_modules/modules.py": "cb230ed492ceed88114f2ab3510e738b9519ecd119313dc317373e5ee1226c80",
+  "sailrecon/dependency/track_modules/track_refine.py": "1ccdbecac42c2fa5cdc2d1caa648be53173c67b1bb40efd94776b731687ebf74",
+  "sailrecon/dependency/track_modules/utils.py": "90cf9a6c23c7ae48ed684570fe03dc4c063822b21918af2e13146b9a49098447",
+  "sailrecon/dependency/track_predict.py": "43eb5469aee5655d6bdc70ac2de2c1ab0bae048218bc9c6a1f64756fe1638f0a",
+  "sailrecon/dependency/vggsfm_tracker.py": "d9cb5126977bd2c2be0a92007571d3092700c7476ea27343479e5b48e9c3a61c",
+  "sailrecon/dependency/vggsfm_utils.py": "3c32da5f99f928258df8c77579366980abd260e36bc06a77a9e46b593e2c388f",
+  "sailrecon/heads/camera_head.py": "0042641b1ba504d8d9838470147eeb9c28c2a1fd650357b3262a03b65995ab03",
+  "sailrecon/heads/dpt_head.py": "59cd71ddcdca9d35806bb8b0aac5442f8f4641879c4a29d53d8985042b1ad338",
+  "sailrecon/heads/head_act.py": "095880cb5343ea7ebae55ff7fd9a5c7e798006522b0346180d48ff1d40c032bd",
+  "sailrecon/heads/track_head.py": "2e2a4e705789df655114c61e7311314856aba2780e7e9ec342b2ec8144ee2a19",
+  "sailrecon/heads/track_modules/__init__.py": "3cb62676c3fad403516673b2e99cdb6cbc0b71f814c3bb7f551bfa1e8495e062",
+  "sailrecon/heads/track_modules/base_track_predictor.py": "cfc80b80f2073fd4ef59ca0165a958cd11a436ef2c26cc201336feb9a4e5f94f",
+  "sailrecon/heads/track_modules/blocks.py": "59b0103bfbba107223d70e921a079ed1329e16f17d01393cf004f11ac04334ad",
+  "sailrecon/heads/track_modules/modules.py": "1df89959e1a42dd73ad13ba297c1cb9852d9911c1411f999012345cb33f90452",
+  "sailrecon/heads/track_modules/utils.py": "bee517a319e14d835e811f495b252935c74e40934fe4186c830c5402e141c97b",
+  "sailrecon/heads/utils.py": "b8ae70e7a8c3cb9c7d4ea9f63f9e517b912ec3e41293cbb908e5c614b916b8b6",
+  "sailrecon/layers/__init__.py": "d5534a019f3c53e0cb865dfcf8043ab436fd82554f14fe738943cb0b92c156eb",
+  "sailrecon/layers/attention.py": "1f1165adb161277fda269dda5754f0e5b2ad6c6a00ae79b5ce42339915b09024",
+  "sailrecon/layers/block.py": "4d54cd9b11dc275c585a1b5b5a4bfc132b76cba91a52a1b868feb5c47f8b469e",
+  "sailrecon/layers/drop_path.py": "81471280a70c0282f24f482b4b27656aac652851472a2bd06cf5b2bb44cb1783",
+  "sailrecon/layers/layer_scale.py": "f2d96c65368edae1b7b5390471ba66c7b73e7ae2ff606def1ed5a45ee4fdcc55",
+  "sailrecon/layers/mlp.py": "255825c73b60a916dd00eb1e38aacbcdbf316e40d6a005efb46e245b7edb43aa",
+  "sailrecon/layers/patch_embed.py": "8595e64e2b6b924c23ff6cabae4e1e638f289e97eac397f1e1d3b2704213ec1f",
+  "sailrecon/layers/rope.py": "8b5ca112412150aa4b68856ddb76edd5ae91e5f0d1c801f4a67147b699981de0",
+  "sailrecon/layers/swiglu_ffn.py": "503e87f475fbc4d3b3dac194a74402cf3fdf99181c97071e3487b5ef149d5fd5",
+  "sailrecon/layers/vision_transformer.py": "d77f71c40160406d188945a6196cc5c8a50496277548ca2d6917a0665124c7ce",
+  "sailrecon/models/aggregator.py": "cbb4e739f55d352b3bad59a14b8c662beb296bc4313a80c3c1334c3ce7eb40f8",
+  "sailrecon/models/sail_recon.py": "d3d234e3e495f28d354ce74d0d8c3d81660105efa8b31f2aaf535a210b440455",
+  "sailrecon/utils/geometry.py": "cb09559badda623d9eb3fd442a2ede845813434fbcc7cd611e72bf332585671e",
+  "sailrecon/utils/helper.py": "194a4b6545b4b7223b26160436d8cf49ccb0fff9da0d9e235c34c37034dc5430",
+  "sailrecon/utils/load_fn.py": "accec94c1c0fab052331de32bc2e229fa6ad98bebea5d5788c6a58d5236bde00",
+  "sailrecon/utils/pose_enc.py": "e5956705344e853088fa08ab3a450ed37b819aca9a3819dc23aeb9f0e6e9a427",
+  "sailrecon/utils/rotation.py": "ba95e2b1279361affe60425aadb6665c93089a4251496cd801ce3299f7b93736",
+  "sailrecon/utils/visual_track.py": "538cc598384600fcc703b0e09b0ac203a2b7792c520fa9cfa543538c71b0f87a"
+}