From 0a6c5272f87fe5a8865c74acb217a0e5fea983a6 Mon Sep 17 00:00:00 2001 From: JakimPL Date: Sun, 20 Sep 2026 21:43:48 +0200 Subject: [PATCH] Compressed: the payload a binary document is written as --- docs/formats/instruction-libraries.md | 6 +- docs/formats/projects.md | 3 +- docs/formats/reconstructions.md | 12 ++- src/sampletones_core/data/document.py | 78 +++++++++++++++++ src/sampletones_core/data/model.py | 5 +- src/sampletones_core/data/stored.py | 16 ++-- src/sampletones_core/library/data.py | 3 +- .../reconstruction/reconstruction.py | 6 +- .../sampletones_core/data/test_document.py | 84 +++++++++++++++++++ .../unit/sampletones_core/data/test_stored.py | 24 ++++++ .../reconstruction/test_reconstruction.py | 37 ++++++++ 11 files changed, 255 insertions(+), 19 deletions(-) create mode 100644 src/sampletones_core/data/document.py create mode 100644 tests/unit/sampletones_core/data/test_document.py diff --git a/docs/formats/instruction-libraries.md b/docs/formats/instruction-libraries.md index fedcc4343..6b25856a6 100644 --- a/docs/formats/instruction-libraries.md +++ b/docs/formats/instruction-libraries.md @@ -41,8 +41,10 @@ method mean is covered in [Reconstruction algorithms](../concepts/reconstruction ## File format -Libraries are stored as `.ins` files in the documents folder, with the -configuration embedded in the file name: +Libraries are stored as `.ins` files in the documents folder. A file holds a +deflated [MessagePack](https://msgpack.org/) payload, the framing described in +[Reconstructions](reconstructions.md#storage-and-export), and carries its +configuration in the file name: ``` sr_44100_nf_60_ws_13579_tg_0_sm_cqt_ch_384e710987cb958adf2b214df1267d10.ins diff --git a/docs/formats/projects.md b/docs/formats/projects.md index d099a8e34..41099f438 100644 --- a/docs/formats/projects.md +++ b/docs/formats/projects.md @@ -13,7 +13,8 @@ A `.stp` file is a zip archive with two kinds of member: * **`project.json`** — the project document (below). * **`reconstructions/.stn`** — one [reconstruction](reconstructions.md) per sample, stored as its own `.stn` member and referenced from the document by its - id. + id. The archive deflates its members, so a member carries the reconstruction's + payload as it stands. Keeping the reconstructions in separate members lets `project.json` stay small while the larger audio data travels alongside it in the same archive. A diff --git a/docs/formats/reconstructions.md b/docs/formats/reconstructions.md index 59a551f16..dfaec57cd 100644 --- a/docs/formats/reconstructions.md +++ b/docs/formats/reconstructions.md @@ -114,9 +114,13 @@ The current data version is 2.2. ## Storage and export -`.stn` files live in the documents folder. They are binary -([MessagePack](https://msgpack.org/)) and self-contained: everything needed to -play a reconstruction is the instructions, the stems assignment and the frozen -configuration the file carries. The instruction streams can be exported to a +`.stn` files live in the documents folder. They hold a deflated +[MessagePack](https://msgpack.org/) payload and are self-contained: everything +needed to play a reconstruction is the instructions, the stems assignment and the +frozen configuration the file carries. A payload names every field of every +frame, and a reconstruction holds one frame per channel per frame of audio, so +the names repeat thousands of times over and deflate to a small fraction of the +file. A payload stored plainly reads as it stands, so a file written by an +earlier build opens as it is. The instruction streams can be exported to a tracker — one instrument per channel, or a whole module — as described in [FamiTracker export](famitracker.md) and [Bitphase export](bitphase.md). diff --git a/src/sampletones_core/data/document.py b/src/sampletones_core/data/document.py new file mode 100644 index 000000000..1bfc3d737 --- /dev/null +++ b/src/sampletones_core/data/document.py @@ -0,0 +1,78 @@ +import gzip +from contextlib import contextmanager +from typing import Final, Iterator, Protocol + +from sampletones_shared.types.path import Pathlike + +DOCUMENT_MAGIC: Final[bytes] = b"\x1f\x8b" +COMPRESSION_LEVEL: Final[int] = 9 +STATED_TIMESTAMP: Final[int] = 0 + + +class ByteStream(Protocol): + """A source bytes are read from a piece at a time, which is all a streaming read asks of one.""" + + def read(self, size: int = ...) -> bytes: + """The next ``size`` bytes, or the rest of the stream where the read names no size.""" + + +def compress_document(payload: bytes) -> bytes: + """The bytes a stored document is written as, its payload deflated. + + A document states its fields as names spelled out once per record, which is most of what a + file of thousands of records holds, so the payload deflates to a fraction of its size. The + bytes carry the deflate format's own magic, which tells a stored document apart from the + payload of one written before this framing. + + The timestamp the framing carries is stated rather than taken from the clock, so saving one + document twice writes the same bytes both times. + + Args: + payload: The document's serialized payload. + + Returns: + bytes: The bytes to store. + """ + return gzip.compress(payload, compresslevel=COMPRESSION_LEVEL, mtime=STATED_TIMESTAMP) + + +def decompress_document(stored: bytes) -> bytes: + """The payload a stored document holds. + + A document written before this framing carries its payload as it stands, and reads that way. + + Args: + stored: The bytes read from the document. + + Returns: + bytes: The document's serialized payload. + """ + if stored.startswith(DOCUMENT_MAGIC): + return gzip.decompress(stored) + + return stored + + +@contextmanager +def open_document(path: Pathlike) -> Iterator[ByteStream]: + """Opens the document at ``path`` as a stream over the payload it holds. + + A read that ends at the front of a document reads the front of the file, which is what lets a + library state its header ahead of its entries and be read by that header alone. + + Args: + path: The stored document. + + Yields: + ByteStream: The payload, from its first byte. + """ + with open(path, "rb") as file: + if file.read(len(DOCUMENT_MAGIC)) == DOCUMENT_MAGIC: + file.seek(0) + with gzip.GzipFile(fileobj=file, mode="rb") as stream: + yield stream + + return + + file.seek(0) + yield file diff --git a/src/sampletones_core/data/model.py b/src/sampletones_core/data/model.py index 41e6a36d8..b489e0626 100644 --- a/src/sampletones_core/data/model.py +++ b/src/sampletones_core/data/model.py @@ -21,6 +21,7 @@ import numpy as np from pydantic import BaseModel +from sampletones_core.data.document import compress_document, decompress_document from sampletones_shared.array import to_numpy from sampletones_shared.exceptions import ( DeserializationError, @@ -77,11 +78,11 @@ def deserialize( return cls.deserialize_inner(data, validation, fast=fast) def save(self, path: Pathlike) -> None: - save_binary(path, self.serialize()) + save_binary(path, compress_document(self.serialize())) @classmethod def load(cls, path: Pathlike, fast: bool = True) -> Self: - return cls.deserialize(load_binary(path), fast=fast) + return cls.deserialize(decompress_document(load_binary(path)), fast=fast) @classmethod def _construct(cls, fast: bool = True, **data: Any) -> Self: diff --git a/src/sampletones_core/data/stored.py b/src/sampletones_core/data/stored.py index c6e3093bc..a2ad7495f 100644 --- a/src/sampletones_core/data/stored.py +++ b/src/sampletones_core/data/stored.py @@ -1,7 +1,10 @@ +import gzip +import zlib from typing import FrozenSet import msgpack +from sampletones_core.data.document import open_document from sampletones_shared.types.data import SerializedData from sampletones_shared.types.path import Pathlike @@ -10,10 +13,11 @@ def read_leading_fields(path: Pathlike, names: FrozenSet[str]) -> SerializedData """The named top-level fields of a stored document, read from the front of its file. A stored :class:`DataModel` is one map whose fields follow the model's declaration order, so - the fields a model declares first sit in the first bytes of the file. The read decodes those + the fields a model declares first sit in the first bytes of its payload. The read decodes those fields and steps over any other in its way, and it ends once every named field is found, which - keeps it to the front of the file however large the rest is. A file whose front decodes as no - map reads as holding none of the fields. + keeps it to the front of the payload however large the rest is. A payload whose front decodes + as no map reads as holding none of the fields, a file whose framing the read cannot follow + included. Args: path: The stored document. @@ -23,8 +27,8 @@ def read_leading_fields(path: Pathlike, names: FrozenSet[str]) -> SerializedData SerializedData: Each named field the file holds, as stored. """ found: SerializedData = {} - with open(path, "rb") as file: - unpacker = msgpack.Unpacker(file, raw=False) + with open_document(path) as payload: + unpacker = msgpack.Unpacker(payload, raw=False) try: for _ in range(unpacker.read_map_header()): name = unpacker.unpack() @@ -35,7 +39,7 @@ def read_leading_fields(path: Pathlike, names: FrozenSet[str]) -> SerializedData found[name] = unpacker.unpack() if len(found) == len(names): break - except (ValueError, msgpack.OutOfData): + except (ValueError, msgpack.OutOfData, EOFError, gzip.BadGzipFile, zlib.error): return {} return found diff --git a/src/sampletones_core/library/data.py b/src/sampletones_core/library/data.py index c62d07e63..7e0b390a4 100644 --- a/src/sampletones_core/library/data.py +++ b/src/sampletones_core/library/data.py @@ -9,6 +9,7 @@ from sampletones_core.configs import Config, InstructionsLibraryConfig from sampletones_core.constants.enums import GeneratorClassName from sampletones_core.data import DataModel, Metadata, MetadataContract +from sampletones_core.data.document import decompress_document from sampletones_core.generators import GeneratorClassNames from sampletones_core.instructions import InstructionUnion from sampletones_shared.application import SAMPLETONES_LIBRARY_DATA_VERSION @@ -117,7 +118,7 @@ def load(cls, path: Pathlike, fast: bool = True) -> InstructionLibraryData: try: return InstructionLibraryData.deserialize( - binary, + decompress_document(binary), validation=cls.validate_metadata, fast=fast, ) diff --git a/src/sampletones_core/reconstructions/reconstruction/reconstruction.py b/src/sampletones_core/reconstructions/reconstruction/reconstruction.py index a5db9948d..1ee1a1a45 100644 --- a/src/sampletones_core/reconstructions/reconstruction/reconstruction.py +++ b/src/sampletones_core/reconstructions/reconstruction/reconstruction.py @@ -28,6 +28,7 @@ from sampletones_core.constants.algorithm import RESTING_STEM_ID from sampletones_core.constants.enums import ChannelName, FeatureKey from sampletones_core.data import DataModel, Metadata, MetadataContract +from sampletones_core.data.document import decompress_document from sampletones_core.exporters import ( CHANNEL_TO_EXPORTER_MAP, INSTRUCTION_TO_EXPORTER_MAP, @@ -436,9 +437,8 @@ def _invalidate_derived_caches(reconstruction: Reconstruction) -> None: @classmethod def load(cls, path: Pathlike, fast: bool = True) -> Reconstruction: - binary = load_binary(path) return cls.deserialize_data( - binary, + load_binary(path), source=Path(path), validation=cls.validate_metadata, fast=fast, @@ -453,7 +453,7 @@ def deserialize_data( fast: bool = True, ) -> Reconstruction: try: - binary = upgrade_binary(ObjectKind.RECONSTRUCTION, binary) + binary = upgrade_binary(ObjectKind.RECONSTRUCTION, decompress_document(binary)) return cls.deserialize(binary, validation=validation, fast=fast) except (ValidationError, TypeError, ValueError, struct.error, IndexError) as exception: raise InvalidReconstructionValuesError( diff --git a/tests/unit/sampletones_core/data/test_document.py b/tests/unit/sampletones_core/data/test_document.py new file mode 100644 index 000000000..334fc6b60 --- /dev/null +++ b/tests/unit/sampletones_core/data/test_document.py @@ -0,0 +1,84 @@ +from pathlib import Path +from typing import Final + +import msgpack +import pytest + +from sampletones_core.data.document import ( + DOCUMENT_MAGIC, + compress_document, + decompress_document, + open_document, +) + +RECORDS: Final[int] = 512 +FIELD: Final[str] = "leading" + + +def _payload() -> bytes: + """A payload of the shape a stored document holds: one map per record, keyed by name.""" + records = [{"on": index % 2 == 0, "pitch": 33, "volume": index % 16} for index in range(RECORDS)] + return bytes(msgpack.packb({FIELD: "2.2", "records": records}, use_bin_type=True)) + + +class TestWhatAStoredDocumentHolds: + def test_a_payload_comes_back_as_it_was_written(self) -> None: + payload = _payload() + + assert decompress_document(compress_document(payload)) == payload + + def test_a_payload_written_before_the_framing_reads_as_it_stands(self) -> None: + payload = _payload() + + assert decompress_document(payload) == payload + + def test_the_stored_bytes_carry_the_framing_magic(self) -> None: + assert compress_document(_payload()).startswith(DOCUMENT_MAGIC) + + def test_the_repeated_field_names_deflate_away(self) -> None: + payload = _payload() + + assert len(compress_document(payload)) < len(payload) + + def test_storing_one_payload_twice_writes_the_same_bytes(self) -> None: + """The framing states its timestamp, so a document's bytes follow from its payload alone.""" + payload = _payload() + + assert compress_document(payload) == compress_document(payload) + + def test_an_empty_payload_survives_the_round_trip(self) -> None: + assert decompress_document(compress_document(b"")) == b"" + + +class TestReadingADocumentAsAStream: + def test_a_stored_document_streams_as_its_payload(self, tmp_path: Path) -> None: + payload = _payload() + path = tmp_path / "document.bin" + path.write_bytes(compress_document(payload)) + + with open_document(path) as stream: + assert stream.read() == payload + + def test_a_document_written_before_the_framing_streams_as_it_stands(self, tmp_path: Path) -> None: + payload = _payload() + path = tmp_path / "document.bin" + path.write_bytes(payload) + + with open_document(path) as stream: + assert stream.read() == payload + + def test_the_stream_reaches_the_front_without_the_rest(self, tmp_path: Path) -> None: + payload = _payload() + path = tmp_path / "document.bin" + path.write_bytes(compress_document(payload)) + + with open_document(path) as stream: + unpacker = msgpack.Unpacker(stream, raw=False) + unpacker.read_map_header() + + assert unpacker.unpack() == FIELD + + def test_a_missing_document_is_reported(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + with open_document(tmp_path / "absent.bin"): + pass diff --git a/tests/unit/sampletones_core/data/test_stored.py b/tests/unit/sampletones_core/data/test_stored.py index d05c24715..e2b47c47f 100644 --- a/tests/unit/sampletones_core/data/test_stored.py +++ b/tests/unit/sampletones_core/data/test_stored.py @@ -3,6 +3,7 @@ import msgpack +from sampletones_core.data.document import compress_document from sampletones_core.data.stored import read_leading_fields LEADING: Final[str] = "leading" @@ -39,6 +40,29 @@ def test_the_read_ends_once_every_named_field_is_found(self, tmp_path: Path) -> assert read_leading_fields(path, frozenset({LEADING, FOLLOWING})) == {LEADING: 1, FOLLOWING: 2} +class TestAStoredDocumentsFraming: + def test_the_named_fields_read_through_the_framing(self, tmp_path: Path) -> None: + document = {LEADING: {"version": "2.1"}, FOLLOWING: [1, 2], TRAILING: b"\x00" * 4096} + payload = msgpack.packb(document, use_bin_type=True) + path = _stored(tmp_path / "document.bin", compress_document(payload)) + + fields = read_leading_fields(path, frozenset({LEADING, FOLLOWING})) + + assert fields == {LEADING: document[LEADING], FOLLOWING: document[FOLLOWING]} + + def test_a_document_whose_framing_is_cut_short_holds_no_fields(self, tmp_path: Path) -> None: + stored = compress_document(msgpack.packb({LEADING: "x" * 4096}, use_bin_type=True)) + path = _stored(tmp_path / "document.bin", stored[:24]) + + assert not read_leading_fields(path, frozenset({LEADING})) + + def test_a_document_whose_framing_is_damaged_holds_no_fields(self, tmp_path: Path) -> None: + stored = compress_document(msgpack.packb({LEADING: "x" * 4096}, use_bin_type=True)) + path = _stored(tmp_path / "document.bin", stored[:20] + b"\xff" * 256) + + assert not read_leading_fields(path, frozenset({LEADING})) + + class TestAFrontThatDecodesAsNoMap: def test_an_empty_file_holds_no_fields(self, tmp_path: Path) -> None: path = _stored(tmp_path / "document.bin", b"") diff --git a/tests/unit/sampletones_core/reconstructions/reconstruction/test_reconstruction.py b/tests/unit/sampletones_core/reconstructions/reconstruction/test_reconstruction.py index df602226d..62ee9c2ab 100644 --- a/tests/unit/sampletones_core/reconstructions/reconstruction/test_reconstruction.py +++ b/tests/unit/sampletones_core/reconstructions/reconstruction/test_reconstruction.py @@ -17,6 +17,7 @@ bending_channels, ) from sampletones_core.data import Metadata +from sampletones_core.data.document import DOCUMENT_MAGIC from sampletones_core.features import resting_held_features, resting_reference from sampletones_core.instructions import PulseInstruction from sampletones_core.reconstructions import Reconstruction @@ -264,6 +265,30 @@ def test_detached_source_round_trips_as_empty( assert loaded.audio_filepath == () + def test_the_stored_file_carries_the_framing( + self, + tmp_path: Path, + reconstruction_factory: ReconstructionFactory, + ) -> None: + reconstruction = reconstruction_factory() + path = tmp_path / "framed.stn" + + reconstruction.save(path) + + assert path.read_bytes().startswith(DOCUMENT_MAGIC) + assert path.stat().st_size < len(reconstruction.serialize()) + + def test_a_file_written_before_the_framing_still_loads( + self, + tmp_path: Path, + reconstruction_factory: ReconstructionFactory, + ) -> None: + reconstruction = reconstruction_factory() + path = tmp_path / "plain.stn" + path.write_bytes(reconstruction.serialize()) + + assert Reconstruction.load(path).id == reconstruction.id + class TestDetachSource: def test_detach_clears_the_source_location( @@ -293,6 +318,18 @@ def test_corrupt_binary_raises_invalid_values(self) -> None: source="corrupt.stn", ) + def test_a_damaged_framing_raises_a_load_error( + self, + tmp_path: Path, + reconstruction_factory: ReconstructionFactory, + ) -> None: + path = tmp_path / "damaged.stn" + reconstruction_factory().save(path) + path.write_bytes(path.read_bytes()[:32] + b"\xff" * 256) + + with pytest.raises(LoadReconstructionError): + Reconstruction.load(path) + class TestLoadFileAccess(BaseTestSuite): @dataclass(frozen=True, kw_only=True)