codegen: add a table-driven codec strategy

.changes/unreleased/added-20260923-codec-strategy-table.yaml

@@ -0,0 +1,4 @@
+kind: Added
+body: |-
+    **Table-driven message codec** (#469, refs #463). `buffa_build::Config::codec_strategy(CodecStrategy::Table)` (plugin option `codec_strategy=table`, `CodeGenConfig::codec_strategy`) generates each message's binary `Message` implementation from a static table and interpreters that every message shares, instead of code specialised to the message's fields. It roughly halves the compiled size of a large schema at `opt-level = "z"`, and messages made of many small fields are slower. `CodecStrategy::Unrolled` stays the default. `codec_strategy_in(strategy, &[paths])` (plugin option `codec_strategy_in=<path>=<strategy>`, repeatable) chooses the strategy for matching messages and the messages nested in them, and the last matching rule wins. The wire format does not change. A message that has a `oneof`, `map`, or group field, or holds a message that is not a table, stays unrolled, and `CodeGenWarning::TableCodecFallbackSummary` counts those; a rule that names such a message by its exact path is an error. The generated code needs Rust 1.77 or later, which `buffa-build` checks, and compiles in a crate with `#![forbid(unsafe_code)]`; see the guide's "Smaller generated code" section for the rest.
+time: 2026-09-23T03:10:00+00:00

.github/workflows/ci.yml

@@ -198,7 +198,8 @@ jobs:
   # ── MSRV check ───────────────────────────────────────────────────────────
   msrv-check:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    # Two toolchains, and a test build on the second.
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
 
@@ -219,6 +220,13 @@ jobs:
         with:
           toolchain: '1.75'
 
+      # The table codec needs `offset_of!`, stable in 1.77; the workspace
+      # check above compiles it out at 1.75. This is the oldest compiler that
+      # builds the table interpreters and the generated table code.
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: '1.77'
+
       - name: Install protoc
         run: |
           curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" -o /tmp/protoc.zip
@@ -235,6 +243,13 @@ jobs:
           RUSTUP_TOOLCHAIN: '1.75'
         run: cargo check --workspace --all-targets --locked
 
+      - name: Test the table codec (Rust 1.77)
+        env:
+          RUSTUP_TOOLCHAIN: '1.77'
+        run: |
+          cargo test -p buffa --lib table:: --locked
+          cargo test -p buffa-test --lib table_codec --locked
+
   # ── Miri: proves the `set_len`-publishing `unsafe` reads no uninit memory ──
   # SizeCache stores its inline slots as `MaybeUninit<u32>` to avoid zeroing the
   # whole array on every encode; `consume_next` reads them via `assume_init`.

CONTRIBUTING.md

@@ -154,7 +154,7 @@ GitHub Actions CI (`.github/workflows/ci.yml`) runs on every push to `main` and
 
 - **lint-and-test** — clippy + strict rustdoc (`task doc` locally) + `cargo test --workspace` on stable
 - **lint-markdown** — markdownlint over all `*.md` (config: `.markdownlint.json`)
-- **msrv-check** — `cargo check --workspace` on Rust 1.75 (the declared `rust-version`)
+- **msrv-check** — `cargo check --workspace` on Rust 1.75 (the declared `rust-version`), then the table codec's tests on Rust 1.77, the oldest compiler that builds it
 - **check-nostd** — no_std (host + bare-metal ARM) and 32-bit compilation checks
 - **check-generated-code** — regenerates bootstrap descriptor types and fails if the checked-in code is stale
 - **conformance** — builds the tools and conformance Docker images, runs the full protobuf conformance suite

DESIGN.md

@@ -36,6 +36,7 @@ Buffa fills this gap: a pure Rust implementation designed from the ground up wit
 The runtime library that generated code depends on. Contains:
 
 - **`Message` trait**: The central trait for owned message types, with two-pass `compute_size()` / `write_to()` serialization.
+- **`table`** (hidden): the interpreters behind table-driven message codecs; see [decision 13](#13-table-driven-codec-codecstrategytable).
 - **`MessageView` trait**: The trait for borrowed/zero-copy message views.
 - **`OwnedView<V>`**: Self-referential container that pairs a `Bytes` buffer with a decoded view, producing a `'static + Send + Sync` type suitable for async and RPC frameworks.
 - **`MessageField<T>`**: Ergonomic wrapper for optional message fields that dereferences to a default instance when unset.
@@ -555,6 +556,18 @@ Generating the *same* `.proto` in two crates without `extern_path`, and choosing
 
 Within that bound, the shortcuts do not pay off. `merge` does not help: it consumes wire bytes, so a `Foo` must still be encoded first, and merging only reuses the target allocation rather than avoiding the round-trip. Reflection does not help today either — `a::Foo` can be read reflectively into a `DynamicMessage` (which erases the representation, since `ValueRef::String` is `&str`), but `ReflectMessageMut` is implemented only on `DynamicMessage`, not on generated types, so the return leg falls back to encode/decode. The theoretically cheapest conversion is a static field-by-field `impl From<a::Foo> for b::Foo` — no varint, no buffer, no dynamic dispatch, only the unavoidable destination allocations — but it must name both types, because Rust has no structural typing on which to hang a generic conversion. The recommendation is therefore to avoid the divergence with `extern_path`, and where a genuine two-representation boundary exists, to hand-write the `From` rather than reach for a wire round-trip or new machinery.
 
+### 13. Table-Driven Codec (`CodecStrategy::Table`)
+
+The size, write, and merge code of an unrolled message is specialised to its fields. `CodecStrategy::Table` (a per-message option, `Unrolled` by default) replaces it with a static `buffa::table::Table<M>` and a `Message` impl that forwards to interpreters in `buffa::table`. The measurements and the decision to keep `Unrolled` as the default are in [#463](https://github.com/anthropics/buffa/issues/463).
+
+A table holds a sorted array of 12-byte entries `{tag, offset, kind, tag_len, aux}`, a dense array that maps field numbers below 64 to entries, and the offset of the unknown-fields slot. `kind` is the field type crossed with its cardinality, so the interpreter dispatches once per field. Message, repeated-message, and enum fields carry a small descriptor (`Aux`) with the accessors that their storage needs, because a `MessageField`, a `Vec`, and an `EnumValue` cannot be read through an offset alone. Offsets come from `core::mem::offset_of!`, so the table needs Rust 1.77 and the generated code refers to it through `buffa::__table!`, which is a compile error on an older compiler.
+
+Three decisions shape the runtime:
+
+- **The `unsafe` lives in `buffa`.** `__table!` and `__table_entry!` contain the `unsafe` blocks and witness each field's type against its kind, so a table that names the wrong kind for a field does not compile, and generated code compiles under `#![forbid(unsafe_code)]`. `Table::new` also checks the layout constants at compile time. The interpreters run under Miri in CI.
+- **The interpreters are not generic over the sink or the input where that is avoidable.** `Message::encode` and its siblings write any `BufMut` through one shared, non-generic cursor (`buffa/src/encode_sink.rs`), and decoding runs over a contiguous `&[u8]`, so the interpreters are compiled once in `buffa`, at its `opt-level`, and not once per caller.
+- **A table refers to the tables of its children,** so a message can use the table only if every message it holds does. The planner in `buffa-codegen` (`table_plan.rs`) starts from the messages that asked for the table, removes those the interpreters cannot handle (oneofs, maps, groups and the types of group fields, `MessageSet`, extension ranges with JSON, custom string, bytes, or collection types), and then removes every message that holds a removed one, until none is left. A message that holds a type from another crate, such as a well-known type, is removed the same way, because the static table of that type is not visible.
+
 ### Owned decode: intentional throughput trade-offs
 
 Owned decode (`Message::decode_from_slice`) benchmarks within roughly ±10% of prost in most cases. The costs are intentional and attributable to specific features:

README.md

@@ -390,7 +390,7 @@ Compatibility is tested against protoc v21.12, v22.5, v25.5, v27.3, v29.5, and v
 
 ## Minimum supported Rust version
 
-The current MSRV is **1.75**.
+The current MSRV is **1.75**. The opt-in `CodecStrategy::Table` generates code that needs Rust 1.77 (see [Smaller generated code](docs/guide.md#smaller-generated-code-codec_strategy)); `buffa-build` returns an error on an older compiler when it is requested.
 
 buffa is a foundational codec crate, so its `rust-version` is set to the lowest toolchain the released code actually compiles on, not to a calendar target. CI verifies the workspace builds at the MSRV and at stable on every change. With cargo's MSRV-aware resolver (`resolver = "3"`, Rust 1.84+), a downstream project on an older toolchain will automatically resolve to the newest buffa release whose `rust-version` fits — so an accurate declaration matters more than a conservative one.
 

buffa-build/src/lib.rs

@@ -39,7 +39,7 @@ pub use buffa_codegen::FeatureGateNames;
 #[doc(inline)]
 pub use buffa_codegen::ReflectMode;
 #[doc(inline)]
-pub use buffa_codegen::{BytesRepr, MapRepr, PointerRepr, RepeatedRepr, StringRepr};
+pub use buffa_codegen::{BytesRepr, CodecStrategy, MapRepr, PointerRepr, RepeatedRepr, StringRepr};
 #[doc(inline)]
 pub use buffa_codegen::{EnumTypeOverride, FeatureOverride};
 
@@ -1446,6 +1446,81 @@ impl Config {
         self
     }
 
+    /// Choose how the binary `Message` implementation of every message is
+    /// generated (default: [`CodecStrategy::Unrolled`]).
+    ///
+    /// On a schema it fully covers, [`CodecStrategy::Table`] makes the
+    /// compiled size about half as big at `opt-level = "z"`, and it slows
+    /// messages made of many small fields; a message that cannot use it keeps
+    /// its size. [`CodecStrategy::Table`] has the measurements, says which
+    /// messages stay unrolled, and lists how a table message behaves
+    /// differently. This build reports
+    /// those in one `cargo:warning`. The option never changes the wire
+    /// format. The generated code needs Rust 1.77 or later, and `compile`
+    /// returns an error on an older compiler when a build script runs it (the
+    /// compiler is read from `RUSTC`).
+    ///
+    /// Path-scoped rules for individual messages go in
+    /// [`codec_strategy_in`](Self::codec_strategy_in), and take precedence
+    /// over this setting whatever the call order.
+    ///
+    /// ```rust,ignore
+    /// // build.rs
+    /// buffa_build::Config::new()
+    ///     .files(&["proto/wa.proto"])
+    ///     .includes(&["proto/"])
+    ///     .codec_strategy(buffa_build::CodecStrategy::Table)
+    ///     .codec_strategy_in(buffa_build::CodecStrategy::Unrolled, &[".wa.Message"])
+    ///     .compile()?;
+    /// ```
+    #[must_use]
+    pub fn codec_strategy(mut self, strategy: CodecStrategy) -> Self {
+        self.codegen_config.codec_strategy = strategy;
+        self
+    }
+
+    /// Choose the [`CodecStrategy`] of the matching messages, on top of the
+    /// global [`codec_strategy`](Self::codec_strategy).
+    ///
+    /// Each path is a fully-qualified proto path prefix, e.g. `".wa.Message"`
+    /// for one message or `".wa"` for a package (same matching as
+    /// [`preserve_unknown_fields_in`](Self::preserve_unknown_fields_in));
+    /// `"."` matches every message. A leading dot is added if missing,
+    /// trailing dots are trimmed, and a path that is empty after that prints a
+    /// `cargo:warning` and is ignored. A rule covers the message it names
+    /// **and every message nested inside it**. The **last** matching rule wins,
+    /// so call this after any broader rule.
+    ///
+    /// A message selected for the table by a rule that cannot use it stays
+    /// unrolled and is counted in the warning, and it is an error if the rule
+    /// names the message by its exact path. A rule that matches no message
+    /// produces a warning.
+    ///
+    /// A table message holds only table messages, and a rule does not extend
+    /// to the messages a message holds. Selecting a message with a rule
+    /// therefore also needs rules for everything it holds, unless the global
+    /// setting is [`CodecStrategy::Table`]. Choosing [`CodecStrategy::Unrolled`]
+    /// for a message keeps every message that holds it unrolled, with no
+    /// warning, and that usually includes the root message an application
+    /// encodes.
+    #[must_use]
+    pub fn codec_strategy_in(mut self, strategy: CodecStrategy, paths: &[impl AsRef<str>]) -> Self {
+        for raw in paths.iter().map(AsRef::as_ref) {
+            let normalized = normalize_override_path(raw);
+            if normalized.is_empty() {
+                println!(
+                    "cargo:warning=buffa: codec_strategy_in path '{raw}' \
+                     normalizes to empty and will be ignored"
+                );
+                continue;
+            }
+            self.codegen_config
+                .codec_strategy_in
+                .push((normalized, strategy));
+        }
+        self
+    }
+
     /// Map every message field (and boxed oneof variant) to the given [`PointerRepr`].
     /// Convenience for `.box_type_in(repr, &["."])`. Call before any
     /// [`box_type_in`](Self::box_type_in) overrides, since the last matching
@@ -1905,9 +1980,35 @@ impl Config {
     ///   missing imports, etc.)
     /// - a precompiled descriptor set file cannot be read
     /// - the descriptor set bytes cannot be decoded as a `FileDescriptorSet`
-    /// - code generation fails (e.g. unsupported proto feature)
+    /// - code generation fails (e.g. unsupported proto feature), including a
+    ///   [`codec_strategy_in`](Self::codec_strategy_in) rule that names by its
+    ///   exact path a message that cannot use the table codec
+    /// - [`CodecStrategy::Table`] is requested and the compiler is older than
+    ///   Rust 1.77
     /// - the output directory cannot be created or written to
     pub fn compile(self) -> Result<(), Box<dyn std::error::Error>> {
+        // Table code needs `offset_of!`, stable in Rust 1.77. Say so once here,
+        // and not once per field in the compiler's output.
+        let table_codec_requested = self.codegen_config.codec_strategy == CodecStrategy::Table
+            || self
+                .codegen_config
+                .codec_strategy_in
+                .iter()
+                .any(|(_, strategy)| *strategy == CodecStrategy::Table);
+        // Outside a build script `RUSTC` is unset, and the toolchain that
+        // compiles the generated code is unknown, so it is not checked.
+        if table_codec_requested {
+            if let Some(minor) = rustc_minor_version() {
+                if minor < 77 {
+                    return Err(format!(
+                        "CodecStrategy::Table needs Rust 1.77 or later, and this build uses \
+                         1.{minor}; use a newer compiler or select CodecStrategy::Unrolled"
+                    )
+                    .into());
+                }
+            }
+        }
+
         // Reject malformed `exclude_package` entries before protoc runs; the
         // codegen normalizes them again, but a typo should not cost a protoc
         // invocation to surface.
@@ -2158,6 +2259,29 @@ fn normalize_attr_path(mut path: String) -> String {
     path
 }
 
+/// The minor version of the compiler that builds the crate, if `RUSTC` names
+/// one and it can be read.
+fn rustc_minor_version() -> Option<u32> {
+    let rustc = std::env::var_os("RUSTC")?;
+    let output = std::process::Command::new(rustc)
+        .arg("--version")
+        .output()
+        .ok()?;
+    parse_rustc_minor_version(&String::from_utf8_lossy(&output.stdout))
+}
+
+/// The minor version in the output of `rustc --version`, such as `75` in
+/// `rustc 1.75.0 (82e1608df 2023-12-21)`.
+fn parse_rustc_minor_version(version_output: &str) -> Option<u32> {
+    version_output
+        .split_whitespace()
+        .nth(1)?
+        .split('.')
+        .nth(1)?
+        .parse()
+        .ok()
+}
+
 /// Normalize an `override_feature_in` / `preserve_unknown_fields_in` /
 /// `deny_unknown_json_fields_in` path:
 /// trim whitespace, prepend the leading dot if absent, and strip trailing
@@ -2590,6 +2714,46 @@ mod tests {
         );
     }
 
+    #[test]
+    fn codec_strategy_in_normalizes_paths_and_keeps_rule_order() {
+        let config = Config::new()
+            .codec_strategy(CodecStrategy::Table)
+            .codec_strategy_in(CodecStrategy::Unrolled, &["my.pkg.Msg", ".my.pkg.Other."])
+            .codec_strategy_in(CodecStrategy::Table, &[" .my.pkg.Msg.Inner ", "."])
+            .codegen_config;
+        assert_eq!(config.codec_strategy, CodecStrategy::Table);
+        assert_eq!(
+            config.codec_strategy_in,
+            vec![
+                (".my.pkg.Msg".to_string(), CodecStrategy::Unrolled),
+                (".my.pkg.Other".to_string(), CodecStrategy::Unrolled),
+                (".my.pkg.Msg.Inner".to_string(), CodecStrategy::Table),
+                (".".to_string(), CodecStrategy::Table),
+            ]
+        );
+    }
+
+    #[test]
+    fn rustc_minor_version_is_read_from_the_version_line() {
+        assert_eq!(
+            parse_rustc_minor_version("rustc 1.75.0 (82e1608df 2023-12-21)"),
+            Some(75)
+        );
+        assert_eq!(
+            parse_rustc_minor_version("rustc 1.98.0-nightly (abc 2027-01-01)\n"),
+            Some(98)
+        );
+        assert_eq!(parse_rustc_minor_version(""), None);
+        assert_eq!(parse_rustc_minor_version("not rustc"), None);
+    }
+
+    #[test]
+    fn codec_strategy_defaults_to_unrolled() {
+        let config = Config::new().codegen_config;
+        assert_eq!(config.codec_strategy, CodecStrategy::Unrolled);
+        assert!(config.codec_strategy_in.is_empty());
+    }
+
     #[test]
     fn deny_unknown_json_fields_in_normalizes_paths() {
         let config = Config::new()