From bd9b59ee35baa7103125d5eb2851c4a8f5e7ff19 Mon Sep 17 00:00:00 2001 From: Iain McGinniss <309153+iainmcgin@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:28:37 -0700 Subject: [PATCH 1/5] codegen: add a table-driven codec strategy `CodecStrategy::Table` generates a message's binary `Message` impl from a static `buffa::table::Table` and the shared interpreters, instead of code specialised to its fields. `CodecStrategy::Unrolled` stays the default. The strategy is set globally with `Config::codec_strategy` (plugin option `codec_strategy=`) and per message with `Config::codec_strategy_in` (plugin option `codec_strategy_in==`, repeatable). Paths use the same prefix matching as the other `_in` rules, and the last matching rule wins. A table refers to the tables of its children, so a planner selects the largest set of requested messages that can use it: oneofs, maps, groups, custom representations, and messages that hold a message outside the set stay unrolled. A rule that selects such a message warns, the global setting produces one summary warning, and a rule that names the message by its exact path is an error. buffa-test compiles the same schemas with both strategies and checks that they encode to the same bytes and decode to the same values, including on truncated, bit-flipped, and random input. --- .../added-20260923-codec-strategy-table.yaml | 4 + .github/workflows/ci.yml | 17 +- CONTRIBUTING.md | 2 +- DESIGN.md | 13 + README.md | 2 +- buffa-build/src/lib.rs | 168 ++- buffa-codegen/src/context.rs | 83 ++ buffa-codegen/src/features.rs | 20 +- buffa-codegen/src/impl_message.rs | 114 +- buffa-codegen/src/lib.rs | 186 +++- buffa-codegen/src/message.rs | 17 +- buffa-codegen/src/table_codec.rs | 279 +++++ buffa-codegen/src/table_plan.rs | 529 ++++++++++ buffa-codegen/src/tests/mod.rs | 1 + buffa-codegen/src/tests/table_codec.rs | 579 ++++++++++ buffa-test/build.rs | 184 ++++ buffa-test/protos/table_codec.proto | 187 ++++ buffa-test/protos/table_codec2.proto | 79 ++ buffa-test/protos/table_codec3.proto | 38 + buffa-test/src/lib.rs | 89 ++ buffa-test/src/tests/mod.rs | 2 + buffa-test/src/tests/table_codec.rs | 985 ++++++++++++++++++ docs/guide.md | 42 + protoc-gen-buffa/src/main.rs | 67 +- 24 files changed, 3622 insertions(+), 65 deletions(-) create mode 100644 .changes/unreleased/added-20260923-codec-strategy-table.yaml create mode 100644 buffa-codegen/src/table_codec.rs create mode 100644 buffa-codegen/src/table_plan.rs create mode 100644 buffa-codegen/src/tests/table_codec.rs create mode 100644 buffa-test/protos/table_codec.proto create mode 100644 buffa-test/protos/table_codec2.proto create mode 100644 buffa-test/protos/table_codec3.proto create mode 100644 buffa-test/src/tests/table_codec.rs diff --git a/.changes/unreleased/added-20260923-codec-strategy-table.yaml b/.changes/unreleased/added-20260923-codec-strategy-table.yaml new file mode 100644 index 00000000..d2909f92 --- /dev/null +++ b/.changes/unreleased/added-20260923-codec-strategy-table.yaml @@ -0,0 +1,4 @@ +kind: Added +body: |- + **Table-driven message codec** (#NNN, refs #463). `buffa_build::Config::codec_strategy(CodecStrategy::Table)` (plugin option `codec_strategy=table`, `CodeGenConfig::codec_strategy`) generates each message's binary `Message` implementation from a static table and interpreters that every message shares, instead of code specialised to the message's fields. It roughly halves the compiled size of a large schema at `opt-level = "z"`, and messages made of many small fields are slower. `CodecStrategy::Unrolled` stays the default. `codec_strategy_in(strategy, &[paths])` (plugin option `codec_strategy_in==`, repeatable) chooses the strategy for matching messages and the messages nested in them, and the last matching rule wins. The wire format does not change. A message that has a `oneof`, `map`, or group field, or holds a message that is not a table, stays unrolled, and `CodeGenWarning::TableCodecFallbackSummary` counts those; a rule that names such a message by its exact path is an error. The generated code needs Rust 1.77 or later, which `buffa-build` checks, and compiles in a crate with `#![forbid(unsafe_code)]`; see the guide's "Smaller generated code" section for the rest. +time: 2026-09-23T03:10:00+00:00 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04469e81..91f16c97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,7 +200,8 @@ jobs: # ── MSRV check ─────────────────────────────────────────────────────────── msrv-check: runs-on: ubuntu-latest - timeout-minutes: 10 + # Two toolchains, and a test build on the second. + timeout-minutes: 20 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 (sha-pinned) @@ -221,6 +222,13 @@ jobs: with: toolchain: '1.75' + # The table codec needs `offset_of!`, stable in 1.77; the workspace + # check above compiles it out at 1.75. This is the oldest compiler that + # builds the table interpreters and the generated table code. + - uses: dtolnay/rust-toolchain@master + with: + toolchain: '1.77' + - name: Install protoc run: | curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" -o /tmp/protoc.zip @@ -237,6 +245,13 @@ jobs: RUSTUP_TOOLCHAIN: '1.75' run: cargo check --workspace --all-targets --locked + - name: Test the table codec (Rust 1.77) + env: + RUSTUP_TOOLCHAIN: '1.77' + run: | + cargo test -p buffa --lib table:: --locked + cargo test -p buffa-test --lib table_codec --locked + # ── Miri: proves the `set_len`-publishing `unsafe` reads no uninit memory ── # SizeCache stores its inline slots as `MaybeUninit` to avoid zeroing the # whole array on every encode; `consume_next` reads them via `assume_init`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 612856be..5b3d8594 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -154,7 +154,7 @@ GitHub Actions CI (`.github/workflows/ci.yml`) runs on every push to `main` and - **lint-and-test** — clippy + strict rustdoc (`task doc` locally) + `cargo test --workspace` on stable - **lint-markdown** — markdownlint over all `*.md` (config: `.markdownlint.json`) -- **msrv-check** — `cargo check --workspace` on Rust 1.75 (the declared `rust-version`) +- **msrv-check** — `cargo check --workspace` on Rust 1.75 (the declared `rust-version`), then the table codec's tests on Rust 1.77, the oldest compiler that builds it - **check-nostd** — no_std (host + bare-metal ARM) and 32-bit compilation checks - **check-generated-code** — regenerates bootstrap descriptor types and fails if the checked-in code is stale - **conformance** — builds the tools and conformance Docker images, runs the full protobuf conformance suite diff --git a/DESIGN.md b/DESIGN.md index 76f8ab5e..ee41af9f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -36,6 +36,7 @@ Buffa fills this gap: a pure Rust implementation designed from the ground up wit The runtime library that generated code depends on. Contains: - **`Message` trait**: The central trait for owned message types, with two-pass `compute_size()` / `write_to()` serialization. +- **`table`** (hidden): the interpreters behind table-driven message codecs; see [decision 13](#13-table-driven-codec-codecstrategytable). - **`MessageView` trait**: The trait for borrowed/zero-copy message views. - **`OwnedView`**: Self-referential container that pairs a `Bytes` buffer with a decoded view, producing a `'static + Send + Sync` type suitable for async and RPC frameworks. - **`MessageField`**: Ergonomic wrapper for optional message fields that dereferences to a default instance when unset. @@ -555,6 +556,18 @@ Generating the *same* `.proto` in two crates without `extern_path`, and choosing Within that bound, the shortcuts do not pay off. `merge` does not help: it consumes wire bytes, so a `Foo` must still be encoded first, and merging only reuses the target allocation rather than avoiding the round-trip. Reflection does not help today either — `a::Foo` can be read reflectively into a `DynamicMessage` (which erases the representation, since `ValueRef::String` is `&str`), but `ReflectMessageMut` is implemented only on `DynamicMessage`, not on generated types, so the return leg falls back to encode/decode. The theoretically cheapest conversion is a static field-by-field `impl From for b::Foo` — no varint, no buffer, no dynamic dispatch, only the unavoidable destination allocations — but it must name both types, because Rust has no structural typing on which to hang a generic conversion. The recommendation is therefore to avoid the divergence with `extern_path`, and where a genuine two-representation boundary exists, to hand-write the `From` rather than reach for a wire round-trip or new machinery. +### 13. Table-Driven Codec (`CodecStrategy::Table`) + +The size, write, and merge code of an unrolled message is specialised to its fields. `CodecStrategy::Table` (a per-message option, `Unrolled` by default) replaces it with a static `buffa::table::Table` and a `Message` impl that forwards to interpreters in `buffa::table`. The measurements and the decision to keep `Unrolled` as the default are in [#463](https://github.com/anthropics/buffa/issues/463). + +A table holds a sorted array of 12-byte entries `{tag, offset, kind, tag_len, aux}`, a dense array that maps field numbers below 64 to entries, and the offset of the unknown-fields slot. `kind` is the field type crossed with its cardinality, so the interpreter dispatches once per field. Message, repeated-message, and enum fields carry a small descriptor (`Aux`) with the accessors that their storage needs, because a `MessageField`, a `Vec`, and an `EnumValue` cannot be read through an offset alone. Offsets come from `core::mem::offset_of!`, so the table needs Rust 1.77 and the generated code refers to it through `buffa::__table!`, which is a compile error on an older compiler. + +Three decisions shape the runtime: + +- **The `unsafe` lives in `buffa`.** `__table!` and `__table_entry!` contain the `unsafe` blocks and witness each field's type against its kind, so a table that names the wrong kind for a field does not compile, and generated code compiles under `#![forbid(unsafe_code)]`. `Table::new` also checks the layout constants at compile time. The interpreters run under Miri in CI. +- **The interpreters are not generic over the sink or the input where that is avoidable.** `Message::encode` and its siblings write any `BufMut` through one shared, non-generic cursor (`buffa/src/encode_sink.rs`), and decoding runs over a contiguous `&[u8]`, so the interpreters are compiled once in `buffa`, at its `opt-level`, and not once per caller. +- **A table refers to the tables of its children,** so a message can use the table only if every message it holds does. The planner in `buffa-codegen` (`table_plan.rs`) starts from the messages that asked for the table, removes those the interpreters cannot handle (oneofs, maps, groups and the types of group fields, `MessageSet`, extension ranges with JSON, custom string, bytes, or collection types), and then removes every message that holds a removed one, until none is left. A message that holds a type from another crate, such as a well-known type, is removed the same way, because the static table of that type is not visible. + ### Owned decode: intentional throughput trade-offs Owned decode (`Message::decode_from_slice`) benchmarks within roughly ±10% of prost in most cases. The costs are intentional and attributable to specific features: diff --git a/README.md b/README.md index 0a396151..0b0f4988 100644 --- a/README.md +++ b/README.md @@ -390,7 +390,7 @@ Compatibility is tested against protoc v21.12, v22.5, v25.5, v27.3, v29.5, and v ## Minimum supported Rust version -The current MSRV is **1.75**. +The current MSRV is **1.75**. The opt-in `CodecStrategy::Table` generates code that needs Rust 1.77 (see [Smaller generated code](docs/guide.md#smaller-generated-code-codec_strategy)); `buffa-build` returns an error on an older compiler when it is requested. buffa is a foundational codec crate, so its `rust-version` is set to the lowest toolchain the released code actually compiles on, not to a calendar target. CI verifies the workspace builds at the MSRV and at stable on every change. With cargo's MSRV-aware resolver (`resolver = "3"`, Rust 1.84+), a downstream project on an older toolchain will automatically resolve to the newest buffa release whose `rust-version` fits — so an accurate declaration matters more than a conservative one. diff --git a/buffa-build/src/lib.rs b/buffa-build/src/lib.rs index 683705e1..78c64771 100644 --- a/buffa-build/src/lib.rs +++ b/buffa-build/src/lib.rs @@ -41,7 +41,7 @@ pub use buffa_codegen::FeatureGateNames; #[doc(inline)] pub use buffa_codegen::ReflectMode; #[doc(inline)] -pub use buffa_codegen::{BytesRepr, MapRepr, PointerRepr, RepeatedRepr, StringRepr}; +pub use buffa_codegen::{BytesRepr, CodecStrategy, MapRepr, PointerRepr, RepeatedRepr, StringRepr}; #[doc(inline)] pub use buffa_codegen::{EnumTypeOverride, FeatureOverride}; @@ -1551,6 +1551,81 @@ impl Config { self } + /// Choose how the binary `Message` implementation of every message is + /// generated (default: [`CodecStrategy::Unrolled`]). + /// + /// On a schema it fully covers, [`CodecStrategy::Table`] makes the + /// compiled size about half as big at `opt-level = "z"`, and it slows + /// messages made of many small fields; a message that cannot use it keeps + /// its size. [`CodecStrategy::Table`] has the measurements, says which + /// messages stay unrolled, and lists how a table message behaves + /// differently. This build reports + /// those in one `cargo:warning`. The option never changes the wire + /// format. The generated code needs Rust 1.77 or later, and `compile` + /// returns an error on an older compiler when a build script runs it (the + /// compiler is read from `RUSTC`). + /// + /// Path-scoped rules for individual messages go in + /// [`codec_strategy_in`](Self::codec_strategy_in), and take precedence + /// over this setting whatever the call order. + /// + /// ```rust,ignore + /// // build.rs + /// buffa_build::Config::new() + /// .files(&["proto/wa.proto"]) + /// .includes(&["proto/"]) + /// .codec_strategy(buffa_build::CodecStrategy::Table) + /// .codec_strategy_in(buffa_build::CodecStrategy::Unrolled, &[".wa.Message"]) + /// .compile()?; + /// ``` + #[must_use] + pub fn codec_strategy(mut self, strategy: CodecStrategy) -> Self { + self.codegen_config.codec_strategy = strategy; + self + } + + /// Choose the [`CodecStrategy`] of the matching messages, on top of the + /// global [`codec_strategy`](Self::codec_strategy). + /// + /// Each path is a fully-qualified proto path prefix, e.g. `".wa.Message"` + /// for one message or `".wa"` for a package (same matching as + /// [`preserve_unknown_fields_in`](Self::preserve_unknown_fields_in)); + /// `"."` matches every message. A leading dot is added if missing, + /// trailing dots are trimmed, and a path that is empty after that prints a + /// `cargo:warning` and is ignored. A rule covers the message it names + /// **and every message nested inside it**. The **last** matching rule wins, + /// so call this after any broader rule. + /// + /// A message selected for the table by a rule that cannot use it stays + /// unrolled and is counted in the warning, and it is an error if the rule + /// names the message by its exact path. A rule that matches no message + /// produces a warning. + /// + /// A table message holds only table messages, and a rule does not extend + /// to the messages a message holds. Selecting a message with a rule + /// therefore also needs rules for everything it holds, unless the global + /// setting is [`CodecStrategy::Table`]. Choosing [`CodecStrategy::Unrolled`] + /// for a message keeps every message that holds it unrolled, with no + /// warning, and that usually includes the root message an application + /// encodes. + #[must_use] + pub fn codec_strategy_in(mut self, strategy: CodecStrategy, paths: &[impl AsRef]) -> Self { + for raw in paths.iter().map(AsRef::as_ref) { + let normalized = normalize_override_path(raw); + if normalized.is_empty() { + println!( + "cargo:warning=buffa: codec_strategy_in path '{raw}' \ + normalizes to empty and will be ignored" + ); + continue; + } + self.codegen_config + .codec_strategy_in + .push((normalized, strategy)); + } + self + } + /// Map every message field (and boxed oneof variant) to the given [`PointerRepr`]. /// Convenience for `.box_type_in(repr, &["."])`. Call before any /// [`box_type_in`](Self::box_type_in) overrides, since the last matching @@ -2059,9 +2134,35 @@ impl Config { /// missing imports, etc.) /// - a precompiled descriptor set file cannot be read /// - the descriptor set bytes cannot be decoded as a `FileDescriptorSet` - /// - code generation fails (e.g. unsupported proto feature) + /// - code generation fails (e.g. unsupported proto feature), including a + /// [`codec_strategy_in`](Self::codec_strategy_in) rule that names by its + /// exact path a message that cannot use the table codec + /// - [`CodecStrategy::Table`] is requested and the compiler is older than + /// Rust 1.77 /// - the output directory cannot be created or written to pub fn compile(self) -> Result<(), Box> { + // Table code needs `offset_of!`, stable in Rust 1.77. Say so once here, + // and not once per field in the compiler's output. + let table_codec_requested = self.codegen_config.codec_strategy == CodecStrategy::Table + || self + .codegen_config + .codec_strategy_in + .iter() + .any(|(_, strategy)| *strategy == CodecStrategy::Table); + // Outside a build script `RUSTC` is unset, and the toolchain that + // compiles the generated code is unknown, so it is not checked. + if table_codec_requested { + if let Some(minor) = rustc_minor_version() { + if minor < 77 { + return Err(format!( + "CodecStrategy::Table needs Rust 1.77 or later, and this build uses \ + 1.{minor}; use a newer compiler or select CodecStrategy::Unrolled" + ) + .into()); + } + } + } + // Reject malformed `exclude_package` entries before protoc runs; the // codegen normalizes them again, but a typo should not cost a protoc // invocation to surface. @@ -2315,6 +2416,29 @@ fn normalize_attr_path(mut path: String) -> String { path } +/// The minor version of the compiler that builds the crate, if `RUSTC` names +/// one and it can be read. +fn rustc_minor_version() -> Option { + let rustc = std::env::var_os("RUSTC")?; + let output = std::process::Command::new(rustc) + .arg("--version") + .output() + .ok()?; + parse_rustc_minor_version(&String::from_utf8_lossy(&output.stdout)) +} + +/// The minor version in the output of `rustc --version`, such as `75` in +/// `rustc 1.75.0 (82e1608df 2023-12-21)`. +fn parse_rustc_minor_version(version_output: &str) -> Option { + version_output + .split_whitespace() + .nth(1)? + .split('.') + .nth(1)? + .parse() + .ok() +} + /// Normalize a path-scoped rule's proto path: trim whitespace, prepend the /// leading dot if absent, and strip trailing dots. Unlike /// [`normalize_attr_path`], an entry that normalizes to empty (e.g. `"..."`) @@ -2823,6 +2947,46 @@ mod tests { ); } + #[test] + fn codec_strategy_in_normalizes_paths_and_keeps_rule_order() { + let config = Config::new() + .codec_strategy(CodecStrategy::Table) + .codec_strategy_in(CodecStrategy::Unrolled, &["my.pkg.Msg", ".my.pkg.Other."]) + .codec_strategy_in(CodecStrategy::Table, &[" .my.pkg.Msg.Inner ", "."]) + .codegen_config; + assert_eq!(config.codec_strategy, CodecStrategy::Table); + assert_eq!( + config.codec_strategy_in, + vec![ + (".my.pkg.Msg".to_string(), CodecStrategy::Unrolled), + (".my.pkg.Other".to_string(), CodecStrategy::Unrolled), + (".my.pkg.Msg.Inner".to_string(), CodecStrategy::Table), + (".".to_string(), CodecStrategy::Table), + ] + ); + } + + #[test] + fn rustc_minor_version_is_read_from_the_version_line() { + assert_eq!( + parse_rustc_minor_version("rustc 1.75.0 (82e1608df 2023-12-21)"), + Some(75) + ); + assert_eq!( + parse_rustc_minor_version("rustc 1.98.0-nightly (abc 2027-01-01)\n"), + Some(98) + ); + assert_eq!(parse_rustc_minor_version(""), None); + assert_eq!(parse_rustc_minor_version("not rustc"), None); + } + + #[test] + fn codec_strategy_defaults_to_unrolled() { + let config = Config::new().codegen_config; + assert_eq!(config.codec_strategy, CodecStrategy::Unrolled); + assert!(config.codec_strategy_in.is_empty()); + } + #[test] fn deny_unknown_json_fields_in_normalizes_paths() { let config = Config::new() diff --git a/buffa-codegen/src/context.rs b/buffa-codegen/src/context.rs index 15bd0059..4603e8e1 100644 --- a/buffa-codegen/src/context.rs +++ b/buffa-codegen/src/context.rs @@ -121,6 +121,10 @@ pub struct CodeGenContext<'a> { /// /// [`generate_with_diagnostics`]: crate::generate_with_diagnostics warnings: std::cell::RefCell>, + /// The messages generated with the table codec, planned by + /// [`plan_table_codec`](Self::plan_table_codec). Unset, which is the same + /// as empty, unless a table codec was requested. + table_plan: std::cell::OnceCell, /// Field-rename exceptions for `CodeGenConfig::idiomatic_field_names`: /// `(proto_name, field_number)` → final Rust source name, present only /// for fields whose context-free snake_case conversion was adjusted by a @@ -477,6 +481,7 @@ impl<'a> CodeGenContext<'a> { nested_module_names, unboxed_oneof_variants, inlined_message_fields, + table_plan: std::cell::OnceCell::new(), field_renames, oneof_keep_verbatim, warnings: std::cell::RefCell::new(plan_warnings), @@ -1099,6 +1104,84 @@ impl<'a> CodeGenContext<'a> { .map_or(self.config.preserve_unknown_fields, |(_, enabled)| *enabled) } + /// The last [`CodeGenConfig::codec_strategy_in`] rule that covers this + /// message, if any: the rule that decided its strategy, as opposed to the + /// global default. `msg_fqn` is the message's proto path, with or without + /// a leading dot. + pub(crate) fn codec_strategy_rule( + &self, + msg_fqn: &str, + ) -> Option<&'a (String, crate::CodecStrategy)> { + let dotted = if msg_fqn.starts_with('.') { + Cow::Borrowed(msg_fqn) + } else { + Cow::Owned(format!(".{msg_fqn}")) + }; + self.config + .codec_strategy_in + .iter() + .rev() + .find(|(prefix, _)| matches_proto_prefix(prefix, &dotted)) + } + + /// The codec strategy requested for this message. + /// + /// Starts from [`CodeGenConfig::codec_strategy`] and applies + /// [`CodeGenConfig::codec_strategy_in`]; the **last** matching rule wins. + /// This is what was asked for, and a message that cannot use + /// [`CodecStrategy::Table`] still gets it here: whether it does is + /// [`uses_table_codec`](Self::uses_table_codec). + pub(crate) fn codec_strategy(&self, msg_fqn: &str) -> crate::CodecStrategy { + self.codec_strategy_rule(msg_fqn) + .map_or(self.config.codec_strategy, |(_, strategy)| *strategy) + } + + /// Whether any message may be asked for the table codec, so that + /// [`plan_table_codec`](Self::plan_table_codec) has work to do. + pub(crate) fn table_codec_requested(&self) -> bool { + self.config.codec_strategy == crate::CodecStrategy::Table + || self + .config + .codec_strategy_in + .iter() + .any(|(_, s)| *s == crate::CodecStrategy::Table) + } + + /// Decide which messages use the table codec. Call once, after + /// construction, when [`table_codec_requested`](Self::table_codec_requested) + /// is true; warnings go to the diagnostics sink. + /// + /// # Errors + /// + /// Returns an error for a `codec_strategy_in` rule that names, by its exact + /// path, a message that cannot use the table codec. + pub(crate) fn plan_table_codec( + &self, + files: &[crate::generated::descriptor::FileDescriptorProto], + files_to_generate: &[String], + ) -> Result<(), crate::CodeGenError> { + let (plan, warnings) = crate::table_plan::plan(self, files, files_to_generate)?; + for w in warnings { + self.warn(w); + } + let planned = self.table_plan.set(plan); + debug_assert!(planned.is_ok(), "the table codec is planned once per run"); + Ok(()) + } + + /// Whether this message is generated with the table codec. + /// `msg_fqn` is the message's proto path, with or without a leading dot. + pub(crate) fn uses_table_codec(&self, msg_fqn: &str) -> bool { + let Some(plan) = self.table_plan.get() else { + return false; + }; + if msg_fqn.starts_with('.') { + plan.contains(msg_fqn) + } else { + plan.contains(&format!(".{msg_fqn}")) + } + } + /// Whether this message's generated JSON deserializer rejects unknown /// fields. /// diff --git a/buffa-codegen/src/features.rs b/buffa-codegen/src/features.rs index 86687ed7..254e2bb2 100644 --- a/buffa-codegen/src/features.rs +++ b/buffa-codegen/src/features.rs @@ -16,7 +16,7 @@ pub use buffa_descriptor::features::*; use crate::context::CodeGenContext; use crate::generated::descriptor::field_descriptor_proto::Type; -use crate::generated::descriptor::FieldDescriptorProto; +use crate::generated::descriptor::{DescriptorProto, FieldDescriptorProto}; /// Compute a field's resolved features, including enum closedness lookup. /// @@ -63,3 +63,21 @@ pub fn resolve_field( } resolved } + +/// The features the generator resolves a message's fields under: those of the +/// file for a top-level message, whose own message-level features do not apply +/// to it, and the parent's with the message's own for a nested one. +/// +/// The table codec's plan calls this too, so that it judges a message's fields +/// under the same features as the code that emits them. +pub(crate) fn message_scope_features( + parent: &ResolvedFeatures, + msg: &DescriptorProto, + top_level: bool, +) -> ResolvedFeatures { + if top_level { + *parent + } else { + resolve_child(parent, message_features(msg)) + } +} diff --git a/buffa-codegen/src/impl_message.rs b/buffa-codegen/src/impl_message.rs index 04768d68..bb2b130b 100644 --- a/buffa-codegen/src/impl_message.rs +++ b/buffa-codegen/src/impl_message.rs @@ -400,6 +400,7 @@ pub fn generate_message_impl( oneof_idents: &std::collections::HashMap, oneof_prefix: &TokenStream, nesting: usize, + table_impl: Option, ) -> Result { let name_ident = format_ident!("{}", rust_name); @@ -418,7 +419,10 @@ pub fn generate_message_impl( let mut write_stmts: Vec = Vec::with_capacity(fields.len()); let mut merge_arms: Vec = Vec::with_capacity(fields.len()); let mut clear_stmts: Vec = Vec::with_capacity(fields.len()); - for kind in &fields { + // A table message forwards its `Message` methods to the shared + // interpreters, so it needs none of the per-field statements. + let per_field: &[_] = if table_impl.is_some() { &[] } else { &fields }; + for kind in per_field { match kind { FieldKind::Scalar(f) => { compute_stmts.push(scalar_compute_size_stmt(ctx, f, features)?); @@ -671,6 +675,62 @@ pub fn generate_message_impl( quote! {} }; + let message_impl = table_impl.unwrap_or_else(|| { + quote! { + impl ::buffa::Message for #name_ident { + /// Returns the total encoded size in bytes. + /// + /// Accumulates in `u64` (which cannot overflow for in-memory + /// data) and saturates to `u32` at return, so a message whose + /// encoded size exceeds the 2 GiB protobuf limit yields a value + /// above [`::buffa::MAX_MESSAGE_BYTES`] that the encode entry + /// points reject, never a silently wrapped size. + #[allow(clippy::let_and_return)] + fn compute_size(&self, #cache_ident: &mut ::buffa::SizeCache) -> u32 { + #[allow(unused_imports)] + use ::buffa::Enumeration as _; + #size_decl + #(#compute_stmts)* + #unknown_fields_size_stmt + ::buffa::saturate_size(size) + } + + fn write_to( + &self, + #cache_ident: &mut ::buffa::SizeCache, + #buf_param, + ) { + #[allow(unused_imports)] + use ::buffa::Enumeration as _; + #(#write_stmts)* + #unknown_fields_write_stmt + } + + fn merge_field( + &mut self, + tag: ::buffa::encoding::Tag, + buf: &mut impl ::buffa::bytes::Buf, + ctx: ::buffa::DecodeContext<'_>, + ) -> ::core::result::Result<(), ::buffa::DecodeError> { + #[allow(unused_imports)] + use ::buffa::bytes::Buf as _; + #[allow(unused_imports)] + use ::buffa::Enumeration as _; + match tag.field_number() { + #(#merge_arms)* + #unknown_fields_merge_arm + } + ::core::result::Result::Ok(()) + } + + fn clear(&mut self) { + #(#clear_stmts)* + #unknown_fields_clear_stmt + } + } + } + }); + Ok(quote! { ::buffa::impl_default_instance!(#name_ident); @@ -678,57 +738,7 @@ pub fn generate_message_impl( #message_name_impl - impl ::buffa::Message for #name_ident { - /// Returns the total encoded size in bytes. - /// - /// Accumulates in `u64` (which cannot overflow for in-memory - /// data) and saturates to `u32` at return, so a message whose - /// encoded size exceeds the 2 GiB protobuf limit yields a value - /// above [`::buffa::MAX_MESSAGE_BYTES`] that the encode entry - /// points reject, never a silently wrapped size. - #[allow(clippy::let_and_return)] - fn compute_size(&self, #cache_ident: &mut ::buffa::SizeCache) -> u32 { - #[allow(unused_imports)] - use ::buffa::Enumeration as _; - #size_decl - #(#compute_stmts)* - #unknown_fields_size_stmt - ::buffa::saturate_size(size) - } - - fn write_to( - &self, - #cache_ident: &mut ::buffa::SizeCache, - #buf_param, - ) { - #[allow(unused_imports)] - use ::buffa::Enumeration as _; - #(#write_stmts)* - #unknown_fields_write_stmt - } - - fn merge_field( - &mut self, - tag: ::buffa::encoding::Tag, - buf: &mut impl ::buffa::bytes::Buf, - ctx: ::buffa::DecodeContext<'_>, - ) -> ::core::result::Result<(), ::buffa::DecodeError> { - #[allow(unused_imports)] - use ::buffa::bytes::Buf as _; - #[allow(unused_imports)] - use ::buffa::Enumeration as _; - match tag.field_number() { - #(#merge_arms)* - #unknown_fields_merge_arm - } - ::core::result::Result::Ok(()) - } - - fn clear(&mut self) { - #(#clear_stmts)* - #unknown_fields_clear_stmt - } - } + #message_impl #extension_set_impl }) diff --git a/buffa-codegen/src/lib.rs b/buffa-codegen/src/lib.rs index eddbac03..0caa2227 100644 --- a/buffa-codegen/src/lib.rs +++ b/buffa-codegen/src/lib.rs @@ -43,6 +43,8 @@ pub(crate) mod owned_view; pub(crate) mod reflect; pub(crate) mod reflect_owned; pub(crate) mod reflect_view; +pub(crate) mod table_codec; +pub(crate) mod table_plan; pub(crate) mod view; use crate::generated::descriptor::{DescriptorProto, EnumDescriptorProto, FileDescriptorProto}; @@ -1069,6 +1071,78 @@ pub enum EnumTypeOverride { Open, } +/// How a message's binary [`Message`](https://docs.rs/buffa/latest/buffa/trait.Message.html) +/// implementation is generated. Select it with +/// [`CodeGenConfig::codec_strategy`] and [`CodeGenConfig::codec_strategy_in`]. +/// +/// The strategy changes only the code that sizes, writes and decodes a message +/// in the binary wire format. It never changes the wire format, and JSON, text, +/// view and reflection code are generated the same way under either. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +#[non_exhaustive] +pub enum CodecStrategy { + /// Emit code specialised to each message: the fastest code, and the + /// largest. The default. + #[default] + Unrolled, + /// Emit one static table per message and forward `Message` to interpreters + /// shared by every message, in the `buffa` crate. On a schema the table + /// fully covers, it roughly halves the compiled size at + /// `opt-level = "z"`; a message that cannot use it keeps its size. It costs + /// run time on messages made of many small fields. The measurements are in + /// anthropics/buffa#463. + /// + /// Not every message can use it. These stay [`Unrolled`](Self::Unrolled): + /// + /// - a message with a `oneof`, a `map` field, or a group field; + /// - the message type of a group field; + /// - a message that uses the `MessageSet` wire format; + /// - a message with extension ranges, when JSON code is generated and + /// unknown fields are preserved; + /// - a message with a field of a non-default string, bytes, or collection + /// type, such as `use_bytes_type`, `string_type`, `bytes_type`, and + /// `repeated_type` select; + /// - a message with a field whose message type stays `Unrolled` for any + /// reason, is not selected for the table, or is generated by another + /// crate, such as a well-known type. + /// + /// A table message therefore holds only table messages, and selecting a + /// message does not select the messages it holds. + /// + /// A table message differs from an unrolled one in three ways. A field that + /// declares a length past the end of its enclosing message fails at once + /// with `DecodeError::UnexpectedEof`, where unrolled code can report a + /// different error for the same rejected input. `clear()` resets the message + /// to its default, which releases the capacity of its strings and vectors. + /// Decoding gathers a `Buf` that is not contiguous into one buffer first. + /// + /// A message that another crate or codegen run uses as the type of a group + /// or editions `DELIMITED` field must not be selected, because + /// `Message::merge_field` on a table message cannot gather a + /// non-contiguous buffer, and the default group decoding calls it. Codegen + /// sees only the group fields of its own run. + /// + /// Codegen reports the messages that fall back in one + /// [`CodeGenWarning::TableCodecFallbackSummary`]. The table code needs Rust + /// 1.77 or later, and contains `unsafe` code inside `buffa` macros, so it + /// compiles under `#![forbid(unsafe_code)]`. + Table, +} + +/// Why some messages selected for [`CodecStrategy::Table`] fell back to +/// [`CodecStrategy::Unrolled`], for +/// [`CodeGenWarning::TableCodecFallbackSummary`]. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct TableCodecFallbackReason { + /// The reason as a predicate, such as `has a oneof`. The wording is for + /// people and may change between releases. + pub reason: String, + /// The proto paths of all of them, with a leading dot, in declaration + /// order. + pub messages: Vec, +} + /// Configuration for code generation. #[derive(Debug, Clone)] #[non_exhaustive] @@ -1831,6 +1905,37 @@ pub struct CodeGenConfig { /// [`CodeGenWarning::ExcludePackageMatchedNothing`] for an entry that /// matches no package in the input. pub exclude_packages: Vec, + /// The [`CodecStrategy`] of every message that no + /// [`codec_strategy_in`](Self::codec_strategy_in) rule covers (default: + /// [`CodecStrategy::Unrolled`]). + /// + /// Under [`CodecStrategy::Table`] a message that cannot use the table + /// stays unrolled, and codegen reports one + /// [`CodeGenWarning::TableCodecFallbackSummary`] for the run. + pub codec_strategy: CodecStrategy, + /// Path-scoped overrides for [`codec_strategy`](Self::codec_strategy). + /// Each entry is `(proto_path_prefix, strategy)`, with a leading dot + /// (`".pkg.Msg"`). + /// + /// Matching uses the same proto-segment-aware prefix rules as + /// [`preserve_unknown_fields_in`](Self::preserve_unknown_fields_in): a rule + /// covers the message it names *and every message nested inside it*, `"."` + /// covers everything, and the **last** matching rule wins. + /// + /// A [`CodecStrategy::Table`] rule that covers a message that cannot use + /// the table keeps it unrolled and counts it in the + /// [`CodeGenWarning::TableCodecFallbackSummary`]. If the rule names the + /// message by its exact path, that is an error instead, because the rule + /// asked for something impossible. A rule that matches no generated message + /// produces a [`CodeGenWarning::CodecStrategyRuleMatchedNothing`]. + /// + /// A table message holds only table messages, and a rule does not extend to + /// the messages a message holds. A [`CodecStrategy::Table`] rule for a + /// message therefore needs rules for the messages it holds, or a global + /// [`CodecStrategy::Table`]; an exact-path rule fails without them. + /// Setting a message to [`CodecStrategy::Unrolled`] keeps every message that + /// holds it unrolled too, without a warning. + pub codec_strategy_in: Vec<(String, CodecStrategy)>, } impl Default for CodeGenConfig { @@ -1877,6 +1982,8 @@ impl Default for CodeGenConfig { feature_gate_names: FeatureGateNames::default(), type_name_prefix: String::new(), exclude_packages: Vec::new(), + codec_strategy: CodecStrategy::Unrolled, + codec_strategy_in: Vec::new(), } } } @@ -2270,6 +2377,32 @@ pub enum CodeGenWarning { /// so, in place of the path hint. matches_ungenerated: bool, }, + /// A [`codec_strategy_in`](CodeGenConfig::codec_strategy_in) rule matched no + /// message being generated, so it changed nothing. Usually a typo, a field + /// path instead of a message path, or a rule for a package mapped through + /// `extern_path`. + #[non_exhaustive] + CodecStrategyRuleMatchedNothing { + /// The rule's path as configured. + rule: String, + }, + /// Some messages selected for [`CodecStrategy::Table`], by the global + /// [`codec_strategy`](CodeGenConfig::codec_strategy) or by a + /// [`codec_strategy_in`](CodeGenConfig::codec_strategy_in) rule, cannot use + /// the table and are generated [`CodecStrategy::Unrolled`]. One warning + /// covers the whole run. A message that falls back only because a message + /// it holds is set to `Unrolled` is not counted, because the setting is + /// the reason. + #[non_exhaustive] + TableCodecFallbackSummary { + /// The number of messages that fell back. + fallbacks: usize, + /// The number of messages selected for the table, not counting those + /// kept unrolled only by a message the user set to `Unrolled`. + selected: usize, + /// Why they fell back, most common reason first. + reasons: Vec, + }, } /// The `buffa-build` methods that add to the field-type rule list `option`, @@ -2413,6 +2546,46 @@ impl core::fmt::Display for CodeGenWarning { check the path against the fully-qualified proto message names" ) } + Self::CodecStrategyRuleMatchedNothing { rule } => { + write!( + f, + "codec_strategy_in rule '{rule}' matched no generated message; those \ + messages keep the global codec_strategy setting — check the path \ + against the fully-qualified proto message names" + ) + } + Self::TableCodecFallbackSummary { + fallbacks, + selected, + reasons, + } => { + write!( + f, + "{fallbacks} of {selected} messages selected for the table codec use the \ + unrolled codec instead, which works but is larger" + )?; + for (i, r) in reasons.iter().enumerate() { + let sep = if i == 0 { ": " } else { "; " }; + let shown = r.messages.len().min(3); + let examples = r.messages[..shown].join(", "); + let more = match r.messages.len() - shown { + 0 => String::new(), + n => format!(", and {n} more"), + }; + write!( + f, + "{sep}{} ({}: {examples}{more})", + r.reason, + r.messages.len() + )?; + } + write!( + f, + ". To silence this warning, set the messages that fell back to the unrolled \ + codec (buffa-build: `.codec_strategy_in(CodecStrategy::Unrolled, \ + &[\"\"])`; plugin: `codec_strategy_in==unrolled`)" + ) + } Self::DenyUnknownJsonFieldsRuleMatchedNothing { rule } => { write!( f, @@ -3251,6 +3424,17 @@ pub fn generate_with_diagnostics( } } + // A rule that names no generated message (typo, field path, extern-mapped + // package) leaves its messages on the global strategy. + for (rule, _) in &config.codec_strategy_in { + if !rule_matches_generated_message(rule, file_descriptors, files_to_generate) { + ctx.warn(CodeGenWarning::CodecStrategyRuleMatchedNothing { rule: rule.clone() }); + } + } + if ctx.table_codec_requested() { + ctx.plan_table_codec(file_descriptors, files_to_generate)?; + } + // Lazy views need the eager view machinery; warn once per run. if config.lazy_views && !config.generate_views { ctx.warn(CodeGenWarning::LazyViewsRequireViews); @@ -4015,7 +4199,7 @@ fn generate_proto_content( current_package, &rust_name, &proto_fqn, - &features, + &crate::features::message_scope_features(&features, message_type, true), &resolver, )?; owned.extend(owned_top); diff --git a/buffa-codegen/src/message.rs b/buffa-codegen/src/message.rs index 17e1e96c..f38f0288 100644 --- a/buffa-codegen/src/message.rs +++ b/buffa-codegen/src/message.rs @@ -171,8 +171,7 @@ fn generate_message_with_nesting( let nested_proto_name = nested.name.as_deref().unwrap_or(""); let nested_fqn = format!("{}.{}", proto_fqn, nested_proto_name); let nested_rust_name = ctx.config.prefixed_type_name(nested_proto_name); - let msg_features = - crate::features::resolve_child(features, crate::features::message_features(nested)); + let msg_features = crate::features::message_scope_features(features, nested, false); generate_message_with_nesting( scope.nested(&nested_fqn, &msg_features), nested, @@ -440,6 +439,13 @@ fn generate_message_with_nesting( }) .collect::, _>>()?; + let table_impl = if ctx.uses_table_codec(proto_fqn) { + Some(crate::table_codec::generate_table_impl( + scope, msg, rust_name, resolver, + )?) + } else { + None + }; let message_impl = crate::impl_message::generate_message_impl( ctx, msg, @@ -451,6 +457,7 @@ fn generate_message_with_nesting( &oneof_idents, &oneof_prefix, nesting, + table_impl, )?; let text_impl = crate::impl_text::generate_text_impl( @@ -1544,8 +1551,8 @@ fn is_wkt_wrapper_type(type_name: Option<&str>) -> bool { /// Resolved Rust type and map-entry metadata for a single field. #[derive(Debug)] -struct FieldInfo { - rust_type: TokenStream, +pub(crate) struct FieldInfo { + pub(crate) rust_type: TokenStream, /// Type to use in the struct field declaration. Differs from `rust_type` /// only for self-referential message fields, where it uses `Self` instead /// of the concrete name. `rust_type` stays concrete for serde-deserialize @@ -1616,7 +1623,7 @@ struct FieldInfo { /// Shared by `generate_field` (struct declaration) and the custom /// deserialize codegen to avoid duplicating the type-resolution /// if/else chain. -fn classify_field( +pub(crate) fn classify_field( scope: MessageScope<'_>, msg: &DescriptorProto, field: &crate::generated::descriptor::FieldDescriptorProto, diff --git a/buffa-codegen/src/table_codec.rs b/buffa-codegen/src/table_codec.rs new file mode 100644 index 00000000..6799f794 --- /dev/null +++ b/buffa-codegen/src/table_codec.rs @@ -0,0 +1,279 @@ +//! The table codec: one static `buffa::table::Table` per message and a +//! `Message` impl that forwards to the interpreters in `buffa::table`. +//! +//! Which messages get one is decided by [`crate::table_plan`]; this module +//! emits the code for a message the plan selected. + +use proc_macro2::TokenStream; +use quote::{format_ident, quote}; + +use crate::context::MessageScope; +use crate::generated::descriptor::field_descriptor_proto::Type; +use crate::generated::descriptor::DescriptorProto; +use crate::idents::rust_path_to_tokens; +use crate::message::classify_field; +use crate::table_plan::{table_fields, Card, TableField}; +use crate::CodeGenError; + +/// The name of the static table of the message struct `rust_name`. +fn table_ident(rust_name: &str) -> proc_macro2::Ident { + format_ident!("__BUFFA_TABLE_{}", rust_name) +} + +/// The path of the static table of the message type whose struct is at +/// `type_path`: the last segment is replaced by the table's name. +fn table_path(type_path: &str) -> Result { + if type_path.starts_with("::") || type_path.starts_with("crate::") { + return Err(CodeGenError::Other(format!( + "table codec: message type `{type_path}` is generated by another crate, \ + so it has no table here" + ))); + } + let (head, last) = match type_path.rsplit_once("::") { + Some((head, last)) => (format!("{head}::"), last), + None => (String::new(), type_path), + }; + Ok(rust_path_to_tokens(&format!("{head}__BUFFA_TABLE_{last}"))) +} + +/// The static table and `impl Message` of a message the plan selected. +/// +/// # Errors +/// +/// Returns an error if the message cannot use the table, which the plan +/// should have ruled out. +pub(crate) fn generate_table_impl( + scope: MessageScope<'_>, + msg: &DescriptorProto, + rust_name: &str, + resolver: &crate::imports::ImportResolver, +) -> Result { + let MessageScope { + ctx, + proto_fqn, + features, + .. + } = scope; + let name = format_ident!("{}", rust_name); + let table = table_ident(rust_name); + let dotted_fqn = format!(".{proto_fqn}"); + + let fields = table_fields(ctx, msg, &dotted_fqn, features).map_err(|why| { + CodeGenError::Other(format!( + "table codec: {dotted_fqn} cannot use the table: {}", + why.detail + )) + })?; + + let mut entries: Vec = Vec::with_capacity(fields.len()); + let mut aux: Vec = Vec::new(); + for f in &fields { + let (entry, aux_item) = field_entry(scope, msg, &name, f, aux.len(), resolver)?; + entries.push(entry); + aux.extend(aux_item); + } + + let dense = dense_lookup(&fields); + + let unknown = if ctx.preserve_unknown_fields(proto_fqn) { + quote! { __buffa_unknown_fields } + } else { + quote! { none } + }; + + Ok(quote! { + #[doc(hidden)] + #[allow(non_upper_case_globals)] + pub(crate) static #table: ::buffa::table::Table<#name> = ::buffa::__table!( + #name, + abi = ::buffa::table::ABI, + entries = [#(#entries),*], + dense = &[#(#dense),*], + aux = [#(#aux),*], + unknown = #unknown, + ); + + impl ::buffa::Message for #name { + fn compute_size(&self, cache: &mut ::buffa::SizeCache) -> u32 { + #table.compute_size(self, cache) + } + + fn write_to(&self, cache: &mut ::buffa::SizeCache, buf: &mut impl ::buffa::EncodeSink) { + #table.write_to(self, cache, buf); + } + + fn merge_field( + &mut self, + tag: ::buffa::encoding::Tag, + buf: &mut impl ::buffa::bytes::Buf, + ctx: ::buffa::DecodeContext<'_>, + ) -> ::core::result::Result<(), ::buffa::DecodeError> { + #table.merge_field(self, tag, buf, ctx) + } + + fn merge_to_limit( + &mut self, + buf: &mut impl ::buffa::bytes::Buf, + ctx: ::buffa::DecodeContext<'_>, + limit: usize, + ) -> ::core::result::Result<(), ::buffa::DecodeError> { + #table.merge_to_limit(self, buf, ctx, limit) + } + + fn merge_length_delimited( + &mut self, + buf: &mut impl ::buffa::bytes::Buf, + ctx: ::buffa::DecodeContext<'_>, + ) -> ::core::result::Result<(), ::buffa::DecodeError> { + #table.merge_length_delimited(self, buf, ctx) + } + + fn clear(&mut self) { + *self = ::core::default::Default::default(); + } + } + }) +} + +/// `dense[n]` is one plus the index of the field numbered `n`, or `0`, for +/// the numbers up to the highest field number below 64. Empty, so that the +/// table searches instead, for a message with 255 fields or more, because +/// `Table::new` accepts a dense array only when every index fits below 255. +fn dense_lookup(fields: &[TableField<'_>]) -> Vec { + if fields.len() >= 255 { + return Vec::new(); + } + let top = fields + .iter() + .map(|f| f.number) + .filter(|&n| n < 64) + .max() + .unwrap_or(0) as usize; + let mut dense = vec![0u8; top + 1]; + for (i, f) in fields.iter().enumerate() { + if let Some(slot) = dense.get_mut(f.number as usize) { + // At most 254 fields, so this fits. + *slot = (i + 1) as u8; + } + } + dense +} + +/// The `__table_entry!` for one field, and the aux descriptor it needs, which +/// goes at index `aux_index` of the table's aux array. +fn field_entry( + scope: MessageScope<'_>, + msg: &DescriptorProto, + name: &proc_macro2::Ident, + f: &TableField<'_>, + aux_index: usize, + resolver: &crate::imports::ImportResolver, +) -> Result<(TokenStream, Option), CodeGenError> { + let MessageScope { + ctx, + current_package, + nesting, + .. + } = scope; + let field = f.field; + let field_name = field.name.as_deref().unwrap_or(""); + let ident = ctx.field_ident(field_name, field.number.unwrap_or(0)); + let kind = format_ident!("{}", f.kind); + let number = f.number; + + let aux_u16 = || { + u16::try_from(aux_index).map_err(|_| { + CodeGenError::Other(format!( + "table codec: {}.{field_name}: a message has more than 65535 fields \ + that need a descriptor", + scope.proto_fqn + )) + }) + }; + let type_path = |what: &str| -> Result { + let type_name = field + .type_name + .as_deref() + .ok_or(CodeGenError::MissingField("field.type_name"))?; + ctx.rust_type_relative(type_name, current_package, nesting) + .ok_or_else(|| CodeGenError::Other(format!("{what} type '{type_name}' not found"))) + }; + + // The table is not among the imports that `idiomatic_imports` shortens + // paths with, so its path is built from the unshortened one. + let unshortened_path = || -> Result { + let type_name = field + .type_name + .as_deref() + .ok_or(CodeGenError::MissingField("field.type_name"))?; + let split = ctx + .rust_type_relative_split(type_name, current_package, nesting) + .ok_or_else(|| CodeGenError::Other(format!("message type '{type_name}' not found")))?; + Ok(if split.to_package.is_empty() { + split.within_package + } else { + format!("{}::{}", split.to_package, split.within_package) + }) + }; + + match f.ty { + Type::TYPE_MESSAGE => { + let child = type_path("message")?; + let child_table = table_path(&unshortened_path()?)?; + let child_ty = rust_path_to_tokens(&child); + let (slot, aux_item) = if f.card == Card::Repeated { + ( + quote! { ::buffa::alloc::vec::Vec<#child_ty> }, + quote! { + ::buffa::table::Aux::Rep(&::buffa::table::RepVt::new::<#child_ty>(&#child_table)) + }, + ) + } else { + let slot = classify_field(scope, msg, field, resolver)?.rust_type; + ( + slot.clone(), + quote! { + ::buffa::table::Aux::Msg(&::buffa::table::MsgVt::new::<#slot>(&#child_table)) + }, + ) + }; + let aux = aux_u16()?; + Ok(( + quote! { + ::buffa::__table_entry!(#name, #ident, #kind, #number, aux = #aux, slot = #slot) + }, + Some(aux_item), + )) + } + Type::TYPE_ENUM => { + let enum_ty = rust_path_to_tokens(&type_path("enum")?); + let repeated = matches!(f.card, Card::Repeated | Card::Packed); + let shape = match (repeated, f.card == Card::Optional, f.closed_enum) { + (true, _, true) => quote! { RepeatedClosed }, + (true, _, false) => quote! { RepeatedOpen }, + (false, true, true) => quote! { OptionalClosed }, + (false, true, false) => quote! { OptionalOpen }, + (false, false, true) => quote! { ImplicitClosed }, + (false, false, false) => quote! { ImplicitOpen }, + }; + let shape = quote! { ::buffa::table::#shape<#enum_ty> }; + let aux = aux_u16()?; + Ok(( + quote! { + ::buffa::__table_entry!( + #name, #ident, #kind, #number, + aux = #aux, + slot = <#shape as ::buffa::table::EnumShape>::Slot + ) + }, + Some(quote! { + ::buffa::table::Aux::Enum(&::buffa::table::EnumVt::new::<#shape>()) + }), + )) + } + _ => Ok(( + quote! { ::buffa::__table_entry!(#name, #ident, #kind, #number) }, + None, + )), + } +} diff --git a/buffa-codegen/src/table_plan.rs b/buffa-codegen/src/table_plan.rs new file mode 100644 index 00000000..451c1b8d --- /dev/null +++ b/buffa-codegen/src/table_plan.rs @@ -0,0 +1,529 @@ +//! Which messages are generated with [`CodecStrategy::Table`], and what each +//! table field looks like. +//! +//! A message can use the table if the interpreters in `buffa::table` cover +//! every field and every message it holds is a table message too, because a +//! table records the tables of its children. The set of table messages is the +//! largest set of requested, locally eligible messages closed under that +//! rule, found by removing messages until none is left holding a non-table +//! child. + +use std::collections::{HashMap, HashSet}; + +use crate::context::CodeGenContext; +use crate::features::ResolvedFeatures; +use crate::generated::descriptor::field_descriptor_proto::{Label, Type}; +use crate::generated::descriptor::{DescriptorProto, FieldDescriptorProto, FileDescriptorProto}; +use crate::impl_message::{ + effective_type, is_explicit_presence_scalar, is_field_packed, is_real_oneof_member, + is_required_field, +}; +use crate::message::{find_map_entry, is_closed_enum}; +use crate::{CodeGenError, CodeGenWarning, CodecStrategy, TableCodecFallbackReason}; + +/// The cardinality half of a field's `buffa::table::Kind`. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Card { + Implicit, + Required, + Optional, + Repeated, + Packed, +} + +impl Card { + fn name(self) -> &'static str { + match self { + Card::Implicit => "Implicit", + Card::Required => "Required", + Card::Optional => "Optional", + Card::Repeated => "Repeated", + Card::Packed => "Packed", + } + } +} + +/// A field of a message that can use the table, in the terms the table uses. +pub(crate) struct TableField<'a> { + pub(crate) field: &'a FieldDescriptorProto, + pub(crate) number: u32, + pub(crate) ty: Type, + pub(crate) card: Card, + /// The name of the `buffa::table::Kind` variant of this field. + pub(crate) kind: String, + /// For an enum field: whether the enum is closed. + pub(crate) closed_enum: bool, +} + +/// The `Kind` variant name of a field type, or `None` for a group, which has +/// no kind. +fn type_stem(ty: Type, card: Card) -> Option<&'static str> { + Some(match ty { + Type::TYPE_INT32 => "Int32", + Type::TYPE_INT64 => "Int64", + Type::TYPE_UINT32 => "Uint32", + Type::TYPE_UINT64 => "Uint64", + Type::TYPE_SINT32 => "Sint32", + Type::TYPE_SINT64 => "Sint64", + Type::TYPE_BOOL => "Bool", + Type::TYPE_FIXED32 => "Fixed32", + Type::TYPE_FIXED64 => "Fixed64", + Type::TYPE_SFIXED32 => "Sfixed32", + Type::TYPE_SFIXED64 => "Sfixed64", + Type::TYPE_FLOAT => "Float", + Type::TYPE_DOUBLE => "Double", + Type::TYPE_STRING => "Str", + Type::TYPE_BYTES => "Bytes", + Type::TYPE_ENUM => "Enum", + // A message has one kind per cardinality class, not per `Card`. + Type::TYPE_MESSAGE if card == Card::Repeated => return Some("MsgRepeated"), + Type::TYPE_MESSAGE => return Some("MsgSingular"), + Type::TYPE_GROUP => return None, + }) +} + +/// Why a message cannot use the table. +#[derive(Clone, Debug)] +pub(crate) struct Ineligible { + /// The reason in a few words, which the summary warning groups messages + /// by. For a message that holds another that cannot use the table, this + /// includes the other's reason. + pub(crate) reason: String, + /// The reason for this message, naming the field or type. + pub(crate) detail: String, + /// The message's own reason if it is the cause of a fallback, and + /// otherwise the cause of the message it holds, which is followed down to + /// the message that cannot use the table itself. + pub(crate) root: String, + /// Whether the fallback follows from a strategy the user chose (a message + /// it holds is set to `Unrolled`), so that it needs no warning. + pub(crate) silent: bool, + /// What to do about it when a rule that names the message exactly asked + /// for the table, if it differs from the general advice. + pub(crate) hint: Option, +} + +/// An [`Ineligible`] whose reason needs no more detail. +fn same(reason: &str) -> Ineligible { + ineligible(reason, format!("it {reason}")) +} + +fn ineligible(reason: impl Into, detail: impl Into) -> Ineligible { + let reason = reason.into(); + Ineligible { + root: reason.clone(), + reason, + detail: detail.into(), + silent: false, + hint: None, + } +} + +/// The table view of the fields of `msg`, or why it cannot use the table, +/// judged without regard to the other messages. +/// +/// `fqn` is the message's proto path with a leading dot. +pub(crate) fn table_fields<'a>( + ctx: &CodeGenContext, + msg: &'a DescriptorProto, + fqn: &str, + features: &ResolvedFeatures, +) -> Result>, Ineligible> { + if msg + .options + .as_option() + .and_then(|o| o.message_set_wire_format) + .unwrap_or(false) + { + return Err(same("uses the MessageSet wire format")); + } + // With JSON and extension ranges the unknown fields sit in a wrapper + // struct that the table cannot address as `UnknownFields`. + if ctx.config.generate_json + && !msg.extension_range.is_empty() + && ctx.preserve_unknown_fields(fqn) + { + return Err(same("has extension ranges and JSON code is generated")); + } + + let mut fields = Vec::with_capacity(msg.field.len()); + for f in &msg.field { + let name = f.name.as_deref().unwrap_or(""); + if is_real_oneof_member(f) { + return Err(ineligible( + "has a oneof", + format!("field `{name}` is in a oneof"), + )); + } + if find_map_entry(msg, f).is_some() { + return Err(ineligible( + "has a map field", + format!("field `{name}` is a map"), + )); + } + let ty = effective_type(ctx, f, features); + let field_fqn = format!("{fqn}.{name}"); + let repeated = f.label.unwrap_or_default() == Label::LABEL_REPEATED; + let custom = match ty { + Type::TYPE_STRING => !ctx.string_repr(&field_fqn).is_default(), + Type::TYPE_BYTES => !ctx.bytes_repr(&field_fqn).is_default(), + _ => false, + } || (repeated && !ctx.repeated_repr(&field_fqn).is_default()); + if custom { + return Err(ineligible( + "has a field with a custom string, bytes or collection type", + format!("field `{name}` has a custom string, bytes or collection type"), + )); + } + let number = crate::impl_message::validated_field_number(f) + .map_err(|e| ineligible("has an invalid field number", e.to_string()))?; + let card = if repeated { + if is_field_packed(f, features) { + Card::Packed + } else { + Card::Repeated + } + } else if is_explicit_presence_scalar(f, ty, features) { + Card::Optional + } else if is_required_field(f, features) { + Card::Required + } else { + Card::Implicit + }; + let closed_enum = ty == Type::TYPE_ENUM + && is_closed_enum(&crate::features::resolve_field(ctx, f, features)); + let Some(stem) = type_stem(ty, card) else { + return Err(ineligible( + "has a group field", + format!("field `{name}` is a group"), + )); + }; + let kind = if ty == Type::TYPE_MESSAGE { + stem.to_string() + } else { + format!("{stem}{}", card.name()) + }; + fields.push(TableField { + field: f, + number, + ty, + card, + kind, + closed_enum, + }); + } + fields.sort_by_key(|f| f.number); + Ok(fields) +} + +/// One message of the run and what the plan needs to know about it. +struct Candidate<'a> { + fqn: String, + /// The proto paths of the message types of its fields. + children: Vec, + fields: Result>, Ineligible>, +} + +/// Every message of `messages` and the messages nested in them that has a +/// `Message` impl and is generated by this crate, with its parents' `scope`. +/// +/// A message that another crate generates, though this run also holds its +/// descriptor, is not collected, because the types that refer to it name the +/// other crate's copy. +/// +/// `features` is what `message.rs` gives each message's scope: the file's +/// features for a top-level message, whose own message-level features do not +/// apply to it, and the parent's features with the message's own for a nested +/// one. The emitter recomputes a message's fields under that scope, so the plan +/// must judge them under the same one. +fn collect<'a>( + ctx: &CodeGenContext, + messages: &'a [DescriptorProto], + (package, scope): (&str, &str), + parent_features: &ResolvedFeatures, + top_level: bool, + out: &mut Vec>, + group_types: &mut HashSet, +) { + for msg in messages { + let is_map_entry = msg + .options + .as_option() + .is_some_and(|o| o.map_entry.unwrap_or(false)); + let fqn = format!("{scope}.{}", msg.name.as_deref().unwrap_or("")); + let features = crate::features::message_scope_features(parent_features, msg, top_level); + let is_extern = ctx + .rust_type_relative(&fqn, package, 0) + .is_some_and(|path| path.starts_with("::") || path.starts_with("crate::")); + if !is_map_entry && !is_extern { + let mut children = Vec::new(); + for f in &msg.field { + match effective_type(ctx, f, &features) { + Type::TYPE_MESSAGE => children.extend(f.type_name.clone()), + Type::TYPE_GROUP => group_types.extend(f.type_name.clone()), + _ => {} + } + } + out.push(Candidate { + fields: table_fields(ctx, msg, &fqn, &features), + fqn: fqn.clone(), + children, + }); + } + collect( + ctx, + &msg.nested_type, + (package, &fqn), + &features, + false, + out, + group_types, + ); + } +} + +/// The messages generated with the table codec in one run. +#[derive(Default)] +pub(crate) struct TablePlan { + /// Proto paths with a leading dot. + tables: HashSet, +} + +impl TablePlan { + pub(crate) fn contains(&self, fqn: &str) -> bool { + self.tables.contains(fqn) + } +} + +/// Why a message that holds `child` cannot use the table, when `child` has no +/// table. +fn child_without_table( + ctx: &CodeGenContext, + child: &str, + reasons: &HashMap<&str, Ineligible>, + generated: &HashSet<&str>, +) -> Ineligible { + if let Some(held) = reasons.get(child) { + return Ineligible { + reason: format!("holds a message that {}", held.root), + detail: format!( + "it has a field of message type `{child}`, which {}", + held.root + ), + root: held.root.clone(), + silent: held.silent, + hint: held.hint.clone(), + }; + } + if !generated.contains(child) { + return ineligible( + "holds a message that another crate or run generates", + format!("it has a field of message type `{child}`, which is not generated here"), + ); + } + // Not selected: the strategy for it is unrolled, by the user's rule or by + // the global default. + if ctx.codec_strategy_rule(child).is_some() { + Ineligible { + silent: true, + ..ineligible( + "holds a message set to the unrolled codec", + format!( + "it has a field of message type `{child}`, which is set to the unrolled codec" + ), + ) + } + } else { + Ineligible { + hint: Some(format!( + "Select `{child}` as well, and every message it holds (buffa-build: \ + `.codec_strategy_in(CodecStrategy::Table, &[\"{child}\"])`; plugin: \ + `codec_strategy_in={child}=table`)" + )), + ..ineligible( + "holds a message not selected for the table", + format!("it has a field of message type `{child}`, which is not selected for the table codec"), + ) + } + } +} + +/// Decide which messages of `files_to_generate` use the table codec. +/// +/// Returns the plan and a summary warning about the messages that asked for +/// the table and cannot have it, unless the user's own choice of `Unrolled` +/// for a message they hold is the only reason. +/// +/// # Errors +/// +/// A `codec_strategy_in` rule that names, by its exact path, a message that +/// cannot use the table. +pub(crate) fn plan( + ctx: &CodeGenContext, + files: &[FileDescriptorProto], + files_to_generate: &[String], +) -> Result<(TablePlan, Vec), CodeGenError> { + let mut candidates = Vec::new(); + let mut group_types = HashSet::new(); + for file in files.iter().filter(|f| { + f.name + .as_deref() + .is_some_and(|n| files_to_generate.iter().any(|g| g == n)) + }) { + let package = file.package.as_deref().unwrap_or(""); + let scope = if package.is_empty() { + String::new() + } else { + format!(".{package}") + }; + collect( + ctx, + &file.message_type, + (package, &scope), + &crate::features::for_file(file), + true, + &mut candidates, + &mut group_types, + ); + } + let generated: HashSet<&str> = candidates.iter().map(|c| c.fqn.as_str()).collect(); + + // The messages that asked for the table, each with the reason it cannot + // have it, if there is one. + let mut reasons: HashMap<&str, Ineligible> = HashMap::new(); + let mut selected: Vec<&Candidate> = Vec::new(); + for c in &candidates { + if ctx.codec_strategy(&c.fqn) != CodecStrategy::Table { + continue; + } + selected.push(c); + if let Err(why) = &c.fields { + reasons.insert(&c.fqn, why.clone()); + } else if group_types.contains(&c.fqn) { + reasons.insert(&c.fqn, same("is the type of a group field")); + } + } + + // Remove every message that holds a child without a table, until none is + // left. + let mut remaining: HashSet<&str> = selected + .iter() + .filter(|c| !reasons.contains_key(c.fqn.as_str())) + .map(|c| c.fqn.as_str()) + .collect(); + loop { + let mut removed = Vec::new(); + for c in selected + .iter() + .filter(|c| remaining.contains(c.fqn.as_str())) + { + // The reason for the first child without a table that the user did + // not choose, if there is one, and otherwise for the first without + // one at all. + let whys: Vec = c + .children + .iter() + .filter(|ch| !remaining.contains(ch.as_str())) + .map(|child| child_without_table(ctx, child, &reasons, &generated)) + .collect(); + if let Some(why) = whys.iter().find(|w| !w.silent).or(whys.first()) { + removed.push((c.fqn.as_str(), why.clone())); + } + } + if removed.is_empty() { + break; + } + for (fqn, why) in removed { + remaining.remove(fqn); + reasons.insert(fqn, why); + } + } + + // A holder removed early may have looked like it fell back only because of + // a message the user set to `Unrolled`, before a message it also holds was + // itself removed for a reason of its own. Look again with the final + // reasons, until no silent holder changes. + loop { + let mut changed = Vec::new(); + for c in selected.iter().filter(|c| { + reasons + .get(c.fqn.as_str()) + .is_some_and(|why| why.silent && !c.children.is_empty()) + }) { + let loud = c + .children + .iter() + .filter(|ch| !remaining.contains(ch.as_str())) + .map(|child| child_without_table(ctx, child, &reasons, &generated)) + .find(|why| !why.silent); + if let Some(why) = loud { + changed.push((c.fqn.as_str(), why)); + } + } + if changed.is_empty() { + break; + } + reasons.extend(changed); + } + + // A rule that names a message exactly and cannot be honoured is an error, + // and all of them are reported together. The rest are counted by reason in + // one warning, except for the messages whose fallback the user chose. + let mut errors = Vec::new(); + let mut summary: Vec = Vec::new(); + let mut fallbacks = 0; + let mut held_back = 0; + for c in &selected { + let Some(why) = reasons.get(c.fqn.as_str()) else { + continue; + }; + if let Some((rule, _)) = ctx.codec_strategy_rule(&c.fqn) { + if *rule == c.fqn { + let advice = why.hint.clone().unwrap_or_else(|| { + format!( + "Select the unrolled codec for it instead (buffa-build: \ + `.codec_strategy_in(CodecStrategy::Unrolled, &[\"{rule}\"])`; plugin: \ + `codec_strategy_in={rule}=unrolled`), or remove the rule" + ) + }); + errors.push(format!( + "codec_strategy_in rule '{rule}' selects the table codec for a message that \ + cannot use it: {}. {advice}", + why.detail + )); + continue; + } + } + if why.silent { + held_back += 1; + continue; + } + fallbacks += 1; + let entry = match summary.iter().position(|r| r.reason == why.reason) { + Some(index) => &mut summary[index], + None => { + summary.push(TableCodecFallbackReason { + reason: why.reason.clone(), + messages: Vec::new(), + }); + summary.last_mut().expect("just pushed") + } + }; + entry.messages.push(c.fqn.clone()); + } + if !errors.is_empty() { + return Err(CodeGenError::Other(errors.join("\n"))); + } + let mut warnings = Vec::new(); + if fallbacks > 0 { + summary.sort_by_key(|reason| std::cmp::Reverse(reason.messages.len())); + warnings.push(CodeGenWarning::TableCodecFallbackSummary { + fallbacks, + selected: selected.len() - held_back, + reasons: summary, + }); + } + + let tables = remaining.into_iter().map(str::to_string).collect(); + Ok((TablePlan { tables }, warnings)) +} diff --git a/buffa-codegen/src/tests/mod.rs b/buffa-codegen/src/tests/mod.rs index fd0f5752..b1729a98 100644 --- a/buffa-codegen/src/tests/mod.rs +++ b/buffa-codegen/src/tests/mod.rs @@ -74,6 +74,7 @@ mod shared_corpus_context; mod shared_pool; mod size_arithmetic; mod skip_debug; +mod table_codec; mod view_codegen; /// Wrap paths as `EnumType(Open)` feature overrides — the shape used by the diff --git a/buffa-codegen/src/tests/table_codec.rs b/buffa-codegen/src/tests/table_codec.rs new file mode 100644 index 00000000..17f06167 --- /dev/null +++ b/buffa-codegen/src/tests/table_codec.rs @@ -0,0 +1,579 @@ +//! `CodecStrategy::Table`: which messages get a table, and what codegen says +//! about the ones that cannot. + +use super::*; + +/// A message field of message type `type_name` (a proto path with a leading dot). +fn message_field(name: &str, number: i32, type_name: &str) -> FieldDescriptorProto { + FieldDescriptorProto { + type_name: Some(type_name.to_string()), + ..make_field(name, number, Label::LABEL_OPTIONAL, Type::TYPE_MESSAGE) + } +} + +fn message(name: &str, fields: Vec) -> DescriptorProto { + DescriptorProto { + name: Some(name.to_string()), + field: fields, + ..Default::default() + } +} + +fn scalar(name: &str, number: i32, ty: Type) -> FieldDescriptorProto { + make_field(name, number, Label::LABEL_OPTIONAL, ty) +} + +/// Package `t` with: +/// +/// - `Plain`, `Leaf`, and `HasLeaf` (holds a `Leaf`), which can use the table; +/// - `Oneofy` (has a oneof) and `HasOneofy` (holds an `Oneofy`), which cannot; +/// - `Outer` with a nested `Inner`, both plain. +fn schema() -> FileDescriptorProto { + let mut oneofy = message( + "Oneofy", + vec![FieldDescriptorProto { + oneof_index: Some(0), + ..scalar("a", 1, Type::TYPE_INT32) + }], + ); + oneofy.oneof_decl = vec![OneofDescriptorProto { + name: Some("choice".to_string()), + ..Default::default() + }]; + let mut outer = message("Outer", vec![scalar("x", 1, Type::TYPE_INT32)]); + outer.nested_type = vec![message("Inner", vec![scalar("y", 1, Type::TYPE_STRING)])]; + FileDescriptorProto { + package: Some("t".to_string()), + message_type: vec![ + message( + "Plain", + vec![ + scalar("a", 1, Type::TYPE_INT32), + scalar("s", 2, Type::TYPE_STRING), + ], + ), + message("Leaf", vec![scalar("x", 1, Type::TYPE_INT32)]), + message("HasLeaf", vec![message_field("leaf", 1, ".t.Leaf")]), + oneofy, + message("HasOneofy", vec![message_field("o", 1, ".t.Oneofy")]), + outer, + ], + ..proto3_file("t.proto") + } +} + +fn run(config: &CodeGenConfig) -> Result<(String, Vec), CodeGenError> { + let (files, warnings) = + generate_with_diagnostics(&[schema()], &["t.proto".to_string()], config)?; + Ok((joined(&files), warnings)) +} + +fn table_config(strategy: CodecStrategy) -> CodeGenConfig { + CodeGenConfig { + codec_strategy: strategy, + ..Default::default() + } +} + +/// `code` without whitespace: prettyplease does not format the inside of a +/// macro call, so its line breaks are arbitrary. +fn squashed(code: &str) -> String { + code.split_whitespace().collect() +} + +/// The messages with a static table in `code`, in order of appearance. +fn tables(code: &str) -> Vec { + code.match_indices("static __BUFFA_TABLE_") + .map(|(i, _)| { + code[i + "static __BUFFA_TABLE_".len()..] + .split(':') + .next() + .unwrap() + .to_string() + }) + .collect() +} + +/// The reasons of the summary warning, as `(reason, number of messages)`, and the counts of +/// fallen-back and selected messages. +fn summary(warnings: &[CodeGenWarning]) -> ((usize, usize), Vec<(&str, usize)>) { + let found: Vec<_> = warnings + .iter() + .filter_map(|w| match w { + CodeGenWarning::TableCodecFallbackSummary { + fallbacks, + selected, + reasons, + } => Some(( + (*fallbacks, *selected), + reasons + .iter() + .map(|r| (r.reason.as_str(), r.messages.len())) + .collect(), + )), + _ => None, + }) + .collect(); + assert_eq!(found.len(), 1, "expected one summary in {warnings:?}"); + found.into_iter().next().unwrap() +} + +fn table_warnings(warnings: &[CodeGenWarning]) -> Vec<&CodeGenWarning> { + warnings + .iter() + .filter(|w| { + matches!( + w, + CodeGenWarning::TableCodecFallbackSummary { .. } + | CodeGenWarning::CodecStrategyRuleMatchedNothing { .. } + ) + }) + .collect() +} + +#[test] +fn the_default_is_unrolled_and_says_nothing() { + let (code, warnings) = run(&CodeGenConfig::default()).unwrap(); + assert!(tables(&code).is_empty()); + assert!(!code.contains("buffa::table")); + assert!(table_warnings(&warnings).is_empty(), "{warnings:?}"); +} + +#[test] +fn the_global_setting_gives_a_table_to_every_message_that_can_use_one() { + let (code, warnings) = run(&table_config(CodecStrategy::Table)).unwrap(); + assert_eq!( + tables(&code), + ["Plain", "Leaf", "HasLeaf", "Outer", "Inner"] + ); + // The other two fall back, and one warning covers the run. + let (counts, reasons) = summary(&warnings); + assert_eq!(counts, (2, 7)); + assert_eq!( + reasons, + [("has a oneof", 1), ("holds a message that has a oneof", 1),] + ); + let text = table_warnings(&warnings)[0].to_string(); + assert!(text.starts_with("2 of 7 messages selected for the table codec")); + assert!(text.contains("has a oneof (1: .t.Oneofy)"), "{text}"); + assert!( + text.contains("holds a message that has a oneof (1: .t.HasOneofy)"), + "{text}" + ); + assert!(text.contains("codec_strategy_in==unrolled"), "{text}"); +} + +#[test] +fn a_table_message_forwards_message_to_the_shared_interpreters() { + let (code, _) = run(&table_config(CodecStrategy::Table)).unwrap(); + assert!(code.contains("::buffa::__table!")); + assert!(code.contains("::buffa::table::ABI")); + assert!(code.contains("__BUFFA_TABLE_Plain.compute_size(self, cache)")); + assert!(code.contains("__BUFFA_TABLE_Plain.merge_field(self, tag, buf, ctx)")); + // None of the per-field code an unrolled impl has: the size and write + // statements for `a` would name `self.a`. + let plain = code + .split("impl ::buffa::Message for Plain") + .nth(1) + .and_then(|rest| rest.split("impl ::buffa::Message for").next()) + .unwrap(); + assert!(!plain.contains("self.a"), "{plain}"); +} + +#[test] +fn a_child_message_is_referenced_through_its_own_table() { + let (code, _) = run(&table_config(CodecStrategy::Table)).unwrap(); + let code = squashed(&code); + let has_leaf = code.split("static__BUFFA_TABLE_HasLeaf").nth(1).unwrap(); + assert!( + has_leaf.contains("Aux::Msg(&::buffa::table::MsgVt::new::<::buffa::MessageField>>(&__BUFFA_TABLE_Leaf))"), + "{has_leaf}" + ); +} + +#[test] +fn a_rule_selects_messages_when_the_global_setting_is_unrolled() { + let config = CodeGenConfig { + codec_strategy_in: vec![(".t.Plain".to_string(), CodecStrategy::Table)], + ..Default::default() + }; + let (code, warnings) = run(&config).unwrap(); + assert_eq!(tables(&code), ["Plain"]); + assert!(table_warnings(&warnings).is_empty(), "{warnings:?}"); +} + +#[test] +fn the_last_matching_rule_wins() { + let config = CodeGenConfig { + codec_strategy_in: vec![ + (".t".to_string(), CodecStrategy::Table), + (".t.Plain".to_string(), CodecStrategy::Unrolled), + (".t.Outer".to_string(), CodecStrategy::Unrolled), + (".t.Outer.Inner".to_string(), CodecStrategy::Table), + ], + ..Default::default() + }; + let (code, _) = run(&config).unwrap(); + // `Outer` is unrolled, though its nested message is a table. + assert_eq!(tables(&code), ["Leaf", "HasLeaf", "Inner"]); +} + +#[test] +fn an_exact_path_rule_for_a_message_that_holds_an_unrolled_child_is_an_error() { + let config = CodeGenConfig { + codec_strategy_in: vec![ + (".t.HasLeaf".to_string(), CodecStrategy::Table), + (".t.Leaf".to_string(), CodecStrategy::Unrolled), + ], + ..Default::default() + }; + // The exact-path rule for `HasLeaf` cannot be honoured: its `Leaf` has no + // table, so this is an error and not a warning. + let err = run(&config).unwrap_err().to_string(); + assert!( + err.contains("codec_strategy_in rule '.t.HasLeaf'") + && err.contains("`.t.Leaf`, which is set to the unrolled codec") + && err.contains("codec_strategy_in=.t.HasLeaf=unrolled"), + "{err}" + ); +} + +#[test] +fn an_exact_path_rule_does_not_select_the_messages_the_message_holds() { + // The global setting stays unrolled, so `Leaf` has not been selected. + let config = CodeGenConfig { + codec_strategy_in: vec![(".t.HasLeaf".to_string(), CodecStrategy::Table)], + ..Default::default() + }; + let err = run(&config).unwrap_err().to_string(); + assert!( + err.contains("`.t.Leaf`, which is not selected for the table codec") + && err.contains("codec_strategy_in(CodecStrategy::Table, &[\".t.Leaf\"])") + && !err.contains("set to the unrolled"), + "{err}" + ); + + // A rule for the child as well gives both a table. + let config = CodeGenConfig { + codec_strategy_in: vec![ + (".t.HasLeaf".to_string(), CodecStrategy::Table), + (".t.Leaf".to_string(), CodecStrategy::Table), + ], + ..Default::default() + }; + let (code, warnings) = run(&config).unwrap(); + assert_eq!(tables(&code), ["Leaf", "HasLeaf"]); + assert!(table_warnings(&warnings).is_empty()); +} + +#[test] +fn every_exact_path_rule_that_cannot_be_honoured_is_reported_at_once() { + let config = CodeGenConfig { + codec_strategy_in: vec![ + (".t.Oneofy".to_string(), CodecStrategy::Table), + (".t.HasOneofy".to_string(), CodecStrategy::Table), + ], + ..Default::default() + }; + let err = run(&config).unwrap_err().to_string(); + assert!( + err.contains("rule '.t.Oneofy'") && err.contains("rule '.t.HasOneofy'"), + "{err}" + ); +} + +#[test] +fn an_exact_path_rule_for_a_message_that_cannot_use_the_table_is_an_error() { + let config = CodeGenConfig { + codec_strategy_in: vec![(".t.Oneofy".to_string(), CodecStrategy::Table)], + ..Default::default() + }; + let err = run(&config).unwrap_err().to_string(); + assert!(err.contains("cannot use it"), "{err}"); + assert!(err.contains("field `a` is in a oneof"), "{err}"); +} + +#[test] +fn a_broad_rule_that_covers_such_a_message_only_warns() { + let config = CodeGenConfig { + codec_strategy_in: vec![(".t".to_string(), CodecStrategy::Table)], + ..Default::default() + }; + let (code, warnings) = run(&config).unwrap(); + assert_eq!( + tables(&code), + ["Plain", "Leaf", "HasLeaf", "Outer", "Inner"] + ); + // Messages a rule selects are counted like the ones the global setting does. + assert_eq!(summary(&warnings).0, (2, 7)); +} + +#[test] +fn setting_a_message_to_unrolled_keeps_the_messages_that_hold_it_unrolled_quietly() { + let config = CodeGenConfig { + codec_strategy_in: vec![(".t.Leaf".to_string(), CodecStrategy::Unrolled)], + ..table_config(CodecStrategy::Table) + }; + let (code, warnings) = run(&config).unwrap(); + // `HasLeaf` holds the `Leaf` the user set to unrolled: not a table, and + // not in the warning either, which is left with the two that cannot. + assert_eq!(tables(&code), ["Plain", "Outer", "Inner"]); + let (counts, reasons) = summary(&warnings); + assert_eq!(counts, (2, 5)); + assert!( + reasons.iter().all(|(r, _)| !r.contains("set to")), + "{reasons:?}" + ); + + // With the two that cannot set to unrolled as well, nothing is left to say. + let config = CodeGenConfig { + codec_strategy_in: vec![ + (".t.Leaf".to_string(), CodecStrategy::Unrolled), + (".t.Oneofy".to_string(), CodecStrategy::Unrolled), + ], + ..table_config(CodecStrategy::Table) + }; + let (_, warnings) = run(&config).unwrap(); + assert!(table_warnings(&warnings).is_empty(), "{warnings:?}"); +} + +#[test] +fn a_rule_that_matches_no_message_warns() { + let config = CodeGenConfig { + codec_strategy_in: vec![ + (".t.Plain".to_string(), CodecStrategy::Table), + (".t.Nope".to_string(), CodecStrategy::Table), + (".t.Plain.a".to_string(), CodecStrategy::Table), + (".elsewhere".to_string(), CodecStrategy::Unrolled), + ], + ..Default::default() + }; + let (_, warnings) = run(&config).unwrap(); + let mut inert: Vec<&str> = warnings + .iter() + .filter_map(|w| match w { + CodeGenWarning::CodecStrategyRuleMatchedNothing { rule } => Some(rule.as_str()), + _ => None, + }) + .collect(); + inert.sort_unstable(); + assert_eq!(inert, [".elsewhere", ".t.Nope", ".t.Plain.a"]); +} + +#[test] +fn a_message_type_from_another_crate_is_not_a_table() { + // `HasLeaf.leaf` now names a type mapped to another crate. + let mut file = schema(); + file.message_type[2].field[0].type_name = Some(".other.Foreign".to_string()); + let other = FileDescriptorProto { + package: Some("other".to_string()), + message_type: vec![message("Foreign", vec![])], + ..proto3_file("other.proto") + }; + let config = CodeGenConfig { + extern_paths: vec![(".other".to_string(), "::other_crate".to_string())], + ..table_config(CodecStrategy::Table) + }; + let (files, warnings) = + generate_with_diagnostics(&[file, other], &["t.proto".to_string()], &config).unwrap(); + let code = joined(&files); + assert!(!tables(&code).contains(&"HasLeaf".to_string()), "{code}"); + assert!(tables(&code).contains(&"Leaf".to_string())); + let text = table_warnings(&warnings)[0].to_string(); + assert!(text.contains(".t.HasLeaf"), "{text}"); + assert!( + text.contains("holds a message that another crate or run generates"), + "{text}" + ); +} + +#[test] +fn a_child_mapped_to_another_crate_falls_back_though_this_run_generates_it_too() { + let mut file = schema(); + file.message_type[2].field[0].type_name = Some(".other.Foreign".to_string()); + let other = FileDescriptorProto { + package: Some("other".to_string()), + message_type: vec![message("Foreign", vec![scalar("x", 1, Type::TYPE_INT32)])], + ..proto3_file("other.proto") + }; + let config = CodeGenConfig { + extern_paths: vec![(".other".to_string(), "::other_crate".to_string())], + ..table_config(CodecStrategy::Table) + }; + // Both files are generated, but `HasLeaf` names `::other_crate::Foreign`, + // which has no table here, so this must be a fallback and not an error. + let (files, warnings) = generate_with_diagnostics( + &[file, other], + &["t.proto".to_string(), "other.proto".to_string()], + &config, + ) + .unwrap(); + let code = joined(&files); + assert!(!tables(&code).contains(&"HasLeaf".to_string()), "{code}"); + assert!(!tables(&code).contains(&"Foreign".to_string())); + assert!(table_warnings(&warnings)[0] + .to_string() + .contains(".t.HasLeaf")); +} + +#[test] +fn type_name_prefix_names_the_static_and_the_child_table() { + let config = CodeGenConfig { + type_name_prefix: "Rpc".to_string(), + ..table_config(CodecStrategy::Table) + }; + let (code, _) = run(&config).unwrap(); + assert!(tables(&code).contains(&"RpcHasLeaf".to_string()), "{code}"); + let has_leaf = squashed(&code); + let has_leaf = has_leaf + .split("static__BUFFA_TABLE_RpcHasLeaf") + .nth(1) + .unwrap(); + assert!(has_leaf.contains("(&__BUFFA_TABLE_RpcLeaf)"), "{has_leaf}"); +} + +#[test] +fn a_message_that_does_not_preserve_unknown_fields_has_no_unknown_slot() { + let config = CodeGenConfig { + preserve_unknown_fields: false, + ..table_config(CodecStrategy::Table) + }; + let (code, _) = run(&config).unwrap(); + let plain = squashed(&code); + let plain = plain.split("static__BUFFA_TABLE_Plain").nth(1).unwrap(); + assert!(plain + .split("impl::buffa::Message") + .next() + .unwrap() + .contains("unknown=none")); + assert!(!squashed(&code).contains("__buffa_unknown_fields")); +} + +#[test] +fn field_numbers_index_the_dense_array_and_missing_ones_are_zero() { + let mut file = schema(); + file.message_type[0].field = vec![ + scalar("c", 3, Type::TYPE_INT32), + scalar("a", 1, Type::TYPE_INT32), + scalar("far", 5000, Type::TYPE_INT32), + ]; + let (files, _) = generate_with_diagnostics( + &[file], + &["t.proto".to_string()], + &table_config(CodecStrategy::Table), + ) + .unwrap(); + let code = joined(&files); + let plain = code.split("static __BUFFA_TABLE_Plain").nth(1).unwrap(); + let plain = plain.split("impl ::buffa::Message").next().unwrap(); + // Entries are in field-number order, and only numbers below 64 are dense. + let plain = squashed(plain); + let (a, c, far) = ( + plain.find("(Plain,a,Int32Implicit,1u32)").expect("a"), + plain.find("(Plain,c,Int32Implicit,3u32)").expect("c"), + plain + .find("(Plain,far,Int32Implicit,5000u32)") + .expect("far"), + ); + assert!(a < c && c < far, "{plain}"); + assert!(plain.contains("dense=&[0u8,1u8,0u8,2u8]"), "{plain}"); +} + +#[test] +fn the_warning_texts_say_what_to_do() { + let rule = CodeGenWarning::CodecStrategyRuleMatchedNothing { + rule: ".x".to_string(), + }; + assert_eq!( + rule.to_string(), + "codec_strategy_in rule '.x' matched no generated message; those messages keep the \ + global codec_strategy setting — check the path against the fully-qualified proto \ + message names" + ); + let reason = TableCodecFallbackReason { + reason: "has a oneof".to_string(), + messages: [".t.A", ".t.B", ".t.C", ".t.D", ".t.E"] + .map(String::from) + .to_vec(), + }; + let summary = CodeGenWarning::TableCodecFallbackSummary { + fallbacks: 5, + selected: 9, + reasons: vec![reason], + }; + let text = summary.to_string(); + // Three messages are named, and the rest are counted. + assert!( + text.contains("has a oneof (5: .t.A, .t.B, .t.C, and 2 more)"), + "{text}" + ); +} + +#[test] +fn the_plan_judges_fields_under_the_same_features_as_the_generator() { + // The file makes message fields DELIMITED (groups), and `Outer` sets + // LENGTH_PREFIXED on itself. The generator gives a top-level message the + // file's features and ignores its own message-level features, so `Outer.i` + // is a group, and `Outer` cannot use the table. A plan that applied + // `Outer`'s own features would select it, and the emitter would then fail + // the build. + use crate::generated::descriptor::{ + feature_set::MessageEncoding, Edition, FeatureSet, FileOptions, + }; + let features = |encoding| { + FeatureSet { + message_encoding: Some(encoding), + ..Default::default() + } + .into() + }; + let inner = message("Inner", vec![scalar("x", 1, Type::TYPE_INT32)]); + let mut outer = message("Outer", vec![message_field("i", 1, ".Inner")]); + outer.options = MessageOptions { + features: features(MessageEncoding::LENGTH_PREFIXED), + ..Default::default() + } + .into(); + let file = FileDescriptorProto { + name: Some("delim.proto".to_string()), + edition: Some(Edition::EDITION_2023), + options: FileOptions { + features: features(MessageEncoding::DELIMITED), + ..Default::default() + } + .into(), + message_type: vec![inner, outer], + ..Default::default() + }; + let (files, warnings) = generate_with_diagnostics( + &[file], + &["delim.proto".to_string()], + &table_config(CodecStrategy::Table), + ) + .expect("the plan and the emitter must agree"); + assert!(tables(&joined(&files)).is_empty()); + assert_eq!(summary(&warnings).0, (2, 2)); +} + +#[test] +fn a_child_the_user_did_not_choose_does_not_hide_one_that_cannot_use_the_table() { + // `Both` holds `Leaf`, which the user sets to unrolled, and then + // `HasOneofy`, which cannot use the table. The first child alone would + // make the fallback silent, but `Both` falls back for the second too. + let mut file = schema(); + file.message_type.push(message( + "Both", + vec![ + message_field("leaf", 1, ".t.Leaf"), + message_field("o", 2, ".t.HasOneofy"), + ], + )); + let config = CodeGenConfig { + codec_strategy_in: vec![(".t.Leaf".to_string(), CodecStrategy::Unrolled)], + ..table_config(CodecStrategy::Table) + }; + let (_, warnings) = + generate_with_diagnostics(&[file], &["t.proto".to_string()], &config).unwrap(); + let text = table_warnings(&warnings)[0].to_string(); + assert!(text.contains(".t.Both"), "{text}"); +} diff --git a/buffa-test/build.rs b/buffa-test/build.rs index 84519326..59cbfee4 100644 --- a/buffa-test/build.rs +++ b/buffa-test/build.rs @@ -1,4 +1,188 @@ +/// Compile a schema twice, with its `package ` renamed to `u` and +/// generated with the default unrolled codec, and to `t` and generated +/// with `codec_strategy = Table`. A test compares the two codecs on the same +/// schema. `file` names the schema in messages. +fn compile_both_codecs(file: &str, source: &str, base: &str) { + let out = std::path::PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR")); + let package = format!("package {base};"); + assert!(source.contains(&package), "{file} must declare `{package}`"); + for (suffix, strategy) in [ + ("u", buffa_build::CodecStrategy::Unrolled), + ("t", buffa_build::CodecStrategy::Table), + ] { + let renamed = out.join(format!("{base}{suffix}.proto")); + std::fs::write( + &renamed, + source.replace(&package, &format!("package {base}{suffix};")), + ) + .expect("write renamed proto"); + buffa_build::Config::new() + .files(&[renamed]) + .includes(&[&out]) + .generate_json(true) + .generate_text(true) + .codec_strategy(strategy) + .compile() + .unwrap_or_else(|e| panic!("buffa_build failed for {file} ({suffix}): {e}")); + } +} + +/// Two packages, the second holding messages of the first, compiled three +/// ways: unrolled (`xau`, `xbu`), table (`xat`, `xbt`), and table with +/// `file_per_package` and `idiomatic_imports` (`xati`, `xbti`), which shortens +/// the paths of types in other packages and so changes what a table path may +/// be. +fn compile_cross_package() { + let out = std::path::PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR")); + let sources = |suffix: &str| { + let dep = format!( + "syntax = \"proto3\";\npackage xa{suffix};\n\ + message Leaf {{ int32 x = 1; string s = 2; }}\n\ + message Wrap {{ Leaf leaf = 1; repeated Leaf leaves = 2; }}\n" + ); + let user = format!( + "syntax = \"proto3\";\npackage xb{suffix};\nimport \"xa{suffix}.proto\";\n\ + message Holder {{\n\ + xa{suffix}.Leaf leaf = 1;\n\ + repeated xa{suffix}.Leaf leaves = 2;\n\ + xa{suffix}.Wrap wrap = 3;\n\ + Sub sub = 4;\n\ + message Sub {{ xa{suffix}.Leaf l = 1; }}\n\ + }}\n" + ); + (dep, user) + }; + for (suffix, strategy, idiomatic) in [ + ("u", buffa_build::CodecStrategy::Unrolled, false), + ("t", buffa_build::CodecStrategy::Table, false), + ("ti", buffa_build::CodecStrategy::Table, true), + ] { + let (dep, user) = sources(suffix); + let (dep_path, user_path) = ( + out.join(format!("xa{suffix}.proto")), + out.join(format!("xb{suffix}.proto")), + ); + std::fs::write(&dep_path, dep).expect("write proto"); + std::fs::write(&user_path, user).expect("write proto"); + let mut config = buffa_build::Config::new() + .files(&[dep_path, user_path]) + .includes(&[&out]) + .codec_strategy(strategy); + if idiomatic { + let dir = out.join("cross_package_idiomatic"); + std::fs::create_dir_all(&dir).expect("create dir"); + config = config + .file_per_package(true) + .idiomatic_imports(true) + .include_file("_include.rs") + .out_dir(dir); + } + config.compile().unwrap_or_else(|e| { + panic!("buffa_build failed for the cross-package schema ({suffix}): {e}") + }); + } +} + +/// The source of `protos/`, which the build depends on. +fn read_proto(file: &str) -> String { + println!("cargo:rerun-if-changed=protos/{file}"); + std::fs::read_to_string(format!("protos/{file}")).expect("read proto") +} + +/// A message with 300 fields, of which 280 are messages, so that a table needs +/// more than 255 entries (which turns off its dense array) and more than 255 +/// field descriptors. +fn wide_proto() -> String { + let mut proto = String::from( + "syntax = \"proto3\";\npackage wide;\nmessage Leaf { int32 x = 1; }\nmessage Wide {\n", + ); + for n in 1..=280 { + proto.push_str(&format!(" Leaf f{n} = {n};\n")); + } + for n in 281..=300 { + proto.push_str(&format!(" int32 f{n} = {n};\n")); + } + proto.push_str("}\n"); + // Messages one field either side of the 255 that a dense array cannot + // index. + for count in [254, 255, 256] { + proto.push_str(&format!("message W{count} {{\n")); + for n in 1..=count { + proto.push_str(&format!(" int32 f{n} = {n};\n")); + } + proto.push_str("}\n"); + } + proto +} + +/// Compile `protos/` as package `x` with the table codec and the +/// options that change the names and fields the table refers to: a type name +/// prefix (`RpcScalars`), no unknown-field slot, boxed message fields, and the +/// lazy view and reflection code that read the same fields. +fn compile_table_with_options(file: &str, base: &str) { + let out = std::path::PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR")); + let source = read_proto(file); + let renamed = out.join(format!("{base}x.proto")); + std::fs::write( + &renamed, + source.replace(&format!("package {base};"), &format!("package {base}x;")), + ) + .expect("write renamed proto"); + buffa_build::Config::new() + .files(&[renamed]) + .includes(&[&out]) + .codec_strategy(buffa_build::CodecStrategy::Table) + .type_name_prefix("Rpc") + .preserve_unknown_fields(false) + .box_type(buffa_build::PointerRepr::Box) + .lazy_views(true) + .reflect_mode(buffa_build::ReflectMode::VTable) + .compile() + .unwrap_or_else(|e| panic!("buffa_build failed for {file} (options): {e}")); +} + +/// The minor version of the compiler building this crate. +fn rustc_minor() -> u32 { + let rustc = std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); + let output = std::process::Command::new(rustc) + .arg("--version") + .output() + .expect("run rustc --version"); + // "rustc 1.75.0 (82e1608df 2023-12-21)" + let version = String::from_utf8_lossy(&output.stdout); + version + .split_whitespace() + .nth(1) + .and_then(|v| v.split('.').nth(1)) + .and_then(|minor| minor.parse().ok()) + .unwrap_or_else(|| panic!("cannot read the rustc version from {version:?}")) +} + fn main() { + // Each schema in `tests::table_codec` is compiled once with the unrolled + // codec and once with the table codec, under renamed packages, so a test can + // compare them. + // Generated table code needs `core::mem::offset_of!`, stable in Rust 1.77, + // and the workspace MSRV is 1.75, where it is a compile error by design. + println!("cargo:rustc-check-cfg=cfg(has_table_codec)"); + if rustc_minor() >= 77 { + println!("cargo:rustc-cfg=has_table_codec"); + compile_both_codecs("table_codec.proto", &read_proto("table_codec.proto"), "tc"); + compile_both_codecs( + "table_codec2.proto", + &read_proto("table_codec2.proto"), + "tc2", + ); + compile_both_codecs( + "table_codec3.proto", + &read_proto("table_codec3.proto"), + "tc3", + ); + compile_both_codecs("the generated wide schema", &wide_proto(), "wide"); + compile_cross_package(); + compile_table_with_options("table_codec.proto", "tc"); + } + // Basic proto — the original test file. Also the codegen target for // bridge-mode reflection (`generate_reflection(true)` emits // `impl Reflectable` per message + a per-package descriptor pool). diff --git a/buffa-test/protos/table_codec.proto b/buffa-test/protos/table_codec.proto new file mode 100644 index 00000000..5ee90e1e --- /dev/null +++ b/buffa-test/protos/table_codec.proto @@ -0,0 +1,187 @@ +syntax = "proto3"; + +// build.rs renames this package to `tcu` (unrolled) and `tct` +// (`codec_strategy = Table`) and compiles both, so a test can compare the two +// codecs on the same schema. Type references stay unqualified so the rename is +// the only difference. +package tc; + +enum Color { + COLOR_UNSPECIFIED = 0; + RED = 1; + GREEN = 2; +} + +message Inner { + int32 id = 1; + string label = 2; + repeated int32 tags = 3; +} + +// Every scalar type with implicit presence. +message Scalars { + int32 i32 = 1; + int64 i64 = 2; + uint32 u32 = 3; + uint64 u64 = 4; + sint32 s32 = 5; + sint64 s64 = 6; + bool b = 7; + fixed32 f32 = 8; + fixed64 f64 = 9; + sfixed32 sf32 = 10; + sfixed64 sf64 = 11; + float fl = 12; + double db = 13; + string s = 14; + bytes by = 15; + Color color = 16; +} + +// The same with explicit presence. +message Optionals { + optional int32 i32 = 1; + optional int64 i64 = 2; + optional uint32 u32 = 3; + optional uint64 u64 = 4; + optional sint32 s32 = 5; + optional sint64 s64 = 6; + optional bool b = 7; + optional fixed32 f32 = 8; + optional fixed64 f64 = 9; + optional sfixed32 sf32 = 10; + optional sfixed64 sf64 = 11; + optional float fl = 12; + optional double db = 13; + optional string s = 14; + optional bytes by = 15; + optional Color color = 16; +} + +// The same, repeated (packed by default in proto3), and unpacked. +message Repeateds { + repeated int32 i32 = 1; + repeated int64 i64 = 2; + repeated uint32 u32 = 3; + repeated uint64 u64 = 4; + repeated sint32 s32 = 5; + repeated sint64 s64 = 6; + repeated bool b = 7; + repeated fixed32 f32 = 8; + repeated fixed64 f64 = 9; + repeated sfixed32 sf32 = 10; + repeated sfixed64 sf64 = 11; + repeated float fl = 12; + repeated double db = 13; + repeated string s = 14; + repeated bytes by = 15; + repeated Color color = 16; + repeated int32 unpacked_i32 = 17 [packed = false]; + repeated Color unpacked_color = 18 [packed = false]; +} + +// Message-typed fields: singular, repeated, and recursive. +message Nested { + Inner inner = 1; + repeated Inner inners = 2; + Scalars scalars = 3; + Optionals optionals = 4; + Repeateds repeateds = 5; + Nested next = 6; + repeated Nested kids = 7; + int32 tail = 8; +} + +// Field numbers that miss the dense lookup array. +message Sparse { + int32 a = 1; + int32 mid = 100; + string far = 5000; + int32 max = 536870911; +} + +message Empty {} + +// A oneof, a map, and a message that holds one: none can use the table, so +// they stay unrolled when the table is requested. +message WithOneof { + oneof choice { + int32 a = 1; + string b = 2; + Inner i = 5; + } + int32 c = 3; +} + +message WithMap { + map m = 1; + map inners = 2; +} + +message HoldsOneof { + WithOneof o = 1; + int32 x = 2; + WithMap m = 3; +} + +// An unrolled message that holds table messages, next to unrolled ones that +// hold a oneof and a map of them. +message Mixed { + Inner inner = 1; + HoldsOneof holds = 2; + Nested nested = 3; + repeated Nested many = 4; +} + +// Field names that are Rust keywords. +message Keywords { + int32 type = 1; + string match = 2; + repeated int32 fn = 3; + Inner ref = 4; +} + +// Nested declarations, references to them from siblings and from inside them, +// mutual recursion, and field numbers around the dense array's limit of 64. +message Outer { + message Inner { + int32 x = 1; + string s = 2; + repeated Deep deeps = 3; + } + message Deep { + int64 y = 1; + Inner back = 2; + } + int32 a = 1; + Inner inner = 2; + Deep deep = 3; + repeated Inner inners = 4; + Outer.Deep qualified = 5; + Color color = 6; + int32 f63 = 63; + int32 f64 = 64; +} + +// Messages named for what generated code refers to. +message Option { + int32 v = 1; +} +message Vec { + Option o = 1; + repeated Option os = 2; +} +message Table { + Vec v = 1; +} +message Kind { + Table t = 1; +} +message Entry { + Kind k = 1; + string s = 2; +} +message Aux { + Entry e = 1; + repeated Entry es = 2; +} diff --git a/buffa-test/protos/table_codec2.proto b/buffa-test/protos/table_codec2.proto new file mode 100644 index 00000000..63f08e98 --- /dev/null +++ b/buffa-test/protos/table_codec2.proto @@ -0,0 +1,79 @@ +syntax = "proto2"; + +// See table_codec.proto: compiled as `tc2u` (unrolled) and `tc2t` (table). +package tc2; + +// Closed, and its first value is not zero. +enum Color { + RED = 1; + GREEN = 2; + BLUE = 3; +} + +message Inner { + optional int32 id = 1; +} + +message Req { + required int32 a = 1; + required string s = 2; + optional int32 d = 3 [default = 7]; + repeated int32 unpacked = 4; + repeated int32 packed = 5 [packed = true]; + optional Color c = 6; + repeated Color cs = 7; + repeated Color packed_cs = 9 [packed = true]; + optional Inner i = 8; + required Inner ri = 10; + optional bytes by = 11 [default = "ab"]; + optional string st = 12 [default = "xyz"]; + optional bool flag = 13 [default = true]; +} + +// A group field, which the table cannot handle, and the group's own type. +message Grouped { + optional int32 a = 1; + optional group G = 2 { + optional int32 x = 1; + } +} + +// Required fields of every type, and repeated fields of every type, which are +// unpacked in proto2. +message AllRequired { + required int32 i32 = 1; + required int64 i64 = 2; + required uint32 u32 = 3; + required uint64 u64 = 4; + required sint32 s32 = 5; + required sint64 s64 = 6; + required bool b = 7; + required fixed32 f32 = 8; + required fixed64 f64 = 9; + required sfixed32 sf32 = 10; + required sfixed64 sf64 = 11; + required float fl = 12; + required double db = 13; + required string s = 14; + required bytes by = 15; + required Color color = 16; +} + +message AllRepeated { + repeated int32 i32 = 1; + repeated int64 i64 = 2; + repeated uint32 u32 = 3; + repeated uint64 u64 = 4; + repeated sint32 s32 = 5; + repeated sint64 s64 = 6; + repeated bool b = 7; + repeated fixed32 f32 = 8; + repeated fixed64 f64 = 9; + repeated sfixed32 sf32 = 10; + repeated sfixed64 sf64 = 11; + repeated float fl = 12; + repeated double db = 13; + repeated string s = 14; + repeated bytes by = 15; + repeated Color color = 16; +} diff --git a/buffa-test/protos/table_codec3.proto b/buffa-test/protos/table_codec3.proto new file mode 100644 index 00000000..109a9d4b --- /dev/null +++ b/buffa-test/protos/table_codec3.proto @@ -0,0 +1,38 @@ +edition = "2023"; + +// See table_codec.proto: compiled as `tc3u` (unrolled) and `tc3t` (table). +// Editions features that change a field's presence, packing, enum openness, or +// UTF-8 handling. +package tc3; + +enum Closed { + option features.enum_type = CLOSED; + C_A = 0; + C_B = 1; +} + +enum OpenE { + O_A = 0; + O_B = 1; +} + +message Child { + int32 z = 1; +} + +message E { + int32 explicit = 1; + int32 implicit = 2 [features.field_presence = IMPLICIT]; + int32 required = 3 [features.field_presence = LEGACY_REQUIRED]; + Closed closed = 4; + repeated int32 expanded = 6 [features.repeated_field_encoding = EXPANDED]; + repeated int32 packed = 7; + repeated Closed closed_rep = 8; + OpenE open = 9; + string s = 10; + bytes b = 11; + string raw = 12 [features.utf8_validation = NONE]; + Child child = 13; + repeated Child children = 14; + string implicit_s = 15 [features.field_presence = IMPLICIT]; +} diff --git a/buffa-test/src/lib.rs b/buffa-test/src/lib.rs index cd9601f1..8bc38f2c 100644 --- a/buffa-test/src/lib.rs +++ b/buffa-test/src/lib.rs @@ -1099,3 +1099,92 @@ mod tests; pub mod string_copy; pub mod string_copy_counted; + +// The table codec is tested by compiling a schema twice under renamed packages +// and comparing the results (see `tests::table_codec`). `tcu`, `tc2u`, `tc3u` +// and `wideu` use the default unrolled codec; `tct`, `tc2t`, `tc3t` and `widet` +// are the same schemas with `codec_strategy = Table`, and `tcx` is `tct` again +// with options that change the names and fields a table refers to. They exist +// only on Rust 1.77 or later (see build.rs). The table modules forbid unsafe +// code, which checks that the `unsafe` a table needs stays inside `buffa`'s +// macros. +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod tcu { + buffa::include_proto!("tcu"); +} +#[forbid(unsafe_code)] +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod tct { + buffa::include_proto!("tct"); +} +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod tc2u { + buffa::include_proto!("tc2u"); +} +#[forbid(unsafe_code)] +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod tc2t { + buffa::include_proto!("tc2t"); +} +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod tc3u { + buffa::include_proto!("tc3u"); +} +#[forbid(unsafe_code)] +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod tc3t { + buffa::include_proto!("tc3t"); +} +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod wideu { + buffa::include_proto!("wideu"); +} +#[forbid(unsafe_code)] +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod widet { + buffa::include_proto!("widet"); +} +#[forbid(unsafe_code)] +#[allow(clippy::derivable_impls, clippy::match_single_binding)] +#[cfg(has_table_codec)] +pub mod tcx { + buffa::include_proto!("tcx"); +} + +// Two packages, the second holding messages of the first: `xau`/`xbu` unrolled, +// `xat`/`xbt` with the table codec, and `xti` with the table codec and +// `file_per_package` with `idiomatic_imports`, which holds `xati` and `xbti`. +#[cfg(has_table_codec)] +pub mod xau { + buffa::include_proto!("xau"); +} +#[cfg(has_table_codec)] +pub mod xbu { + buffa::include_proto!("xbu"); +} +#[forbid(unsafe_code)] +#[cfg(has_table_codec)] +pub mod xat { + buffa::include_proto!("xat"); +} +#[forbid(unsafe_code)] +#[cfg(has_table_codec)] +pub mod xbt { + buffa::include_proto!("xbt"); +} +#[forbid(unsafe_code)] +#[cfg(has_table_codec)] +pub mod xti { + include!(concat!( + env!("OUT_DIR"), + "/cross_package_idiomatic/_include.rs" + )); +} diff --git a/buffa-test/src/tests/mod.rs b/buffa-test/src/tests/mod.rs index 0cb377ae..18bc6ddf 100644 --- a/buffa-test/src/tests/mod.rs +++ b/buffa-test/src/tests/mod.rs @@ -114,6 +114,8 @@ mod skip_debug; mod strict_json; mod string_map; mod string_type; +#[cfg(has_table_codec)] +mod table_codec; mod textproto; mod type_prefix; mod unbox_oneof; diff --git a/buffa-test/src/tests/table_codec.rs b/buffa-test/src/tests/table_codec.rs new file mode 100644 index 00000000..3a59bb55 --- /dev/null +++ b/buffa-test/src/tests/table_codec.rs @@ -0,0 +1,985 @@ +//! `codec_strategy = Table` against the default unrolled codec. +//! +//! `build.rs` compiles each schema under two package names, one unrolled and +//! one with the table codec, so every message exists in both forms; `lib.rs` +//! lists the modules. The tests build the same value in both, and check that +//! the two codecs produce the same bytes and sizes, decode the same values, +//! and reject the same malformed input with the same error. + +use core::fmt::Debug; + +use buffa::{DecodeError, Message}; + +use super::{length_delimited_field, varint_field}; + +/// The error, or the value's re-encoding and `Debug` text. +type Outcome = Result<(Vec, String), DecodeError>; + +/// The `Debug` text has the `Rpc` prefix of the `tcx` types removed, so that +/// they compare with the unprefixed ones. +fn outcome(wire: &[u8]) -> Outcome { + M::decode_from_slice(wire).map(|m| (m.encode_to_vec(), format!("{m:?}").replace("Rpc", ""))) +} + +/// Decode `wire` with both codecs from a buffer of two chunks, split at every +/// offset, and require the outcomes to equal the one from a single slice. +#[track_caller] +fn assert_same_chained(wire: &[u8]) { + use buffa::bytes::Buf; + let (unrolled, table) = (outcome::(wire), outcome::(wire)); + assert_eq!(unrolled, table); + for split in 0..=wire.len() { + let (head, tail) = wire.split_at(split); + let u = U::decode(&mut head.chain(tail)).map(|m| (m.encode_to_vec(), format!("{m:?}"))); + let t = T::decode(&mut head.chain(tail)) + .map(|m| (m.encode_to_vec(), format!("{m:?}").replace("Rpc", ""))); + assert_eq!(u, unrolled, "unrolled, split at {split}"); + assert_eq!(t, table, "table, split at {split}"); + } +} + +/// Decode `wire` with both codecs and require the same outcome. +/// +/// One difference is allowed, between two rejections, and only when +/// `may_overrun`, which is for a message that has sub-messages. The unrolled +/// codec reads a sub-message's fields from the whole buffer, so a string or +/// group that runs past the end of its sub-message is read, and possibly +/// rejected for its content, before the overrun is noticed. The table decodes a +/// sub-message from a slice of exactly its length, so it reports +/// `UnexpectedEof` first. +#[track_caller] +fn assert_same_decode(wire: &[u8], may_overrun: bool) { + let (unrolled, table) = (outcome::(wire), outcome::(wire)); + let both_rejected_and_table_hit_the_end = + may_overrun && unrolled.is_err() && table == Err(DecodeError::UnexpectedEof); + assert!( + unrolled == table || both_rejected_and_table_hit_the_end, + "codecs disagree on {wire:02x?}: unrolled {unrolled:?}, table {table:?}" + ); +} + +/// Both codecs encode `unrolled` and `table` (the same value in each form) to +/// the same bytes and length, and both decode those bytes to the same value. +#[track_caller] +fn assert_same_codec( + unrolled: &U, + table: &T, +) -> Vec { + let wire = unrolled.encode_to_vec(); + assert_eq!(wire, table.encode_to_vec(), "encoded bytes differ"); + assert_eq!(unrolled.encoded_len(), table.encoded_len()); + assert_eq!(wire.len(), table.encoded_len() as usize); + let (mut framed_u, mut framed_t) = (Vec::new(), Vec::new()); + unrolled.encode_length_delimited(&mut framed_u); + table.encode_length_delimited(&mut framed_t); + assert_eq!(framed_u, framed_t, "length-delimited bytes differ"); + assert_eq!( + unrolled.encode_to_bytes(), + table.encode_to_bytes(), + "encode_to_bytes differs" + ); + let decoded_t = T::decode_from_slice(&wire).expect("table decodes its own output"); + assert_eq!(&decoded_t, table); + assert_eq!(decoded_t.encode_to_vec(), wire); + let decoded_u = U::decode_from_slice(&wire).expect("unrolled decodes its own output"); + assert_eq!(&decoded_u, unrolled); + assert_eq!(format!("{decoded_u:?}"), format!("{decoded_t:?}")); + wire +} + +/// Both codecs give the same outcome on every prefix of `wire`, on every input +/// that differs from it in one byte, and on a run of pseudo-random inputs. +#[track_caller] +fn assert_same_on_corrupt_input( + wire: &[u8], + may_overrun: bool, +) { + for end in 0..=wire.len() { + assert_same_decode::(&wire[..end], may_overrun); + } + let mut flipped = wire.to_vec(); + for i in 0..wire.len() { + let original = flipped[i]; + for xor in [0x01, 0x07, 0x80, 0xff] { + flipped[i] = original ^ xor; + assert_same_decode::(&flipped, may_overrun); + } + flipped[i] = original; + } + // xorshift64, so the run is the same every time. + let mut state = 0x9e37_79b9_7f4a_7c15_u64 ^ wire.len() as u64; + let mut next = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for _ in 0..3000 { + let len = (next() % 48) as usize; + let mut noise: Vec = (0..len).map(|_| next() as u8).collect(); + assert_same_decode::(&noise, may_overrun); + // Noise after a valid prefix reaches the later fields. + let keep = (next() as usize) % (wire.len() + 1); + noise.splice(0..0, wire[..keep].iter().copied()); + assert_same_decode::(&noise, may_overrun); + } +} + +/// Builds the same values in a generated module, `$m`: `tcu` or `tct`. +macro_rules! samples { + ($name:ident, $m:ident) => { + mod $name { + use crate::$m::{Color, Inner, Nested, Optionals, Repeateds, Scalars, Sparse}; + use buffa::{EnumValue, MessageField}; + + pub fn inner(id: i32, label: &str, tags: &[i32]) -> Inner { + Inner { + id, + label: label.into(), + tags: tags.to_vec(), + ..Default::default() + } + } + + pub fn scalars() -> Scalars { + Scalars { + i32: -7, + i64: i64::MIN, + u32: u32::MAX, + u64: u64::MAX, + s32: -300, + s64: i64::MAX, + b: true, + f32: 0xdead_beef, + f64: 0x0123_4567_89ab_cdef, + sf32: i32::MIN, + sf64: -5, + fl: 1.5, + db: -2.25, + s: "héllo".into(), + by: vec![0, 255, 7], + color: EnumValue::from(Color::GREEN), + ..Default::default() + } + } + + /// Explicit presence: every field set, several to a zero value. + pub fn optionals() -> Optionals { + Optionals { + i32: Some(0), + i64: Some(-1), + u32: Some(0), + u64: Some(1 << 40), + s32: Some(0), + s64: Some(-1), + b: Some(false), + f32: Some(0), + f64: Some(9), + sf32: Some(0), + sf64: Some(-9), + fl: Some(0.0), + db: Some(f64::MAX), + s: Some(String::new()), + by: Some(Vec::new()), + color: Some(EnumValue::from(Color::COLOR_UNSPECIFIED)), + ..Default::default() + } + } + + pub fn repeateds() -> Repeateds { + Repeateds { + i32: vec![1, -1, 300, i32::MIN], + i64: vec![0, i64::MAX], + u32: vec![u32::MAX, 0], + u64: vec![u64::MAX], + s32: vec![-1, 1, -300], + s64: vec![i64::MIN, 5], + b: vec![true, false, true], + f32: vec![1, 2, 3], + f64: vec![u64::MAX, 0], + sf32: vec![-1, 1], + sf64: vec![i64::MIN], + fl: vec![0.5, -0.5, f32::INFINITY], + db: vec![1.0e300, -0.0], + s: vec!["a".into(), String::new(), "ccc".into()], + by: vec![vec![1, 2], Vec::new(), vec![255]], + color: vec![ + EnumValue::from(Color::RED), + EnumValue::from(Color::GREEN), + EnumValue::from(9), + ], + unpacked_i32: vec![4, 5, 6], + unpacked_color: vec![EnumValue::from(Color::GREEN), EnumValue::from(-3)], + ..Default::default() + } + } + + pub fn nested() -> Nested { + let leaf = Nested { + tail: 3, + ..Default::default() + }; + let mid = Nested { + tail: 2, + next: MessageField::some(leaf.clone()), + kids: vec![leaf.clone(), Nested::default()], + ..Default::default() + }; + Nested { + inner: MessageField::some(inner(1, "one", &[1, 2, 3])), + inners: vec![inner(2, "two", &[]), Inner::default(), inner(3, "", &[9])], + scalars: MessageField::some(scalars()), + optionals: MessageField::some(optionals()), + repeateds: MessageField::some(repeateds()), + next: MessageField::some(mid.clone()), + kids: vec![mid, leaf], + tail: 1, + ..Default::default() + } + } + + pub fn sparse() -> Sparse { + Sparse { + a: 1, + mid: -2, + far: "far".into(), + max: 3, + ..Default::default() + } + } + } + }; +} + +samples!(u, tcu); +samples!(t, tct); + +#[test] +fn scalars_agree() { + let wire = assert_same_codec(&u::scalars(), &t::scalars()); + assert_same_on_corrupt_input::(&wire, false); +} + +#[test] +fn explicit_presence_is_kept_for_zero_values() { + let wire = assert_same_codec(&u::optionals(), &t::optionals()); + // Every field is on the wire, though many hold the type's zero value. + let decoded = ::decode_from_slice(&wire).unwrap(); + assert_eq!(decoded.i32, Some(0)); + assert_eq!(decoded.s, Some(String::new())); + assert_same_on_corrupt_input::(&wire, false); +} + +#[test] +fn repeated_fields_agree() { + let wire = assert_same_codec(&u::repeateds(), &t::repeateds()); + assert_same_on_corrupt_input::(&wire, false); +} + +#[test] +fn nested_messages_agree() { + let wire = assert_same_codec(&u::nested(), &t::nested()); + assert_same_on_corrupt_input::(&wire, true); +} + +#[test] +fn sparse_field_numbers_agree() { + let wire = assert_same_codec(&u::sparse(), &t::sparse()); + assert_same_on_corrupt_input::(&wire, false); +} + +#[test] +fn empty_and_default_messages_encode_to_nothing() { + assert!(crate::tct::Nested::default().encode_to_vec().is_empty()); + assert!(crate::tct::Empty::default().encode_to_vec().is_empty()); + assert_eq!(crate::tct::Nested::default().encoded_len(), 0); +} + +#[test] +fn unpacked_and_packed_input_are_both_accepted() { + // Field 1 of `Repeateds` is packed; a sender may write it unpacked, and + // field 17 is unpacked but a sender may pack it. + let mut wire = Vec::new(); + wire.extend(varint_field(1, 5)); + wire.extend(varint_field(1, 6)); + wire.extend(length_delimited_field(17, &[7, 8])); + wire.extend(varint_field(17, 9)); + assert_same_decode::(&wire, false); + let decoded = ::decode_from_slice(&wire).unwrap(); + assert_eq!(decoded.i32, [5, 6]); + assert_eq!(decoded.unpacked_i32, [7, 8, 9]); +} + +#[test] +fn unknown_fields_are_kept() { + let mut wire = u::scalars().encode_to_vec(); + wire.extend(varint_field(999, 42)); + wire.extend(length_delimited_field(1000, b"xyz")); + // A group with an unknown field inside. + wire.extend([0xf3, 0x3e]); + wire.extend(varint_field(1, 5)); + wire.extend([0xf4, 0x3e]); + assert_same_decode::(&wire, false); + let decoded = ::decode_from_slice(&wire).unwrap(); + assert!(!decoded.__buffa_unknown_fields.is_empty()); + assert_eq!(decoded.encode_to_vec().len(), wire.len()); + // Unknown fields come back out, and clear() drops them. + let mut cleared = decoded; + cleared.clear(); + assert_eq!(cleared, crate::tct::Scalars::default()); +} + +#[test] +fn a_known_field_with_the_wrong_wire_type_is_rejected_alike() { + // `i32` (field 1) as a length-delimited record, `s` (field 14) as a varint. + let mut wire = length_delimited_field(1, b"ab"); + wire.extend(varint_field(14, 3)); + assert_same_decode::(&wire, false); +} + +#[test] +fn merging_appends_repeated_and_merges_singular_message_fields() { + let first = t::nested().encode_to_vec(); + let second = { + let mut m = t::nested(); + m.tail = 99; + m.inner = buffa::MessageField::some(t::inner(50, "", &[4])); + m.encode_to_vec() + }; + let mut merged_t = crate::tct::Nested::default(); + merged_t.merge_from_slice(&first).unwrap(); + merged_t.merge_from_slice(&second).unwrap(); + let mut merged_u = crate::tcu::Nested::default(); + merged_u.merge_from_slice(&first).unwrap(); + merged_u.merge_from_slice(&second).unwrap(); + assert_eq!(merged_t.encode_to_vec(), merged_u.encode_to_vec()); + assert_eq!(format!("{merged_t:?}"), format!("{merged_u:?}")); + // The repeated field doubled, and the label of the merged `inner` survived + // the second message's empty label. + assert_eq!(merged_t.kids.len(), 4); + assert_eq!(merged_t.inner.label, "one"); + assert_eq!(merged_t.inner.id, 50); + assert_eq!(merged_t.tail, 99); +} + +#[test] +fn recursion_depth_is_limited_like_the_unrolled_codec() { + // `next` nested a thousand deep. + let mut wire = Vec::new(); + for _ in 0..1000 { + let mut outer = length_delimited_field(6, &wire); + std::mem::swap(&mut outer, &mut wire); + } + assert_same_decode::(&wire, true); + assert!(::decode_from_slice(&wire).is_err()); +} + +#[test] +fn invalid_utf8_is_rejected() { + let wire = length_delimited_field(14, &[0xff, 0xfe]); + assert_eq!( + ::decode_from_slice(&wire), + Err(DecodeError::InvalidUtf8) + ); + assert_same_decode::(&wire, false); +} + +#[test] +fn messages_the_table_cannot_handle_still_work() { + // A oneof, a map, and the messages that hold them are unrolled, and a + // table message may sit next to them. + let with_oneof = crate::tct::WithOneof { + choice: Some(crate::tct::with_oneof::Choice::B("x".into())), + c: 4, + ..Default::default() + }; + let decoded = + ::decode_from_slice(&with_oneof.encode_to_vec()).unwrap(); + assert_eq!(decoded, with_oneof); + + let mixed = crate::tct::Mixed { + inner: buffa::MessageField::some(t::inner(1, "a", &[1])), + holds: buffa::MessageField::some(crate::tct::HoldsOneof { + o: buffa::MessageField::some(with_oneof), + x: 2, + ..Default::default() + }), + ..Default::default() + }; + let decoded = + ::decode_from_slice(&mixed.encode_to_vec()).unwrap(); + assert_eq!(decoded, mixed); +} + +// --------------------------------------------------------------------------- +// proto2: required fields, defaults, closed enums +// --------------------------------------------------------------------------- + +macro_rules! req_samples { + ($name:ident, $m:ident) => { + mod $name { + use crate::$m::{Color, Inner, Req}; + use buffa::MessageField; + + pub fn req() -> Req { + Req { + a: 0, + s: String::new(), + d: Some(7), + unpacked: vec![1, 2, 3], + packed: vec![-1, 4, 5], + c: Some(Color::BLUE), + cs: vec![Color::RED, Color::GREEN], + packed_cs: vec![Color::BLUE, Color::RED], + i: MessageField::some(Inner { + id: Some(4), + ..Default::default() + }), + ri: MessageField::some(Inner::default()), + by: Some(b"ab".to_vec()), + st: Some("xyz".into()), + flag: Some(false), + ..Default::default() + } + } + } + }; +} + +req_samples!(req_u, tc2u); +req_samples!(req_t, tc2t); + +#[test] +fn proto2_required_and_defaulted_fields_agree() { + let wire = assert_same_codec(&req_u::req(), &req_t::req()); + // `a` and `s` are required, so they are written though they hold zero and "". + assert!(wire.starts_with(&[0x08, 0x00, 0x12, 0x00])); + assert_same_on_corrupt_input::(&wire, true); +} + +#[test] +fn a_message_that_omits_its_required_fields_encodes_them_anyway() { + let wire = crate::tc2t::Req::default().encode_to_vec(); + assert_eq!(wire, crate::tc2u::Req::default().encode_to_vec()); +} + +#[test] +fn closed_enum_values_without_a_variant_go_to_unknown_fields() { + // `c` (field 6), `cs` (7, unpacked) and `packed_cs` (9) with the values 99 + // and 2 (GREEN). + let mut wire = varint_field(6, 99); + wire.extend(varint_field(7, 99)); + wire.extend(varint_field(7, 2)); + wire.extend(length_delimited_field(9, &[99, 2])); + assert_same_decode::(&wire, false); + let decoded = ::decode_from_slice(&wire).unwrap(); + assert_eq!(decoded.c, None); + assert_eq!(decoded.cs, [crate::tc2t::Color::GREEN]); + assert_eq!(decoded.packed_cs, [crate::tc2t::Color::GREEN]); + assert!(!decoded.__buffa_unknown_fields.is_empty()); +} + +#[test] +fn groups_stay_unrolled() { + let grouped = crate::tc2t::Grouped { + a: Some(1), + g: buffa::MessageField::some(crate::tc2t::grouped::G { + x: Some(2), + ..Default::default() + }), + ..Default::default() + }; + let decoded = + ::decode_from_slice(&grouped.encode_to_vec()).unwrap(); + assert_eq!(decoded, grouped); +} + +#[test] +fn a_table_generated_with_a_type_prefix_boxed_fields_and_no_unknown_slot_agrees() { + // `tcx` has `RpcNested` and friends, boxes its message fields, and drops + // unknown fields, so it agrees with `tcu` on everything it keeps. + let wire = u::nested().encode_to_vec(); + let decoded = ::decode_from_slice(&wire).unwrap(); + assert_eq!(decoded.encode_to_vec(), wire); + assert_eq!(decoded.encoded_len() as usize, wire.len()); + + let mut with_unknown = wire.clone(); + with_unknown.extend(varint_field(999, 1)); + let decoded = ::decode_from_slice(&with_unknown).unwrap(); + assert_eq!( + decoded.encode_to_vec(), + wire, + "the unknown field is dropped" + ); +} + +// --------------------------------------------------------------------------- +// More schema shapes: nested declarations, shadowed names, every kind, editions +// features, and a message wider than the dense array +// --------------------------------------------------------------------------- + +macro_rules! shape_samples { + ($name:ident, $m:ident, $m2:ident, $m3:ident, $wide:ident) => { + mod $name { + use crate::$m::outer::{Deep, Inner}; + use crate::$m::{ + Aux, Color, Entry, Kind, Mixed, Option as Opt, Outer, Table, Vec as Vc, WithOneof, + }; + use buffa::{EnumValue, MessageField}; + + pub fn outer() -> Outer { + let deep = Deep { + y: -5, + back: MessageField::some(Inner { + x: 1, + s: "in".into(), + deeps: vec![Deep { + y: 2, + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + Outer { + a: 1, + inner: MessageField::some(Inner { + x: 2, + s: "s".into(), + deeps: vec![deep.clone(), Deep::default()], + ..Default::default() + }), + deep: MessageField::some(deep.clone()), + inners: vec![ + Inner::default(), + Inner { + x: 9, + ..Default::default() + }, + ], + qualified: MessageField::some(deep), + color: EnumValue::from(Color::RED), + f63: 63, + f64: 64, + ..Default::default() + } + } + + pub fn aux() -> Aux { + let option = |v| Opt { + v, + ..Default::default() + }; + let entry = Entry { + k: MessageField::some(Kind { + t: MessageField::some(Table { + v: MessageField::some(Vc { + o: MessageField::some(option(1)), + os: vec![option(2), option(3)], + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }), + s: "e".into(), + ..Default::default() + }; + Aux { + e: MessageField::some(entry.clone()), + es: vec![entry, Entry::default()], + ..Default::default() + } + } + + pub fn mixed() -> Mixed { + mixed_with_map(true) + } + + /// Without the map, whose entries a corrupted input can multiply and + /// whose `Debug` order then differs between two `HashMap`s. + pub fn mixed_without_map() -> Mixed { + mixed_with_map(false) + } + + fn mixed_with_map(with_map: bool) -> Mixed { + let nested = || crate::$m::Nested { + inner: MessageField::some(inner()), + inners: vec![inner(), crate::$m::Inner::default()], + next: MessageField::some(crate::$m::Nested { + tail: 7, + ..Default::default() + }), + tail: 1, + ..Default::default() + }; + Mixed { + inner: MessageField::some(inner()), + holds: MessageField::some(crate::$m::HoldsOneof { + o: MessageField::some(WithOneof { + choice: Some(crate::$m::with_oneof::Choice::I(Box::new(inner()))), + c: 4, + ..Default::default() + }), + x: 2, + m: if with_map { + MessageField::some(crate::$m::WithMap { + inners: [("k".to_string(), inner())].into_iter().collect(), + ..Default::default() + }) + } else { + MessageField::none() + }, + ..Default::default() + }), + nested: MessageField::some(nested()), + many: vec![nested(), crate::$m::Nested::default()], + ..Default::default() + } + } + + pub fn inner() -> crate::$m::Inner { + crate::$m::Inner { + id: 3, + label: "l".into(), + tags: vec![1, 2], + ..Default::default() + } + } + + pub fn all_required() -> crate::$m2::AllRequired { + use crate::$m2::Color as C2; + crate::$m2::AllRequired { + i32: -1, + i64: -2, + u32: 3, + u64: 4, + s32: -5, + s64: -6, + b: true, + f32: 7, + f64: 8, + sf32: -9, + sf64: -10, + fl: 1.25, + db: -2.5, + s: "req".into(), + by: vec![9, 8], + color: C2::BLUE, + ..Default::default() + } + } + + pub fn all_repeated() -> crate::$m2::AllRepeated { + use crate::$m2::Color as C2; + crate::$m2::AllRepeated { + i32: vec![1, -2], + i64: vec![3, -4], + u32: vec![5, 6], + u64: vec![7], + s32: vec![-8, 9], + s64: vec![-10], + b: vec![true, false], + f32: vec![11, 12], + f64: vec![13], + sf32: vec![-14], + sf64: vec![-15, 16], + fl: vec![0.5, 1.5], + db: vec![2.5], + s: vec!["a".into(), "".into()], + by: vec![vec![1], vec![]], + color: vec![C2::RED, C2::GREEN], + ..Default::default() + } + } + + pub fn editions() -> crate::$m3::E { + use crate::$m3::{Child, Closed, OpenE}; + crate::$m3::E { + explicit: Some(0), + implicit: 5, + required: 0, + closed: Some(Closed::C_B), + expanded: vec![1, 2, 3], + packed: vec![4, 5, 6], + closed_rep: vec![Closed::C_A, Closed::C_B], + open: Some(EnumValue::from(OpenE::O_B)), + s: Some("s".into()), + b: Some(vec![1]), + raw: Some("raw".into()), + child: MessageField::some(Child { + z: Some(1), + ..Default::default() + }), + children: vec![ + Child::default(), + Child { + z: Some(2), + ..Default::default() + }, + ], + implicit_s: "i".into(), + ..Default::default() + } + } + + pub fn wide() -> crate::$wide::Wide { + use crate::$wide::Leaf; + let leaf = |x| { + MessageField::some(Leaf { + x, + ..Default::default() + }) + }; + crate::$wide::Wide { + f1: leaf(1), + f128: leaf(128), + f256: leaf(256), + f280: leaf(280), + f281: 281, + f300: -300, + ..Default::default() + } + } + } + }; +} + +shape_samples!(shapes_u, tcu, tc2u, tc3u, wideu); +shape_samples!(shapes_t, tct, tc2t, tc3t, widet); + +#[test] +fn nested_declarations_and_mutual_recursion_agree() { + let wire = assert_same_codec(&shapes_u::outer(), &shapes_t::outer()); + assert_same_on_corrupt_input::(&wire, true); +} + +#[test] +fn messages_named_like_what_generated_code_uses_agree() { + let wire = assert_same_codec(&shapes_u::aux(), &shapes_t::aux()); + assert_same_on_corrupt_input::(&wire, true); +} + +#[test] +fn a_table_message_inside_an_unrolled_tree_agrees() { + // `Mixed.inner` is a table in `tct`, and `Mixed` and `HoldsOneof` are not. + let wire = assert_same_codec(&shapes_u::mixed(), &shapes_t::mixed()); + assert_same_chained::(&wire); + let wire = assert_same_codec( + &shapes_u::mixed_without_map(), + &shapes_t::mixed_without_map(), + ); + assert_same_on_corrupt_input::(&wire, true); +} + +#[test] +fn required_fields_of_every_type_agree() { + let wire = assert_same_codec(&shapes_u::all_required(), &shapes_t::all_required()); + assert_same_on_corrupt_input::( + &wire, false, + ); + // And the defaults are written too. + let defaults = crate::tc2t::AllRequired::default().encode_to_vec(); + assert_eq!( + defaults, + crate::tc2u::AllRequired::default().encode_to_vec() + ); + assert!(!defaults.is_empty()); +} + +#[test] +fn unpacked_repeated_fields_of_every_type_agree() { + let wire = assert_same_codec(&shapes_u::all_repeated(), &shapes_t::all_repeated()); + assert_same_on_corrupt_input::( + &wire, false, + ); +} + +#[test] +fn editions_features_agree() { + let wire = assert_same_codec(&shapes_u::editions(), &shapes_t::editions()); + assert_same_on_corrupt_input::(&wire, true); + // A closed enum value with no variant is unknown, as in unrolled code. + let mut unknown = varint_field(4, 99); + unknown.extend(varint_field(5, 99)); + assert_same_decode::(&unknown, false); +} + +#[test] +fn a_message_with_more_than_255_fields_agrees() { + let wire = assert_same_codec(&shapes_u::wide(), &shapes_t::wide()); + assert_same_on_corrupt_input::(&wire, true); +} + +#[test] +fn nested_messages_decode_the_same_from_a_buffer_of_two_chunks() { + assert_same_chained::(&u::nested().encode_to_vec()); + let outer = shapes_u::outer().encode_to_vec(); + assert_same_chained::(&outer); + let editions = shapes_u::editions().encode_to_vec(); + assert_same_chained::(&editions); +} + +macro_rules! edge_widths { + ($($name:ident: $ty:ident, $last:ident;)*) => {$( + #[test] + fn $name() { + let unrolled = crate::wideu::$ty { f1: 1, $last: 9, ..Default::default() }; + let table = crate::widet::$ty { f1: 1, $last: 9, ..Default::default() }; + let wire = assert_same_codec(&unrolled, &table); + assert_same_on_corrupt_input::(&wire, false); + } + )*}; +} + +edge_widths! { + a_message_with_254_fields_agrees: W254, f254; + a_message_with_255_fields_agrees: W255, f255; + a_message_with_256_fields_agrees: W256, f256; +} + +#[test] +fn fields_named_like_rust_keywords_agree() { + use buffa::MessageField; + macro_rules! keywords { + ($m:ident) => { + crate::$m::Keywords { + r#type: 1, + r#match: "m".into(), + r#fn: vec![2, 3], + r#ref: MessageField::some(crate::$m::Inner { + id: 4, + ..Default::default() + }), + ..Default::default() + } + }; + } + let wire = assert_same_codec(&keywords!(tcu), &keywords!(tct)); + assert_same_on_corrupt_input::(&wire, true); +} + +#[test] +fn negative_zero_and_nan_are_written_by_both_codecs() { + // Implicit-presence floats are skipped when their bits are zero, and + // `-0.0` and NaN are not zero bits. + macro_rules! floats { + ($m:ident) => { + crate::$m::Scalars { + fl: -0.0, + db: f64::NAN, + ..Default::default() + } + }; + } + let (unrolled, table) = (floats!(tcu), floats!(tct)); + assert_eq!(unrolled.encode_to_vec(), table.encode_to_vec()); + assert!(!table.encode_to_vec().is_empty()); + let zero = crate::tct::Scalars::default(); + assert!(zero.encode_to_vec().is_empty()); +} + +#[test] +fn clear_restores_proto2_defaults_like_unrolled_code() { + let wire = { + let mut wire = varint_field(1, 5); + wire.extend(length_delimited_field(2, b"s")); + wire.extend(varint_field(3, 9)); + wire.extend(length_delimited_field(10, &varint_field(1, 1))); + wire + }; + let mut unrolled = crate::tc2u::Req::decode_from_slice(&wire).unwrap(); + let mut table = crate::tc2t::Req::decode_from_slice(&wire).unwrap(); + unrolled.clear(); + table.clear(); + assert_eq!(format!("{unrolled:?}"), format!("{table:?}")); + assert_eq!(unrolled.encode_to_vec(), table.encode_to_vec()); +} + +/// Referencing a message's table is a compile error when it fell back to +/// unrolled code, so these pin which messages the table really covers. +#[test] +fn the_messages_the_table_can_handle_use_it() { + fn is_table(_: &'static buffa::table::Table) {} + macro_rules! tables { + ($($path:path),* $(,)?) => { $( is_table(&$path); )* }; + } + tables!( + crate::tct::__BUFFA_TABLE_Scalars, + crate::tct::__BUFFA_TABLE_Optionals, + crate::tct::__BUFFA_TABLE_Repeateds, + crate::tct::__BUFFA_TABLE_Nested, + crate::tct::__BUFFA_TABLE_Sparse, + crate::tct::__BUFFA_TABLE_Empty, + crate::tct::__BUFFA_TABLE_Inner, + crate::tct::__BUFFA_TABLE_Outer, + crate::tct::outer::__BUFFA_TABLE_Inner, + crate::tct::outer::__BUFFA_TABLE_Deep, + crate::tct::__BUFFA_TABLE_Option, + crate::tct::__BUFFA_TABLE_Vec, + crate::tct::__BUFFA_TABLE_Table, + crate::tct::__BUFFA_TABLE_Kind, + crate::tct::__BUFFA_TABLE_Entry, + crate::tct::__BUFFA_TABLE_Aux, + crate::tct::__BUFFA_TABLE_Keywords, + crate::tc2t::__BUFFA_TABLE_Req, + crate::tc2t::__BUFFA_TABLE_AllRequired, + crate::tc2t::__BUFFA_TABLE_AllRepeated, + crate::tc3t::__BUFFA_TABLE_E, + crate::tc3t::__BUFFA_TABLE_Child, + crate::widet::__BUFFA_TABLE_Wide, + crate::widet::__BUFFA_TABLE_W254, + crate::widet::__BUFFA_TABLE_W255, + crate::widet::__BUFFA_TABLE_W256, + crate::tcx::__BUFFA_TABLE_RpcNested, + crate::xat::__BUFFA_TABLE_Leaf, + crate::xat::__BUFFA_TABLE_Wrap, + crate::xbt::__BUFFA_TABLE_Holder, + crate::xbt::holder::__BUFFA_TABLE_Sub, + crate::xti::xati::__BUFFA_TABLE_Leaf, + crate::xti::xati::__BUFFA_TABLE_Wrap, + crate::xti::xbti::__BUFFA_TABLE_Holder, + crate::xti::xbti::holder::__BUFFA_TABLE_Sub, + ); +} + +#[test] +fn messages_held_across_packages_agree_in_every_layout() { + use buffa::MessageField; + macro_rules! sample { + ($xa:ident, $xb:ident) => {{ + let leaf = |x| $xa::Leaf { + x, + s: "s".into(), + ..Default::default() + }; + $xb::Holder { + leaf: MessageField::some(leaf(1)), + leaves: vec![leaf(2), leaf(3)], + wrap: MessageField::some($xa::Wrap { + leaf: MessageField::some(leaf(4)), + leaves: vec![leaf(5)], + ..Default::default() + }), + sub: MessageField::some($xb::holder::Sub { + l: MessageField::some(leaf(6)), + ..Default::default() + }), + ..Default::default() + } + }}; + } + use crate::xti::{xati, xbti}; + use crate::{xat, xau, xbt, xbu}; + let unrolled = sample!(xau, xbu); + let table = sample!(xat, xbt); + let idiomatic = sample!(xati, xbti); + let wire = assert_same_codec(&unrolled, &table); + assert_eq!(idiomatic.encode_to_vec(), wire); + assert_eq!( + xbti::Holder::decode_from_slice(&wire) + .unwrap() + .encode_to_vec(), + wire + ); +} diff --git a/docs/guide.md b/docs/guide.md index 3ae710fe..b0117437 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -216,6 +216,8 @@ The macro pulls in `OUT_DIR/.mod.rs`, which in turn includes the per | `.extern_path(proto, rust)` | — | Map a proto package or a single type to an external Rust path (see below) | | `.exclude_package(pkg)` | — | Drop a proto package (and its sub-packages) from code generation. Useful when directory globbing pulls in option-only packages (e.g. `buf.validate`) that you don't want Rust types for. A leading dot is accepted and stripped. Pair with `.extern_path` if kept files reference types from the excluded package; the generator emits a `cargo:warning` for each such cross-package reference. | | `.type_name_prefix(prefix)` | `""` | Prepend a PascalCase prefix (`[A-Z][A-Za-z0-9]*`; anything else is rejected at generation time) to every generated message/enum type name (`message User` → `struct RpcUser`); modules, oneof enums, extern-mapped types, and the wire format are unaffected. A crate referencing these types via `extern_path` must spell out the prefixed name (`::crate_a::RpcUser`) | +| `.codec_strategy(strategy)` | `Unrolled` | Generate each message's binary `Message` code specialised to its fields (`CodecStrategy::Unrolled`), or from a static table and interpreters shared by every message (`CodecStrategy::Table`), which on a schema it fully covers is about half the compiled size and slower on messages of many small fields; see [Smaller generated code](#smaller-generated-code-codec_strategy) | +| `.codec_strategy_in(strategy, &[...])` | — | Choose the strategy for matching messages and the messages nested in them (proto-path prefixes; the last matching rule wins), on top of the global setting | | `.use_bytes_type()` | — | Use `bytes::Bytes` for all bytes fields, including `map` values | | `.use_bytes_type_in(&[...])` | — | Use `bytes::Bytes` for matching bytes fields (same `map` rule) | | `.string_type_custom(path)` | `String` | Use a custom owned string representation that implements `ProtoString`, named by Rust path (e.g. `"::my_crate::SmolStr"`), for all string fields (see [String and bytes field representations](#string-and-bytes-field-representations)) | @@ -630,6 +632,8 @@ Passed via `opt:` (works for `remote:` and `local:`): | `idiomatic_enum_aliases=false` | Omit the `UpperCamelCase` associated-const aliases for enum values (`Status::Active`); the `SHOUTY_SNAKE_CASE` variants are unaffected (default: emitted). See [Enums](#enumvaluet--type-safe-open-enums) | | `override_feature_in==:` | Apply a path-scoped editions feature override (currently `enum_type:OPEN`) to the compiled descriptors. Repeatable | | `open_enums_in=` | Shorthand for `override_feature_in==enum_type:OPEN`. Repeatable | +| `codec_strategy=table` | Generate every message's binary `Message` code from a static table and shared interpreters instead of code specialised to its fields (default `unrolled`). The plugin cannot check the compiler version: on Rust before 1.77 the generated code fails to compile. See [Smaller generated code](#smaller-generated-code-codec_strategy) | +| `codec_strategy_in==` | Choose `table` or `unrolled` for matching messages and the messages nested in them. Repeatable; leading dot optional; the last matching rule wins | | `unbox_oneof=true` | Store every non-recursive message/group oneof variant inline instead of `Box`. Recursive variants stay boxed. | | `unbox_oneof_in=` | Store matching non-recursive message/group oneof variants inline instead of `Box`. Repeatable; leading dot optional. Use `.` to match all variants. Recursive variants stay boxed for broad matches; exact recursive matches are rejected. | | `reflection=true` | Emit reflection support (vtable mode) plus an embedded per-package descriptor pool — see [Runtime reflection](#runtime-reflection) | @@ -1094,6 +1098,44 @@ Buffa uses a two-pass model to avoid the exponential-time size computation that `encode()`, `encode_to_vec()`, and `encode_to_bytes()` perform both passes with a fresh `SizeCache` automatically — most callers never name the cache. Use `encoded_len()` if you only need the size. +### Smaller generated code: `codec_strategy` + +By default every generated message contains its own size, write, and merge code, specialised to its fields. `CodecStrategy::Table` replaces it with one static table per message and interpreters in `buffa` that every message shares. On a schema of 334 messages and 3,477 fields, the compiled size at `opt-level = "z"` went from 1,644 KB to 817 KB (measured with oneofs and maps flattened, which the table cannot handle; see [#463](https://github.com/anthropics/buffa/issues/463) for the method). The cost is speed on messages made of many small fields, where encoding takes up to about 3.5 times as long as with the default `CodecStrategy::Unrolled` and decoding up to 1.6 times. Messages dominated by bulk data, such as large strings, bytes, and packed arrays, show no difference. + +```rust,ignore +// build.rs +buffa_build::Config::new() + .files(&["proto/wa.proto"]) + .includes(&["proto/"]) + .codec_strategy(buffa_build::CodecStrategy::Table) + // Keep the hot messages specialised. + .codec_strategy_in(buffa_build::CodecStrategy::Unrolled, &[".wa.Message", ".wa.Receipt"]) + .compile()?; +``` + +A table holds only table messages, so a message that holds an unrolled one is unrolled too, and a `codec_strategy_in` rule for a message does not select the messages it holds: with the global setting left at `Unrolled`, select a message and everything it holds. In the example, every message that contains `.wa.Message` stays unrolled, and that usually includes the root message an application encodes; codegen does not warn about a fallback that follows from your own `Unrolled` rule. + +The option changes only the binary `Message` implementation. The wire format and the JSON, text, view, and reflection code are the same under both strategies, and a table message encodes to the same bytes and decodes the same accepted input as its unrolled twin. It differs in three ways: + +- A field that declares a length past the end of its enclosing message fails at once with `DecodeError::UnexpectedEof`, where unrolled code reads on into the enclosing message and can report a different error for the same rejected input. +- The table decodes from one contiguous slice, so a `Buf` that is not contiguous is gathered into one buffer first. `Message::merge_field` on a table message cannot gather, and returns `UnexpectedEof` for such a buffer; only code that calls it directly is affected, such as the default `merge_group`. A message that another crate or another codegen run uses as the type of a group or `DELIMITED` field must therefore stay `Unrolled`. Within one run, codegen keeps the type of a group field unrolled itself. +- `clear()` resets a table message to `Default`, so it releases the capacity of its strings and vectors instead of keeping it. + +These stay unrolled, whatever the setting: + +- a message with a `oneof`, a `map`, or a group field; +- the message type of a group field; +- a message that uses the `MessageSet` wire format; +- a message with extension ranges, when JSON code is generated and unknown fields are preserved; +- a message with a field of a non-default string, bytes, or collection type, which `use_bytes_type`, `string_type`, `bytes_type`, and `repeated_type` select; +- a message that holds any message that stays unrolled, is not selected for the table, or is generated by another crate, such as a well-known type. + +Codegen prints one warning per run that counts the messages that fell back, groups them by reason, and names a few of each. Setting the messages that cause a fallback to `Unrolled` with `codec_strategy_in` silences it. A `codec_strategy_in` rule that selects the table for a message by its exact path, when the message cannot use it, is an error, because the rule asked for something impossible. + +The table code needs Rust 1.77 or later; `buffa-build` returns an error on an older compiler, and the plugin's output does not compile on one. It contains `unsafe` code, in macros inside `buffa`, so the generated code compiles in a crate with `#![forbid(unsafe_code)]`. The `buffa::table` module the code calls may change in any release, so regenerate the code whenever you update `buffa`; a mismatch is a compile error. + +`compute_size`, decoding from a contiguous buffer, and encoding into a `BufMut` are compiled in `buffa`, at the `opt-level` `buffa` is built with. A build that sets `opt-level = "z"` for everything can spend a little size to recover speed with `[profile.release.package.buffa] opt-level = 3`. Encoding into a sink that is not a `BufMut`, such as `Rope`, and the generic wrappers around decoding are compiled in your crate. + ### Error handling Encoding has exactly one failure condition: the protobuf specification caps diff --git a/protoc-gen-buffa/src/main.rs b/protoc-gen-buffa/src/main.rs index 94c5b5d0..cf62fbbf 100644 --- a/protoc-gen-buffa/src/main.rs +++ b/protoc-gen-buffa/src/main.rs @@ -21,7 +21,7 @@ use buffa_codegen::generated::compiler::code_generator_response::File as CodeGen use buffa_codegen::generated::compiler::CodeGeneratorResponse; use buffa_codegen::generated::descriptor::Edition; -use buffa_codegen::{CodeGenConfig, EnumTypeOverride, FeatureOverride}; +use buffa_codegen::{CodeGenConfig, CodecStrategy, EnumTypeOverride, FeatureOverride}; const HELP: &str = "\ protoc-gen-buffa — protoc plugin for generating Rust code with buffa. @@ -347,6 +347,25 @@ fn parse_config(params: &str) -> Result { .unboxed_oneof_fields .push(normalize_unbox_oneof_path(value.trim())?); } + // `codec_strategy=table` generates the binary `Message` impl of + // every message from a static table and shared interpreters + // (default `unrolled`). Path-scoped rules use the repeatable + // `codec_strategy_in==`, whatever the option + // order; the last matching rule wins. + "codec_strategy" => codegen.codec_strategy = parse_codec_strategy(value)?, + "codec_strategy_in" => { + let (path, strategy) = value.rsplit_once('=').ok_or_else(|| { + format!( + "invalid codec_strategy_in format '{value}', expected \ + 'codec_strategy_in==' \ + (e.g. 'codec_strategy_in=.my.pkg.Msg=unrolled')" + ) + })?; + codegen.codec_strategy_in.push(( + normalize_proto_path(path.trim(), "codec_strategy_in")?, + parse_codec_strategy(strategy)?, + )); + } // `type_name_prefix=Rpc` prepends a prefix to every generated // message/enum type name (and their view types). The value is // passed through verbatim; buffa-codegen rejects anything that @@ -500,6 +519,16 @@ fn parse_config(params: &str) -> Result { Ok(PluginConfig { codegen }) } +fn parse_codec_strategy(value: &str) -> Result { + match value.trim() { + "unrolled" => Ok(CodecStrategy::Unrolled), + "table" => Ok(CodecStrategy::Table), + other => Err(format!( + "invalid codec strategy '{other}', expected 'unrolled' or 'table'" + )), + } +} + fn parse_bool(key: &str, value: &str) -> Result { match value.trim() { "true" => Ok(true), @@ -796,6 +825,42 @@ mod tests { } } + #[test] + fn codec_strategy_sets_the_global_default_and_rules_are_repeatable() { + let config = parse_config( + "codec_strategy=table,codec_strategy_in=my.pkg.Msg=unrolled,\ + codec_strategy_in= .my.pkg.Other. =table", + ) + .unwrap(); + assert_eq!(config.codegen.codec_strategy, CodecStrategy::Table); + assert_eq!( + config.codegen.codec_strategy_in, + vec![ + (".my.pkg.Msg".to_string(), CodecStrategy::Unrolled), + (".my.pkg.Other".to_string(), CodecStrategy::Table), + ] + ); + assert_eq!( + parse_config("").unwrap().codegen.codec_strategy, + CodecStrategy::Unrolled + ); + } + + #[test] + fn codec_strategy_rejects_unknown_strategies_and_malformed_rules() { + let err = parse_err("codec_strategy=fast"); + assert!(err.contains("invalid codec strategy 'fast'"), "{err}"); + let err = parse_err("codec_strategy_in=.my.pkg.Msg"); + assert!( + err.contains("codec_strategy_in=="), + "{err}" + ); + let err = parse_err("codec_strategy_in=.my.pkg.Msg="); + assert!(err.contains("invalid codec strategy ''"), "{err}"); + let err = parse_err("codec_strategy_in==table"); + assert!(err.contains("non-empty proto path"), "{err}"); + } + #[test] fn unbox_oneof_rejects_non_boolean_values() { // A path or a near-miss boolean is an error, never a silent no-op. From 89015eaf8fccd93467976ea819e62e182a5e07d3 Mon Sep 17 00:00:00 2001 From: Iain McGinniss <309153+iainmcgin@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:18:37 -0700 Subject: [PATCH 2/5] changelog: link the table codec pull request --- .changes/unreleased/added-20260923-codec-strategy-table.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/added-20260923-codec-strategy-table.yaml b/.changes/unreleased/added-20260923-codec-strategy-table.yaml index d2909f92..8f240a69 100644 --- a/.changes/unreleased/added-20260923-codec-strategy-table.yaml +++ b/.changes/unreleased/added-20260923-codec-strategy-table.yaml @@ -1,4 +1,4 @@ kind: Added body: |- - **Table-driven message codec** (#NNN, refs #463). `buffa_build::Config::codec_strategy(CodecStrategy::Table)` (plugin option `codec_strategy=table`, `CodeGenConfig::codec_strategy`) generates each message's binary `Message` implementation from a static table and interpreters that every message shares, instead of code specialised to the message's fields. It roughly halves the compiled size of a large schema at `opt-level = "z"`, and messages made of many small fields are slower. `CodecStrategy::Unrolled` stays the default. `codec_strategy_in(strategy, &[paths])` (plugin option `codec_strategy_in==`, repeatable) chooses the strategy for matching messages and the messages nested in them, and the last matching rule wins. The wire format does not change. A message that has a `oneof`, `map`, or group field, or holds a message that is not a table, stays unrolled, and `CodeGenWarning::TableCodecFallbackSummary` counts those; a rule that names such a message by its exact path is an error. The generated code needs Rust 1.77 or later, which `buffa-build` checks, and compiles in a crate with `#![forbid(unsafe_code)]`; see the guide's "Smaller generated code" section for the rest. + **Table-driven message codec** (#469, refs #463). `buffa_build::Config::codec_strategy(CodecStrategy::Table)` (plugin option `codec_strategy=table`, `CodeGenConfig::codec_strategy`) generates each message's binary `Message` implementation from a static table and interpreters that every message shares, instead of code specialised to the message's fields. It roughly halves the compiled size of a large schema at `opt-level = "z"`, and messages made of many small fields are slower. `CodecStrategy::Unrolled` stays the default. `codec_strategy_in(strategy, &[paths])` (plugin option `codec_strategy_in==`, repeatable) chooses the strategy for matching messages and the messages nested in them, and the last matching rule wins. The wire format does not change. A message that has a `oneof`, `map`, or group field, or holds a message that is not a table, stays unrolled, and `CodeGenWarning::TableCodecFallbackSummary` counts those; a rule that names such a message by its exact path is an error. The generated code needs Rust 1.77 or later, which `buffa-build` checks, and compiles in a crate with `#![forbid(unsafe_code)]`; see the guide's "Smaller generated code" section for the rest. time: 2026-09-23T03:10:00+00:00 From 71c8f2341760b94047bd45b63ef496edba6263de Mon Sep 17 00:00:00 2001 From: Iain McGinniss <309153+iainmcgin@users.noreply.github.com> Date: Wed, 23 Sep 2026 12:39:50 -0700 Subject: [PATCH 3/5] table: emit the table ABI from the generator, not from the runtime Generated tables passed `::buffa::table::ABI` to `Table::new`, which compares its argument with the same constant, so the check could never fail. The generator now emits its own number, and a test keeps that number equal to the runtime's. --- buffa-codegen/src/table_codec.rs | 8 +++++++- buffa-codegen/src/tests/table_codec.rs | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/buffa-codegen/src/table_codec.rs b/buffa-codegen/src/table_codec.rs index 6799f794..dd45c29d 100644 --- a/buffa-codegen/src/table_codec.rs +++ b/buffa-codegen/src/table_codec.rs @@ -36,6 +36,11 @@ fn table_path(type_path: &str) -> Result { Ok(rust_path_to_tokens(&format!("{head}__BUFFA_TABLE_{last}"))) } +/// The table ABI this generator emits, which `buffa::table::ABI` must equal +/// for the generated tables to build. Bump both when the meaning of an existing +/// table entry changes. +pub(crate) const TABLE_ABI: u32 = 1; + /// The static table and `impl Message` of a message the plan selected. /// /// # Errors @@ -74,6 +79,7 @@ pub(crate) fn generate_table_impl( } let dense = dense_lookup(&fields); + let abi = proc_macro2::Literal::u32_unsuffixed(TABLE_ABI); let unknown = if ctx.preserve_unknown_fields(proto_fqn) { quote! { __buffa_unknown_fields } @@ -86,7 +92,7 @@ pub(crate) fn generate_table_impl( #[allow(non_upper_case_globals)] pub(crate) static #table: ::buffa::table::Table<#name> = ::buffa::__table!( #name, - abi = ::buffa::table::ABI, + abi = #abi, entries = [#(#entries),*], dense = &[#(#dense),*], aux = [#(#aux),*], diff --git a/buffa-codegen/src/tests/table_codec.rs b/buffa-codegen/src/tests/table_codec.rs index 17f06167..2d06857e 100644 --- a/buffa-codegen/src/tests/table_codec.rs +++ b/buffa-codegen/src/tests/table_codec.rs @@ -577,3 +577,18 @@ fn a_child_the_user_did_not_choose_does_not_hide_one_that_cannot_use_the_table() let text = table_warnings(&warnings)[0].to_string(); assert!(text.contains(".t.Both"), "{text}"); } + +#[test] +fn the_generated_abi_is_a_literal_that_matches_the_runtime() { + // The runtime refuses a table generated for another ABI, so the generator + // must emit its own number: passing `buffa::table::ABI` would compare the + // runtime's constant with itself. + assert_eq!(crate::table_codec::TABLE_ABI, buffa::table::ABI); + let (code, _) = run(&table_config(CodecStrategy::Table)).unwrap(); + let code = squashed(&code); + assert!( + code.contains(&format!("abi={},", crate::table_codec::TABLE_ABI)), + "the table must carry the literal ABI: {code}" + ); + assert!(!code.contains("abi=::buffa::table::ABI"), "{code}"); +} From f00ac7ecae568fbd80034b56b9365a7f8b0a2876 Mon Sep 17 00:00:00 2001 From: Iain McGinniss <309153+iainmcgin@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:02:51 -0700 Subject: [PATCH 4/5] codegen: drop a test assertion on the ABI path the generator no longer emits --- buffa-codegen/src/tests/table_codec.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/buffa-codegen/src/tests/table_codec.rs b/buffa-codegen/src/tests/table_codec.rs index 2d06857e..2366044e 100644 --- a/buffa-codegen/src/tests/table_codec.rs +++ b/buffa-codegen/src/tests/table_codec.rs @@ -167,7 +167,6 @@ fn the_global_setting_gives_a_table_to_every_message_that_can_use_one() { fn a_table_message_forwards_message_to_the_shared_interpreters() { let (code, _) = run(&table_config(CodecStrategy::Table)).unwrap(); assert!(code.contains("::buffa::__table!")); - assert!(code.contains("::buffa::table::ABI")); assert!(code.contains("__BUFFA_TABLE_Plain.compute_size(self, cache)")); assert!(code.contains("__BUFFA_TABLE_Plain.merge_field(self, tag, buf, ctx)")); // None of the per-field code an unrolled impl has: the size and write From d19f2bea5dac66129969e1b6c985f704351c0a39 Mon Sep 17 00:00:00 2001 From: Iain McGinniss <309153+iainmcgin@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:40:43 -0700 Subject: [PATCH 5/5] ci: pin the Rust 1.77 toolchain action to a commit The other toolchain steps were pinned on main after this step was written; use the same commit as the 1.75 step above it. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91f16c97..37a9ae54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,7 +225,7 @@ jobs: # The table codec needs `offset_of!`, stable in 1.77; the workspace # check above compiles it out at 1.75. This is the oldest compiler that # builds the table interpreters and the generated table code. - - uses: dtolnay/rust-toolchain@master + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master (sha-pinned) with: toolchain: '1.77'