docs: Protobuf schema docs shard from protoc source info (closes #9824)

BUILD.bazel

@@ -75,6 +75,7 @@ load("//quality:json_schema.bzl", "json_schema", "json_schemas")
 # gazelle:exclude tools/docs/fortran/fixture
 # gazelle:exclude tools/docs/perl/fixture
 # gazelle:exclude tools/docs/php/fixture
+# gazelle:exclude tools/docs/protobuf/fixture
 # gazelle:exclude tools/docs/go/godoc
 # gazelle:exclude cli/depcheck/testdata
 # gazelle:exclude go/tools/analysis

MODULE.bazel

@@ -1843,6 +1843,7 @@ crate.from_cargo(
         "//tools/docs/fortran/extract:Cargo.toml",
         "//tools/docs/perl/extract:Cargo.toml",
         "//tools/docs/cc/extract:Cargo.toml",
+        "//tools/docs/protobuf/extract:Cargo.toml",
         "//tools/docs/python/extract:Cargo.toml",
         "//tools/docs/powershell/extract:Cargo.toml",
         "//tools/docs/d/extract:Cargo.toml",

cargo-bazel-lock.json

@@ -1,5 +1,5 @@
 {
-  "checksum": "1272c096f994ec07a7692f0815b2a336e089d3ca7ceff1ad34acf906ac7bb17a",
+  "checksum": "62004244cee28568ddcc1f0706af0e8e1331315a1bae368ef7827c3391d1ea25",
   "conditions": {
     "aarch64-apple-darwin": [
       "aarch64-apple-darwin"
@@ -18156,6 +18156,27 @@
       ],
       "version": "1.11.0"
     },
+    "protobuf_extract 0.0.0": {
+      "common_attrs": {
+        "edition": "2024"
+      },
+      "library_target_name": "dx_docs_protobuf_extract",
+      "license": "Apache-2.0",
+      "license_ids": [
+        "Apache-2.0"
+      ],
+      "name": "protobuf_extract",
+      "repository": null,
+      "targets": [
+        {
+          "Library": {
+            "crate_name": "dx_docs_protobuf_extract",
+            "crate_root": "src/lib.rs"
+          }
+        }
+      ],
+      "version": "0.0.0"
+    },
     "pulldown-cmark 0.13.5": {
       "build_script_attrs": {},
       "common_attrs": {
@@ -28822,6 +28843,7 @@
     "php_extract 0.0.0": "tools/docs/php/php_extract",
     "powershell_extract 0.0.0": "tools/docs/powershell/powershell_extract",
     "process_probe 0.0.0": "tools/testing/process_probe",
+    "protobuf_extract 0.0.0": "tools/docs/protobuf/protobuf_extract",
     "python_docs_tests 0.0.0": "tools/docs/python_docs_tests",
     "python_extract 0.0.0": "tools/docs/python/python_extract",
     "qt_rust_bridge 0.0.0": "qt/qt_rust_bridge",

docs/cli/commands/docs.md

@@ -105,6 +105,7 @@ tools/dx docs //lib:site
 | Dart | `@rules_dx//tools/docs/dart:defs.bzl` | `dart_doc_shard` | `lib`, `package` |
 | C# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `csharp_doc_shard` | `library`, `package` |
 | F# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `fsharp_doc_shard` | `library`, `package` |
+| Protobuf | `@rules_dx//tools/docs/protobuf:defs.bzl` | `protobuf_doc_shard` | `proto`, `package` |
 
 Rust, Go, Python, C, C++, CUDA, Fortran, Java, Kotlin, Scala, D, Dart, C#, F# and Perl libraries in scope that have
 no shard get one automatically. A C# library needs `nullable = "disable"` and
@@ -115,4 +116,12 @@ C, C++ and CUDA shards document the public `hdrs` of a `cc_library` or
 Doxygen publishes no Linux arm64 or Windows arm64 build, so those hosts get no
 C, C++ or CUDA API docs.
 
+A `proto_library` in scope that has no shard gets one automatically. Protobuf
+shards come from the protoc of the registered proto toolchain run with
+`--include_source_info`, so no tool beyond protoc is pinned. A message becomes
+a class, a field a field, an enum and its values an enum and its variants, a
+service an interface and an rpc a method. The comment above an item, or after
+it on the same line, is its doc text. A `map<K, V>` field shows as a map and
+its generated entry message is left out. Streaming rpcs say `stream`.
+
 A source error fails the build and writes no pages. Bazel's exit code is kept.

docs/support-matrix.md

@@ -103,7 +103,7 @@ Each cell names the tools `dx lint`, `dx format`, or `dx typecheck` runs, and th
 | Perl | `perlcritic` (not on `windows_arm64`: rules_perl 1.5.0 ships no Windows ARM64 perl build), `typos` | `perltidy` (not on `windows_arm64`: rules_perl 1.5.0 ships no Windows ARM64 perl build) | n/a | `pod (perl)` | `DxLcov` | `perlcritic ProhibitUnusedVariables` |
 | PHP | `phpcs` (not on `windows_arm64`: static-php-cli and windows.php.net ship no Windows arm64 PHP 8.5 build), `typos` | `prettier` | `phpstan` (not on `windows_arm64`: static-php-cli and windows.php.net ship no Windows arm64 PHP 8.5 build) | `phpdocumentor` | `pcov (Linux only)` | `phpcs Generic.CodeAnalysis.UnusedFunctionParameter` |
 | PowerShell | `psscriptanalyzer`, `typos` | `psscriptanalyzer` | n/a | `pwsh` | `pester` | `psscriptanalyzer PSUseDeclaredVarsMoreThanAssignments` |
-| Protobuf | `buf`, `typos` | `buf` | `buf` | no | n/a | none |
+| Protobuf | `buf`, `typos` | `buf` | `buf` | `protoc` | n/a | none |
 | Python | `pydoclint`, `ruff`, `typos` | `ruff` | `ty` | `griffe` | `coverage.py` | `ruff F401, F841` |
 | Ruby | `rubocop` (not on `windows_arm64`: RubyInstaller arm64 Ruby is not pinned or qualified), `typos` | `rubocop` (not on `windows_arm64`: RubyInstaller arm64 Ruby is not pinned or qualified) | `sorbet` (not on `windows_x86_64`: Sorbet ships no Windows build; `windows_arm64`: Sorbet ships no Windows build) | `yard` | `simplecov` | `rubocop Lint/UselessAssignment` |
 | Rust | `clippy`, `typos` | `rustfmt` | `rustc` | `rustdoc` | `llvm-cov` | `rustc dead_code` |

families/generation/README.md

@@ -6,3 +6,11 @@ Avro, Cap'n Proto, FlatBuffers, Thrift and OpenAPI client tests and fixtures. Th
 `rules_dx`, because its quality tools, Gazelle extensions and CLI use them.
 A workspace that only uses the code generation rules needs nothing more than
 `rules_dx`.
+
+## Protobuf docs
+
+`protobuf_doc_shard` in `@rules_dx//tools/docs/protobuf:defs.bzl` documents one
+`proto_library`. It runs the protoc of the registered proto toolchain with
+`--include_source_info` and reads the comments above each message, field, enum,
+enum value, service and rpc from the descriptor set. Avro schemas are not
+covered; their `doc` fields are tracked in #9936.

modules/rust.bzl

@@ -66,6 +66,7 @@ RUST_CRATE_MANIFESTS = [
     "//tools/docs/fortran/extract:Cargo.toml",
     "//tools/docs/perl/extract:Cargo.toml",
     "//tools/docs/cc/extract:Cargo.toml",
+    "//tools/docs/protobuf/extract:Cargo.toml",
     "//tools/docs/python/extract:Cargo.toml",
     "//tools/docs/powershell/extract:Cargo.toml",
     "//tools/docs/d/extract:Cargo.toml",

quality/testdata/BUILD.bazel

@@ -2003,6 +2003,11 @@ real_docs_subject(
     target = "//tools/docs/dart/flutter_fixture:badge",
 )
 
+real_docs_subject(
+    name = "docs_protobuf_library_subject",
+    target = "//tools/docs/protobuf/fixture:shop_proto",
+)
+
 real_docs_subject(
     name = "docs_ruby_library_subject",
     target = "//tools/docs/ruby/fixture:geometry",
@@ -2057,6 +2062,7 @@ real_docs_tests(
         ":docs_php_library_subject",
         ":docs_plain_subject",
         ":docs_powershell_library_subject",
+        ":docs_protobuf_library_subject",
         ":docs_python_library_subject",
         ":docs_ruby_library_subject",
         ":docs_rust_library_subject",

quality/testdata/real_aspect_tests.bzl

@@ -740,6 +740,17 @@ aspect_field field_count=5
 aspect_field has_subject=True
 aspect_field subject_label=//quality/testdata:docs_powershell_library_subject
 aspect_field transitive_count=0
+subject //quality/testdata:docs_protobuf_library_subject
+field docs_shard_count=1
+field docs_shards=tools/docs/protobuf/fixture/_dx_docs/shop_proto.ir.pb
+field dx_docs_sites=(none)
+field dx_results=shop_proto.ir.pb
+field label=//tools/docs/protobuf/fixture:shop_proto
+aspect_field aspect_seen=True
+aspect_field field_count=5
+aspect_field has_subject=True
+aspect_field subject_label=//quality/testdata:docs_protobuf_library_subject
+aspect_field transitive_count=0
 subject //quality/testdata:docs_python_library_subject
 field docs_shard_count=1
 field docs_shards=tools/docs/python/fixture/_dx_docs/mylib.ir.pb

tools/docs/aspect.bzl

@@ -5,6 +5,7 @@ load("@rules_julia//julia:julia_info.bzl", _JuliaInfo = "JuliaInfo")
 load("@rules_powershell//powershell:defs.bzl", _PwshInfo = "PwshInfo")
 load("@rules_py//py:defs.bzl", _PyInfo = "PyInfo")
 load("@rules_perl//perl:defs.bzl", _PerlInfo = "PerlInfo")
+load("@protobuf//bazel/common:proto_info.bzl", _ProtoInfo = "ProtoInfo")
 load("@rules_ruby//ruby/private:providers.bzl", _RubyFilesInfo = "RubyFilesInfo")
 load("@rules_dart//dart:providers.bzl", _DartInfo = "DartInfo")
 load("@rules_d//d/private:providers.bzl", _DInfo = "DInfo")
@@ -17,6 +18,7 @@ load("@rules_java//java/common:java_info.bzl", "JavaInfo")
 load("@rules_rust//rust:defs.bzl", _rust_common = "rust_common")
 load("@aspect_rules_js//js:libs.bzl", "js_binary_lib")
 load("//ocaml/rules:providers.bzl", _OCamlCmtInfo = "OCamlCmtInfo")
+load("//libs/exec_os:defs.bzl", "ExecOsInfo")
 load("//libs/starlark:resources.bzl", "dx_heavy_resources")
 load("//tools/docs/ir:shard.bzl", "DocsShardInfo", "repo_relative")
 load("//tools/docs/cc:defs.bzl", "cc_shard_action", "doxygen_available")
@@ -31,6 +33,7 @@ load("//tools/docs/ocaml:defs.bzl", "ODOC_CONFIG", "ocaml_odoc_available", "ocam
 load("//tools/docs/jvm:shard.bzl", "JAVA_TOOLCHAIN", "JVM_LANGUAGES", "KOTLIN_TOOLCHAIN", "jvm_shard_action")
 load("//tools/docs/perl:defs.bzl", "perl_shard_action")
 load("//tools/docs/powershell:defs.bzl", "powershell_shard_action")
+load("//tools/docs/protobuf:defs.bzl", "PROTO_TOOLCHAIN", "protobuf_shard_action")
 load("//tools/docs/python:shard.bzl", "python_package_name_ok", "python_shard_action")
 load("//tools/docs/typescript:shard.bzl", "TYPEDOC_JSON_SCHEMA", "typedoc_arguments", "typescript_shard_action")
 load("//tools/docs/php:defs.bzl", "php_shard_action")
@@ -208,6 +211,14 @@ def _php_docs_shard(target, ctx):
     if not own or ctx.toolchains[_PHP_EXEC_TOOLCHAIN] == None:
         return None
     return php_shard_action(ctx, own, ctx.label.name, "_dx_docs/" + ctx.label.name, ctx.file._php_phar, ctx.file._php_driver, ctx.executable._php_extract).shard
+def _protobuf_docs_shard(target, ctx):
+    if _ProtoInfo not in target or DocsShardInfo in target:
+        return None
+    proto_info = target[_ProtoInfo]
+    toolchain = ctx.toolchains[PROTO_TOOLCHAIN]
+    if not proto_info.direct_sources or toolchain == None:
+        return None
+    return protobuf_shard_action(ctx, proto_info, toolchain.proto.proto_compiler, ctx.attr._exec_os[ExecOsInfo].windows, ctx.label.name, "_dx_docs/" + ctx.label.name, ctx.executable._protobuf_extract).shard
 
 _TS_ENTRY_EXTENSIONS = ["ts", "tsx", "mts", "cts"]
 _TS_TEST_MARKERS = [".test.", ".spec.", "_test."]
@@ -405,6 +416,9 @@ def _real_docs_aspect_impl(target, ctx):
     php_shard = _php_docs_shard(target, ctx)
     if php_shard:
         parts.append({"dx_results": [php_shard], "dx_docs_shards": [php_shard]})
+    proto_shard = _protobuf_docs_shard(target, ctx)
+    if proto_shard:
+        parts.append({"dx_results": [proto_shard], "dx_docs_shards": [proto_shard]})
     ts_shard = _typescript_docs_shard(target, ctx)
     if ts_shard:
         parts.append({"dx_results": [ts_shard], "dx_docs_shards": [ts_shard]})
@@ -424,7 +438,7 @@ def _real_docs_aspect_impl(target, ctx):
 
 real_docs_aspect = aspect(
     implementation = _real_docs_aspect_impl,
-    doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Dart, C#, F#, PowerShell, Perl, Ruby, Julia, OCaml, Fortran, PHP, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.",
+    doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Dart, C#, F#, PowerShell, Perl, Protobuf, Ruby, Julia, OCaml, Fortran, PHP, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The Protobuf branch runs the protoc of the proto toolchain with --include_source_info over a proto_library's own .proto srcs and the DocIr extractor (//tools/docs/protobuf/extract:protobuf_extract). The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.",
     attrs = {
         "_astro_meta": _component_meta_attr("astro"),
         "_cc_extract": attr.label(
@@ -583,6 +597,16 @@ real_docs_aspect = aspect(
             default = Label("//tools/docs/ocaml/extract:ocaml_extract"),
             executable = True,
         ),
+        "_exec_os": attr.label(
+            default = Label("//libs/exec_os"),
+            cfg = "exec",
+            providers = [ExecOsInfo],
+        ),
+        "_protobuf_extract": attr.label(
+            cfg = "exec",
+            default = Label("//tools/docs/protobuf/extract:protobuf_extract"),
+            executable = True,
+        ),
         "_ruby_driver": attr.label(
             allow_single_file = True,
             default = Label("//tools/docs/ruby:yard_dump.rb"),
@@ -617,6 +641,7 @@ real_docs_aspect = aspect(
         config_common.toolchain_type(KOTLIN_TOOLCHAIN, mandatory = False),
         config_common.toolchain_type(_PERL_TOOLCHAIN, mandatory = False),
         config_common.toolchain_type(_PHP_EXEC_TOOLCHAIN, mandatory = False),
+        config_common.toolchain_type(PROTO_TOOLCHAIN, mandatory = False),
         config_common.toolchain_type(_RUBY_TOOLCHAIN, mandatory = False),
         config_common.toolchain_type(_RUST_DOCS_TOOLCHAIN, mandatory = False),
         config_common.toolchain_type(_CC_DOCS_TOOLCHAIN, mandatory = False),

tools/docs/protobuf/BUILD.bazel

@@ -0,0 +1,32 @@
+load("//quality:fixtures.bzl", "real_source_target")
+load("//rust/rules:defs.bzl", "rust_test")
+load("//tools/visibility:visibility.bzl", "VIS_TOOLS", "VIS_TOOLS_PUBLIC")
+
+package(default_visibility = VIS_TOOLS_PUBLIC)
+
+exports_files(
+    ["defs.bzl"],
+    visibility = VIS_TOOLS,
+)
+
+rust_test(
+    name = "shard_test",
+    srcs = ["shard_test.rs"],
+    crate_name = "shard_test",
+    crate_root = "shard_test.rs",
+    data = ["//tools/docs/protobuf/fixture:shop_docs"],
+    env = {"DX_FIXTURE_SHARD": "$(rootpath //tools/docs/protobuf/fixture:shop_docs)"},
+    deps = [
+        "//tools/docs/ir/ir:documentation_ir",
+        "//tools/testing:dx_testing",
+    ],
+)
+
+real_source_target(
+    name = "corpus_starlark",
+    starlark_srcs = [
+        "BUILD.bazel",
+        "defs.bzl",
+    ],
+    tags = ["corpus"],
+)