diff --git a/BUILD.bazel b/BUILD.bazel index e41ee9c762..0e53494108 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -75,6 +75,7 @@ load("//quality:json_schema.bzl", "json_schema", "json_schemas") # gazelle:exclude tools/docs/fortran/fixture # gazelle:exclude tools/docs/perl/fixture # gazelle:exclude tools/docs/php/fixture +# gazelle:exclude tools/docs/protobuf/fixture # gazelle:exclude tools/docs/go/godoc # gazelle:exclude cli/depcheck/testdata # gazelle:exclude go/tools/analysis diff --git a/MODULE.bazel b/MODULE.bazel index 55ee381f2d..504aff243a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1843,6 +1843,7 @@ crate.from_cargo( "//tools/docs/fortran/extract:Cargo.toml", "//tools/docs/perl/extract:Cargo.toml", "//tools/docs/cc/extract:Cargo.toml", + "//tools/docs/protobuf/extract:Cargo.toml", "//tools/docs/python/extract:Cargo.toml", "//tools/docs/powershell/extract:Cargo.toml", "//tools/docs/d/extract:Cargo.toml", diff --git a/cargo-bazel-lock.json b/cargo-bazel-lock.json index 54178abfc4..48c9e993ee 100644 --- a/cargo-bazel-lock.json +++ b/cargo-bazel-lock.json @@ -1,5 +1,5 @@ { - "checksum": "1272c096f994ec07a7692f0815b2a336e089d3ca7ceff1ad34acf906ac7bb17a", + "checksum": "62004244cee28568ddcc1f0706af0e8e1331315a1bae368ef7827c3391d1ea25", "conditions": { "aarch64-apple-darwin": [ "aarch64-apple-darwin" @@ -18156,6 +18156,27 @@ ], "version": "1.11.0" }, + "protobuf_extract 0.0.0": { + "common_attrs": { + "edition": "2024" + }, + "library_target_name": "dx_docs_protobuf_extract", + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "name": "protobuf_extract", + "repository": null, + "targets": [ + { + "Library": { + "crate_name": "dx_docs_protobuf_extract", + "crate_root": "src/lib.rs" + } + } + ], + "version": "0.0.0" + }, "pulldown-cmark 0.13.5": { "build_script_attrs": {}, "common_attrs": { @@ -28822,6 +28843,7 @@ "php_extract 0.0.0": "tools/docs/php/php_extract", "powershell_extract 0.0.0": "tools/docs/powershell/powershell_extract", "process_probe 0.0.0": "tools/testing/process_probe", + "protobuf_extract 0.0.0": "tools/docs/protobuf/protobuf_extract", "python_docs_tests 0.0.0": "tools/docs/python_docs_tests", "python_extract 0.0.0": "tools/docs/python/python_extract", "qt_rust_bridge 0.0.0": "qt/qt_rust_bridge", diff --git a/cc/context/Cargo.lock b/cc/context/Cargo.lock index 0653fb3681..a9d541c040 100644 --- a/cc/context/Cargo.lock +++ b/cc/context/Cargo.lock @@ -3999,6 +3999,10 @@ dependencies = [ "unarray", ] +[[package]] +name = "protobuf_extract" +version = "0.0.0" + [[package]] name = "pulldown-cmark" version = "0.13.5" diff --git a/docs/cli/commands/docs.md b/docs/cli/commands/docs.md index 85405583d7..d1aeae5cd3 100644 --- a/docs/cli/commands/docs.md +++ b/docs/cli/commands/docs.md @@ -105,6 +105,7 @@ tools/dx docs //lib:site | Dart | `@rules_dx//tools/docs/dart:defs.bzl` | `dart_doc_shard` | `lib`, `package` | | C# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `csharp_doc_shard` | `library`, `package` | | F# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `fsharp_doc_shard` | `library`, `package` | +| Protobuf | `@rules_dx//tools/docs/protobuf:defs.bzl` | `protobuf_doc_shard` | `proto`, `package` | Rust, Go, Python, C, C++, CUDA, Fortran, Java, Kotlin, Scala, D, Dart, C#, F# and Perl libraries in scope that have no shard get one automatically. A C# library needs `nullable = "disable"` and @@ -115,4 +116,12 @@ C, C++ and CUDA shards document the public `hdrs` of a `cc_library` or Doxygen publishes no Linux arm64 or Windows arm64 build, so those hosts get no C, C++ or CUDA API docs. +A `proto_library` in scope that has no shard gets one automatically. Protobuf +shards come from the protoc of the registered proto toolchain run with +`--include_source_info`, so no tool beyond protoc is pinned. A message becomes +a class, a field a field, an enum and its values an enum and its variants, a +service an interface and an rpc a method. The comment above an item, or after +it on the same line, is its doc text. A `map` field shows as a map and +its generated entry message is left out. Streaming rpcs say `stream`. + A source error fails the build and writes no pages. Bazel's exit code is kept. diff --git a/docs/support-matrix.md b/docs/support-matrix.md index 15b5dd912c..598506e967 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -103,7 +103,7 @@ Each cell names the tools `dx lint`, `dx format`, or `dx typecheck` runs, and th | Perl | `perlcritic` (not on `windows_arm64`: rules_perl 1.5.0 ships no Windows ARM64 perl build), `typos` | `perltidy` (not on `windows_arm64`: rules_perl 1.5.0 ships no Windows ARM64 perl build) | n/a | `pod (perl)` | `DxLcov` | `perlcritic ProhibitUnusedVariables` | | PHP | `phpcs` (not on `windows_arm64`: static-php-cli and windows.php.net ship no Windows arm64 PHP 8.5 build), `typos` | `prettier` | `phpstan` (not on `windows_arm64`: static-php-cli and windows.php.net ship no Windows arm64 PHP 8.5 build) | `phpdocumentor` | `pcov (Linux only)` | `phpcs Generic.CodeAnalysis.UnusedFunctionParameter` | | PowerShell | `psscriptanalyzer`, `typos` | `psscriptanalyzer` | n/a | `pwsh` | `pester` | `psscriptanalyzer PSUseDeclaredVarsMoreThanAssignments` | -| Protobuf | `buf`, `typos` | `buf` | `buf` | no | n/a | none | +| Protobuf | `buf`, `typos` | `buf` | `buf` | `protoc` | n/a | none | | Python | `pydoclint`, `ruff`, `typos` | `ruff` | `ty` | `griffe` | `coverage.py` | `ruff F401, F841` | | Ruby | `rubocop` (not on `windows_arm64`: RubyInstaller arm64 Ruby is not pinned or qualified), `typos` | `rubocop` (not on `windows_arm64`: RubyInstaller arm64 Ruby is not pinned or qualified) | `sorbet` (not on `windows_x86_64`: Sorbet ships no Windows build; `windows_arm64`: Sorbet ships no Windows build) | `yard` | `simplecov` | `rubocop Lint/UselessAssignment` | | Rust | `clippy`, `typos` | `rustfmt` | `rustc` | `rustdoc` | `llvm-cov` | `rustc dead_code` | diff --git a/families/generation/README.md b/families/generation/README.md index 435200a2c5..d662e448f6 100644 --- a/families/generation/README.md +++ b/families/generation/README.md @@ -6,3 +6,11 @@ Avro, Cap'n Proto, FlatBuffers, Thrift and OpenAPI client tests and fixtures. Th `rules_dx`, because its quality tools, Gazelle extensions and CLI use them. A workspace that only uses the code generation rules needs nothing more than `rules_dx`. + +## Protobuf docs + +`protobuf_doc_shard` in `@rules_dx//tools/docs/protobuf:defs.bzl` documents one +`proto_library`. It runs the protoc of the registered proto toolchain with +`--include_source_info` and reads the comments above each message, field, enum, +enum value, service and rpc from the descriptor set. Avro schemas are not +covered; their `doc` fields are tracked in #9936. diff --git a/modules/rust.bzl b/modules/rust.bzl index 87d18668af..bf187df9ee 100644 --- a/modules/rust.bzl +++ b/modules/rust.bzl @@ -66,6 +66,7 @@ RUST_CRATE_MANIFESTS = [ "//tools/docs/fortran/extract:Cargo.toml", "//tools/docs/perl/extract:Cargo.toml", "//tools/docs/cc/extract:Cargo.toml", + "//tools/docs/protobuf/extract:Cargo.toml", "//tools/docs/python/extract:Cargo.toml", "//tools/docs/powershell/extract:Cargo.toml", "//tools/docs/d/extract:Cargo.toml", diff --git a/quality/testdata/BUILD.bazel b/quality/testdata/BUILD.bazel index fb16f4fa5f..204041bf92 100644 --- a/quality/testdata/BUILD.bazel +++ b/quality/testdata/BUILD.bazel @@ -2003,6 +2003,11 @@ real_docs_subject( target = "//tools/docs/dart/flutter_fixture:badge", ) +real_docs_subject( + name = "docs_protobuf_library_subject", + target = "//tools/docs/protobuf/fixture:shop_proto", +) + real_docs_subject( name = "docs_ruby_library_subject", target = "//tools/docs/ruby/fixture:geometry", @@ -2057,6 +2062,7 @@ real_docs_tests( ":docs_php_library_subject", ":docs_plain_subject", ":docs_powershell_library_subject", + ":docs_protobuf_library_subject", ":docs_python_library_subject", ":docs_ruby_library_subject", ":docs_rust_library_subject", diff --git a/quality/testdata/real_aspect_tests.bzl b/quality/testdata/real_aspect_tests.bzl index 42c3b4d1fd..327e53ef0f 100644 --- a/quality/testdata/real_aspect_tests.bzl +++ b/quality/testdata/real_aspect_tests.bzl @@ -740,6 +740,17 @@ aspect_field field_count=5 aspect_field has_subject=True aspect_field subject_label=//quality/testdata:docs_powershell_library_subject aspect_field transitive_count=0 +subject //quality/testdata:docs_protobuf_library_subject +field docs_shard_count=1 +field docs_shards=tools/docs/protobuf/fixture/_dx_docs/shop_proto.ir.pb +field dx_docs_sites=(none) +field dx_results=shop_proto.ir.pb +field label=//tools/docs/protobuf/fixture:shop_proto +aspect_field aspect_seen=True +aspect_field field_count=5 +aspect_field has_subject=True +aspect_field subject_label=//quality/testdata:docs_protobuf_library_subject +aspect_field transitive_count=0 subject //quality/testdata:docs_python_library_subject field docs_shard_count=1 field docs_shards=tools/docs/python/fixture/_dx_docs/mylib.ir.pb diff --git a/tools/docs/aspect.bzl b/tools/docs/aspect.bzl index 71737ecfbf..be3fa9cc14 100644 --- a/tools/docs/aspect.bzl +++ b/tools/docs/aspect.bzl @@ -5,6 +5,7 @@ load("@rules_julia//julia:julia_info.bzl", _JuliaInfo = "JuliaInfo") load("@rules_powershell//powershell:defs.bzl", _PwshInfo = "PwshInfo") load("@rules_py//py:defs.bzl", _PyInfo = "PyInfo") load("@rules_perl//perl:defs.bzl", _PerlInfo = "PerlInfo") +load("@protobuf//bazel/common:proto_info.bzl", _ProtoInfo = "ProtoInfo") load("@rules_ruby//ruby/private:providers.bzl", _RubyFilesInfo = "RubyFilesInfo") load("@rules_dart//dart:providers.bzl", _DartInfo = "DartInfo") load("@rules_d//d/private:providers.bzl", _DInfo = "DInfo") @@ -17,6 +18,7 @@ load("@rules_java//java/common:java_info.bzl", "JavaInfo") load("@rules_rust//rust:defs.bzl", _rust_common = "rust_common") load("@aspect_rules_js//js:libs.bzl", "js_binary_lib") load("//ocaml/rules:providers.bzl", _OCamlCmtInfo = "OCamlCmtInfo") +load("//libs/exec_os:defs.bzl", "ExecOsInfo") load("//libs/starlark:resources.bzl", "dx_heavy_resources") load("//tools/docs/ir:shard.bzl", "DocsShardInfo", "repo_relative") load("//tools/docs/cc:defs.bzl", "cc_shard_action", "doxygen_available") @@ -31,6 +33,7 @@ load("//tools/docs/ocaml:defs.bzl", "ODOC_CONFIG", "ocaml_odoc_available", "ocam load("//tools/docs/jvm:shard.bzl", "JAVA_TOOLCHAIN", "JVM_LANGUAGES", "KOTLIN_TOOLCHAIN", "jvm_shard_action") load("//tools/docs/perl:defs.bzl", "perl_shard_action") load("//tools/docs/powershell:defs.bzl", "powershell_shard_action") +load("//tools/docs/protobuf:defs.bzl", "PROTO_TOOLCHAIN", "protobuf_shard_action") load("//tools/docs/python:shard.bzl", "python_package_name_ok", "python_shard_action") load("//tools/docs/typescript:shard.bzl", "TYPEDOC_JSON_SCHEMA", "typedoc_arguments", "typescript_shard_action") load("//tools/docs/php:defs.bzl", "php_shard_action") @@ -208,6 +211,14 @@ def _php_docs_shard(target, ctx): if not own or ctx.toolchains[_PHP_EXEC_TOOLCHAIN] == None: return None return php_shard_action(ctx, own, ctx.label.name, "_dx_docs/" + ctx.label.name, ctx.file._php_phar, ctx.file._php_driver, ctx.executable._php_extract).shard +def _protobuf_docs_shard(target, ctx): + if _ProtoInfo not in target or DocsShardInfo in target: + return None + proto_info = target[_ProtoInfo] + toolchain = ctx.toolchains[PROTO_TOOLCHAIN] + if not proto_info.direct_sources or toolchain == None: + return None + return protobuf_shard_action(ctx, proto_info, toolchain.proto.proto_compiler, ctx.attr._exec_os[ExecOsInfo].windows, ctx.label.name, "_dx_docs/" + ctx.label.name, ctx.executable._protobuf_extract).shard _TS_ENTRY_EXTENSIONS = ["ts", "tsx", "mts", "cts"] _TS_TEST_MARKERS = [".test.", ".spec.", "_test."] @@ -405,6 +416,9 @@ def _real_docs_aspect_impl(target, ctx): php_shard = _php_docs_shard(target, ctx) if php_shard: parts.append({"dx_results": [php_shard], "dx_docs_shards": [php_shard]}) + proto_shard = _protobuf_docs_shard(target, ctx) + if proto_shard: + parts.append({"dx_results": [proto_shard], "dx_docs_shards": [proto_shard]}) ts_shard = _typescript_docs_shard(target, ctx) if ts_shard: parts.append({"dx_results": [ts_shard], "dx_docs_shards": [ts_shard]}) @@ -424,7 +438,7 @@ def _real_docs_aspect_impl(target, ctx): real_docs_aspect = aspect( implementation = _real_docs_aspect_impl, - doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Dart, C#, F#, PowerShell, Perl, Ruby, Julia, OCaml, Fortran, PHP, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.", + doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Dart, C#, F#, PowerShell, Perl, Protobuf, Ruby, Julia, OCaml, Fortran, PHP, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The Protobuf branch runs the protoc of the proto toolchain with --include_source_info over a proto_library's own .proto srcs and the DocIr extractor (//tools/docs/protobuf/extract:protobuf_extract). The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.", attrs = { "_astro_meta": _component_meta_attr("astro"), "_cc_extract": attr.label( @@ -583,6 +597,16 @@ real_docs_aspect = aspect( default = Label("//tools/docs/ocaml/extract:ocaml_extract"), executable = True, ), + "_exec_os": attr.label( + default = Label("//libs/exec_os"), + cfg = "exec", + providers = [ExecOsInfo], + ), + "_protobuf_extract": attr.label( + cfg = "exec", + default = Label("//tools/docs/protobuf/extract:protobuf_extract"), + executable = True, + ), "_ruby_driver": attr.label( allow_single_file = True, default = Label("//tools/docs/ruby:yard_dump.rb"), @@ -617,6 +641,7 @@ real_docs_aspect = aspect( config_common.toolchain_type(KOTLIN_TOOLCHAIN, mandatory = False), config_common.toolchain_type(_PERL_TOOLCHAIN, mandatory = False), config_common.toolchain_type(_PHP_EXEC_TOOLCHAIN, mandatory = False), + config_common.toolchain_type(PROTO_TOOLCHAIN, mandatory = False), config_common.toolchain_type(_RUBY_TOOLCHAIN, mandatory = False), config_common.toolchain_type(_RUST_DOCS_TOOLCHAIN, mandatory = False), config_common.toolchain_type(_CC_DOCS_TOOLCHAIN, mandatory = False), diff --git a/tools/docs/protobuf/BUILD.bazel b/tools/docs/protobuf/BUILD.bazel new file mode 100644 index 0000000000..f87578406c --- /dev/null +++ b/tools/docs/protobuf/BUILD.bazel @@ -0,0 +1,32 @@ +load("//quality:fixtures.bzl", "real_source_target") +load("//rust/rules:defs.bzl", "rust_test") +load("//tools/visibility:visibility.bzl", "VIS_TOOLS", "VIS_TOOLS_PUBLIC") + +package(default_visibility = VIS_TOOLS_PUBLIC) + +exports_files( + ["defs.bzl"], + visibility = VIS_TOOLS, +) + +rust_test( + name = "shard_test", + srcs = ["shard_test.rs"], + crate_name = "shard_test", + crate_root = "shard_test.rs", + data = ["//tools/docs/protobuf/fixture:shop_docs"], + env = {"DX_FIXTURE_SHARD": "$(rootpath //tools/docs/protobuf/fixture:shop_docs)"}, + deps = [ + "//tools/docs/ir/ir:documentation_ir", + "//tools/testing:dx_testing", + ], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = [ + "BUILD.bazel", + "defs.bzl", + ], + tags = ["corpus"], +) diff --git a/tools/docs/protobuf/defs.bzl b/tools/docs/protobuf/defs.bzl new file mode 100644 index 0000000000..832d5920ba --- /dev/null +++ b/tools/docs/protobuf/defs.bzl @@ -0,0 +1,140 @@ +"""Protobuf schema reference shards read from protoc's descriptor set and its source info.""" + +load("@protobuf//bazel/common:proto_info.bzl", "ProtoInfo") +load("//libs/exec_os:defs.bzl", "ExecOsInfo") +load("//tools/docs/ir:shard.bzl", "DocsShardInfo", "repo_relative") + +PROTO_TOOLCHAIN = "@protobuf//bazel/private:proto_toolchain_type" + +def _root_relative(src, root): + path = src.path + if root and path.startswith(root + "/"): + path = path[len(root) + 1:] + return path + +def _path(file): + return file.path + +def protobuf_shard_action(ctx, proto_info, protoc, windows, package, stem, extract_tool): + """Runs protoc and the extractor for one proto_library. + + Args: + ctx: The rule or aspect context. + proto_info: The ProtoInfo of the documented proto_library. + protoc: The FilesToRunProvider of protoc, from the proto toolchain. + windows: Whether the execution platform is Windows, which joins paths with ';'. + package: Package segment of the shard's symbol ids. + stem: Output name prefix, relative to the target's package. + extract_tool: The DocIr extractor executable. + + Returns: + A struct with the DocIr `shard` and the `descriptor` set file. + """ + root = proto_info.proto_source_root + sources = proto_info.direct_sources + if not sources: + fail("protobuf_doc_shard " + str(ctx.label) + ": proto_library lists no .proto srcs") + + # The library's own descriptor set is left out: protoc drops the comments of a file it + # reads from a descriptor set, and the docs come from the comments. + own_set = proto_info.direct_descriptor_set + dep_sets = [f for f in proto_info.transitive_descriptor_sets.to_list() if f != own_set] + + descriptor = ctx.actions.declare_file(stem + ".descriptor.pb") + names = [_root_relative(src, root) for src in sources] + args = ctx.actions.args() + args.add("--proto_path=" + (root if root else ".")) + args.add_joined("--descriptor_set_in", dep_sets, join_with = ";" if windows else ":", map_each = _path, omit_if_empty = True) + args.add("--include_source_info") + args.add("--descriptor_set_out", descriptor) + args.add_all(sources) + ctx.actions.run( + executable = protoc, + arguments = [args], + inputs = depset(sources, transitive = [depset(dep_sets)]), + outputs = [descriptor], + mnemonic = "DxDocsProtoc", + toolchain = None, + progress_message = "Reading Protobuf docs for %{label}", + ) + + shard = ctx.actions.declare_file(stem + ".ir.pb") + extract = ctx.actions.args() + extract.add("--descriptor", descriptor) + extract.add("--package", package) + extract.add("--out", shard) + for name, src in zip(names, sources): + extract.add("--src", name + "=" + repo_relative(src.short_path)) + if not src.is_source: + extract.add("--generated", name) + ctx.actions.run( + executable = extract_tool, + arguments = [extract], + inputs = [descriptor], + outputs = [shard], + mnemonic = "DxDocsProtobufExtract", + toolchain = None, + progress_message = "Extracting Protobuf docs for %{label}", + ) + return struct(shard = shard, descriptor = descriptor) + +def _protobuf_doc_shard_impl(ctx): + toolchain = ctx.toolchains[PROTO_TOOLCHAIN] + if toolchain == None: + fail("protobuf_doc_shard " + str(ctx.label) + ": no protobuf toolchain is registered for this platform") + package = ctx.attr.package or ctx.attr.proto.label.name + action = protobuf_shard_action( + ctx, + ctx.attr.proto[ProtoInfo], + toolchain.proto.proto_compiler, + ctx.attr._exec_os[ExecOsInfo].windows, + package, + ctx.label.name, + ctx.executable._extract, + ) + return [ + DefaultInfo(files = depset([action.shard])), + OutputGroupInfo(descriptor_set = depset([action.descriptor])), + DocsShardInfo(language = "protobuf", package = package, shard = action.shard), + ] + +docs_protobuf_shard = rule( + doc = "Converts one proto_library's messages, enums and services into a DocIr shard with protoc.", + implementation = _protobuf_doc_shard_impl, + attrs = { + "package": attr.string(doc = "Package segment of the symbol ids; defaults to the name of proto."), + "proto": attr.label( + mandatory = True, + providers = [ProtoInfo], + doc = "The documented proto_library; its own .proto srcs are documented.", + ), + "_exec_os": attr.label( + default = Label("//libs/exec_os"), + cfg = "exec", + providers = [ExecOsInfo], + ), + "_extract": attr.label( + cfg = "exec", + default = Label("//tools/docs/protobuf/extract:protobuf_extract"), + executable = True, + ), + }, + provides = [DocsShardInfo], + toolchains = [config_common.toolchain_type(PROTO_TOOLCHAIN, mandatory = False)], +) + +def protobuf_doc_shard(name, proto, package = "", **kwargs): + """Declares the DocIr shard of one proto_library. + + Args: + name: Name of the shard target. + proto: The documented proto_library. + package: Package segment of every symbol id; defaults to the name of proto. + **kwargs: Common attributes such as tags and visibility. + """ + docs_protobuf_shard( + name = name, + proto = proto, + package = package, + **kwargs + ) diff --git a/tools/docs/protobuf/extract/BUILD.bazel b/tools/docs/protobuf/extract/BUILD.bazel new file mode 100644 index 0000000000..a083822e7d --- /dev/null +++ b/tools/docs/protobuf/extract/BUILD.bazel @@ -0,0 +1,87 @@ +load("@crates//:crates.bzl", "aliases", "all_crate_deps") +load("//quality:fixtures.bzl", "real_source_target") +load("//rust/rules:defs.bzl", "rust_binary", "rust_library", "rust_test") +load("//tools/visibility:visibility.bzl", "VIS_TOOLS", "VIS_TOOLS_PUBLIC") + +package(default_visibility = VIS_TOOLS_PUBLIC) + +exports_files( + [ + "Cargo.toml", + "src/lib.rs", + "src/lib_tests.rs", + "src/main.rs", + ], + visibility = VIS_TOOLS, +) + +rust_library( + name = "dx_docs_protobuf_extract", + srcs = ["src/lib.rs"], + aliases = aliases( + package_name = "tools/docs/protobuf/protobuf_extract", + normal = True, + ), + crate_name = "dx_docs_protobuf_extract", + crate_root = "src/lib.rs", + deps = all_crate_deps( + package_name = "tools/docs/protobuf/protobuf_extract", + normal = True, + ) + [ + "//tools/docs/ir/ir:documentation_ir", + "@rules_rust_prost//private/3rdparty/crates:prost", + "@rules_rust_prost//private/3rdparty/crates:prost-types", + ], +) + +rust_binary( + name = "protobuf_extract", + srcs = ["src/main.rs"], + aliases = aliases( + package_name = "tools/docs/protobuf/protobuf_extract", + normal = True, + ), + crate_name = "protobuf_extract", + crate_root = "src/main.rs", + deps = all_crate_deps( + package_name = "tools/docs/protobuf/protobuf_extract", + normal = True, + ) + [":dx_docs_protobuf_extract"], +) + +rust_test( + name = "dx_docs_protobuf_extract_test", + srcs = [ + "src/lib.rs", + "src/lib_tests.rs", + ], + aliases = aliases( + package_name = "tools/docs/protobuf/protobuf_extract", + normal = True, + normal_dev = True, + ), + coverage_lib = ":dx_docs_protobuf_extract", + crate_name = "dx_docs_protobuf_extract", + crate_root = "src/lib.rs", + deps = all_crate_deps( + package_name = "tools/docs/protobuf/protobuf_extract", + normal = True, + normal_dev = True, + ) + [ + "//tools/docs/ir/ir:documentation_ir", + "@rules_rust_prost//private/3rdparty/crates:prost", + "@rules_rust_prost//private/3rdparty/crates:prost-types", + ], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) + +real_source_target( + name = "corpus_toml", + tags = ["corpus"], + toml_srcs = ["Cargo.toml"], +) diff --git a/tools/docs/protobuf/extract/Cargo.toml b/tools/docs/protobuf/extract/Cargo.toml new file mode 100644 index 0000000000..b2e9d73482 --- /dev/null +++ b/tools/docs/protobuf/extract/Cargo.toml @@ -0,0 +1,15 @@ +[package] +license = "Apache-2.0" +name = "protobuf_extract" +version = "0.0.0" +edition = "2024" + +[dependencies] + +[lib] +name = "dx_docs_protobuf_extract" +path = "src/lib.rs" + +[[bin]] +name = "protobuf_extract" +path = "src/main.rs" diff --git a/tools/docs/protobuf/extract/src/lib.rs b/tools/docs/protobuf/extract/src/lib.rs new file mode 100644 index 0000000000..f1c52591c2 --- /dev/null +++ b/tools/docs/protobuf/extract/src/lib.rs @@ -0,0 +1,491 @@ +//! Converts a protoc descriptor set with source info into one validated DocIr shard. + +#![cfg_attr( + not(test), + deny( + clippy::expect_used, + clippy::unwrap_used, + clippy::unreachable, + clippy::todo + ) +)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; + +use documentation_ir::proto::{ + DocIr, Extension, Param, Relations, Returns, SourceRef, Symbol, SymbolKind, Visibility, +}; +use prost::Message; +use prost_types::field_descriptor_proto::{Label, Type}; +use prost_types::{ + DescriptorProto, EnumDescriptorProto, FileDescriptorProto, FileDescriptorSet, + ServiceDescriptorProto, +}; + +const LANGUAGE: &str = "protobuf"; +const KEY_INCOMPLETE: &str = "protobuf.incomplete"; + +// Field numbers of the descriptor messages, used in source info paths. +const FILE_MESSAGE: i32 = 4; +const FILE_ENUM: i32 = 5; +const FILE_SERVICE: i32 = 6; +const MESSAGE_FIELD: i32 = 2; +const MESSAGE_NESTED: i32 = 3; +const MESSAGE_ENUM: i32 = 4; +const ENUM_VALUE: i32 = 2; +const SERVICE_METHOD: i32 = 2; + +/// The inputs of one conversion. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Options { + pub descriptor: String, + pub package: String, + pub out: String, + /// Proto name (as in the descriptor) to workspace-relative path. + pub srcs: BTreeMap, + /// Proto names whose source is build output. + pub generated: BTreeSet, +} + +/// The usage line shown for a rejected invocation. +pub fn usage() -> &'static str { + "usage: protobuf_extract --descriptor FILE --package NAME --out FILE --src PROTO=PATH... [--generated PROTO]..." +} + +/// Parses argv into inputs, or returns the rejection text. +pub fn parse(args: &[String]) -> Result { + let mut options = Options::default(); + let mut index = 0; + while index < args.len() { + let flag = args[index].as_str(); + let Some(value) = args.get(index + 1) else { + return Err(format!("{flag} needs a value")); + }; + if value.is_empty() { + return Err(format!("{flag} must not be empty")); + } + match flag { + "--src" => { + let Some((name, path)) = value.split_once('=') else { + return Err(format!("--src '{value}' is not PROTO=PATH")); + }; + if name.is_empty() || path.is_empty() { + return Err(format!("--src '{value}' is not PROTO=PATH")); + } + if options + .srcs + .insert(name.to_owned(), path.to_owned()) + .is_some() + { + return Err(format!("--src given twice for '{name}'")); + } + } + "--generated" => { + options.generated.insert(value.clone()); + } + "--descriptor" | "--package" | "--out" => { + let slot = match flag { + "--descriptor" => &mut options.descriptor, + "--package" => &mut options.package, + _ => &mut options.out, + }; + if !slot.is_empty() { + return Err(format!("{flag} given more than once")); + } + *slot = value.clone(); + } + _ => return Err(format!("unknown argument '{flag}'")), + } + index += 2; + } + for (flag, value) in [ + ("--descriptor", &options.descriptor), + ("--package", &options.package), + ("--out", &options.out), + ] { + if value.is_empty() { + return Err(format!("{flag} is required")); + } + } + Ok(options) +} + +/// Reads, converts, validates and writes one shard, returning the failure text on error. +pub fn run(options: &Options) -> Result<(), String> { + let bytes = fs::read(&options.descriptor) + .map_err(|error| format!("cannot read {}: {error}", options.descriptor))?; + let shard = convert(&bytes, options)?; + let bytes = documentation_ir::encode_shard(&shard) + .map_err(|error| format!("cannot encode IR: {error:?}"))?; + fs::write(&options.out, bytes).map_err(|error| format!("cannot write {}: {error}", options.out)) +} + +/// Strips the comment markers' shared leading space and trailing blank space. +fn clean_comment(text: &str) -> String { + let text = text.replace("\r\n", "\n"); + let lines: Vec<&str> = text + .lines() + .map(|line| line.strip_prefix(' ').unwrap_or(line)) + .collect(); + lines.join("\n").trim().to_owned() +} + +/// The comments and line of every item in one file, keyed by source info path. +struct Locations { + by_path: BTreeMap, (String, u64)>, +} + +impl Locations { + fn new(file: &FileDescriptorProto) -> Result { + let Some(info) = file.source_code_info.as_ref() else { + return Err(format!( + "{} has no source info; run protoc with --include_source_info", + file.name() + )); + }; + let mut by_path = BTreeMap::new(); + for location in &info.location { + let leading = location.leading_comments(); + let comment = if leading.trim().is_empty() { + location.trailing_comments() + } else { + leading + }; + let line = location.span.first().map_or(0, |line| *line as u64 + 1); + by_path + .entry(location.path.clone()) + .or_insert((clean_comment(comment), line)); + } + Ok(Self { by_path }) + } + + fn get(&self, path: &[i32]) -> (String, u64) { + self.by_path.get(path).cloned().unwrap_or_default() + } +} + +fn type_name(field: &prost_types::FieldDescriptorProto) -> String { + match field.r#type() { + Type::Double => "double".to_owned(), + Type::Float => "float".to_owned(), + Type::Int64 => "int64".to_owned(), + Type::Uint64 => "uint64".to_owned(), + Type::Int32 => "int32".to_owned(), + Type::Fixed64 => "fixed64".to_owned(), + Type::Fixed32 => "fixed32".to_owned(), + Type::Bool => "bool".to_owned(), + Type::String => "string".to_owned(), + Type::Bytes => "bytes".to_owned(), + Type::Uint32 => "uint32".to_owned(), + Type::Sfixed32 => "sfixed32".to_owned(), + Type::Sfixed64 => "sfixed64".to_owned(), + Type::Sint32 => "sint32".to_owned(), + Type::Sint64 => "sint64".to_owned(), + Type::Group | Type::Message | Type::Enum => { + field.type_name().trim_start_matches('.').to_owned() + } + } +} + +struct Builder<'a> { + file: &'a FileDescriptorProto, + locations: Locations, + package: String, + source: String, + generated: bool, + /// Full names of the generated `map` entry messages and their types. + maps: BTreeMap, + symbols: Vec, +} + +impl Builder<'_> { + #[allow(clippy::too_many_arguments)] + fn push( + &mut self, + segments: &[String], + kind: SymbolKind, + signature: String, + path: &[i32], + parent: Option<&[String]>, + params: Vec, + returns: Option, + ) { + let (doc, line) = self.locations.get(path); + let id_of = |segments: &[String]| { + format!( + "{LANGUAGE}:{}:{}", + self.package, + segments + .iter() + .map(|s| documentation_ir::escape_id_segment(s)) + .collect::>() + .join(".") + ) + }; + let mut extensions = Vec::new(); + if doc.is_empty() { + extensions.push(Extension { + key: KEY_INCOMPLETE.to_owned(), + value: b"doc".to_vec(), + }); + } + self.symbols.push(Symbol { + id: id_of(segments), + kind: kind as i32, + signature_text: signature, + doc_markdown: doc, + params, + returns, + source: Some(SourceRef { + file: self.source.clone(), + line, + generated: self.generated, + }), + visibility: Visibility::Public as i32, + relations: parent.map(|parent| Relations { + member_of: vec![id_of(parent)], + implements: Vec::new(), + reexported_as: Vec::new(), + }), + extensions, + display_name: segments.join("."), + ..Symbol::default() + }); + } + + fn collect_maps(&mut self, prefix: &str, messages: &[DescriptorProto]) { + for message in messages { + let full = format!("{prefix}.{}", message.name()); + if message.options.as_ref().is_some_and(|o| o.map_entry()) { + let key = message.field.iter().find(|f| f.number() == 1); + let value = message.field.iter().find(|f| f.number() == 2); + if let (Some(key), Some(value)) = (key, value) { + self.maps.insert( + full.trim_start_matches('.').to_owned(), + (type_name(key), type_name(value)), + ); + } + } + self.collect_maps(&full, &message.nested_type); + } + } + + fn message(&mut self, scope: &[String], nested: bool, path: &[i32], message: &DescriptorProto) { + let mut segments = scope.to_vec(); + segments.push(message.name().to_owned()); + self.push( + &segments, + SymbolKind::Class, + format!("message {}", message.name()), + path, + nested.then_some(scope), + Vec::new(), + None, + ); + for (index, field) in message.field.iter().enumerate() { + let mut field_path = path.to_vec(); + field_path.extend([MESSAGE_FIELD, index as i32]); + let mut field_segments = segments.clone(); + field_segments.push(field.name().to_owned()); + let ty = type_name(field); + let signature = match self.maps.get(&ty) { + Some((key, value)) => { + format!("map<{key}, {value}> {} = {};", field.name(), field.number()) + } + None => { + let label = if field.label() == Label::Repeated { + "repeated " + } else if field.proto3_optional() { + "optional " + } else { + "" + }; + format!("{label}{ty} {} = {};", field.name(), field.number()) + } + }; + self.push( + &field_segments, + SymbolKind::Field, + signature, + &field_path, + Some(&segments), + Vec::new(), + None, + ); + } + for (index, nested) in message.nested_type.iter().enumerate() { + if nested.options.as_ref().is_some_and(|o| o.map_entry()) { + continue; + } + let mut nested_path = path.to_vec(); + nested_path.extend([MESSAGE_NESTED, index as i32]); + self.message(&segments, true, &nested_path, nested); + } + for (index, nested) in message.enum_type.iter().enumerate() { + let mut nested_path = path.to_vec(); + nested_path.extend([MESSAGE_ENUM, index as i32]); + self.enumeration(&segments, true, &nested_path, nested); + } + } + + fn enumeration( + &mut self, + scope: &[String], + nested: bool, + path: &[i32], + item: &EnumDescriptorProto, + ) { + let mut segments = scope.to_vec(); + segments.push(item.name().to_owned()); + self.push( + &segments, + SymbolKind::Enum, + format!("enum {}", item.name()), + path, + nested.then_some(scope), + Vec::new(), + None, + ); + for (index, value) in item.value.iter().enumerate() { + let mut value_path = path.to_vec(); + value_path.extend([ENUM_VALUE, index as i32]); + let mut value_segments = segments.clone(); + value_segments.push(value.name().to_owned()); + self.push( + &value_segments, + SymbolKind::EnumVariant, + format!("{} = {};", value.name(), value.number()), + &value_path, + Some(&segments), + Vec::new(), + None, + ); + } + } + + fn service( + &mut self, + scope: &[String], + nested: bool, + path: &[i32], + item: &ServiceDescriptorProto, + ) { + let mut segments = scope.to_vec(); + segments.push(item.name().to_owned()); + self.push( + &segments, + SymbolKind::Interface, + format!("service {}", item.name()), + path, + nested.then_some(scope), + Vec::new(), + None, + ); + for (index, method) in item.method.iter().enumerate() { + let mut method_path = path.to_vec(); + method_path.extend([SERVICE_METHOD, index as i32]); + let mut method_segments = segments.clone(); + method_segments.push(method.name().to_owned()); + let input = method.input_type().trim_start_matches('.').to_owned(); + let output = method.output_type().trim_start_matches('.').to_owned(); + let in_stream = if method.client_streaming() { + "stream " + } else { + "" + }; + let out_stream = if method.server_streaming() { + "stream " + } else { + "" + }; + let signature = format!( + "rpc {}({in_stream}{input}) returns ({out_stream}{output});", + method.name() + ); + self.push( + &method_segments, + SymbolKind::Method, + signature, + &method_path, + Some(&segments), + vec![Param { + name: "request".to_owned(), + r#type: format!("{in_stream}{input}"), + doc: String::new(), + }], + Some(Returns { + r#type: format!("{out_stream}{output}"), + doc: String::new(), + }), + ); + } + } +} + +/// Converts one descriptor set into a validated IR shard of one package. +pub fn convert(descriptor: &[u8], options: &Options) -> Result { + let package = options.package.as_str(); + if package.is_empty() || package.contains(':') || package.chars().any(char::is_whitespace) { + return Err(format!("invalid package '{package}'")); + } + let set = FileDescriptorSet::decode(descriptor) + .map_err(|error| format!("descriptor set is malformed: {error}"))?; + let mut symbols = Vec::new(); + for file in &set.file { + let Some(source) = options.srcs.get(file.name()) else { + return Err(format!( + "descriptor set holds '{}', which is not one of the library's sources", + file.name() + )); + }; + let mut builder = Builder { + file, + locations: Locations::new(file)?, + package: package.to_owned(), + source: source.clone(), + generated: options.generated.contains(file.name()), + maps: BTreeMap::new(), + symbols: Vec::new(), + }; + let proto_package = builder.file.package().to_owned(); + let scope: Vec = proto_package + .split('.') + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .collect(); + let prefix = format!(".{proto_package}"); + let prefix = if proto_package.is_empty() { + String::new() + } else { + prefix + }; + builder.collect_maps(&prefix, &file.message_type); + for (index, message) in file.message_type.iter().enumerate() { + builder.message(&scope, false, &[FILE_MESSAGE, index as i32], message); + } + for (index, item) in file.enum_type.iter().enumerate() { + builder.enumeration(&scope, false, &[FILE_ENUM, index as i32], item); + } + for (index, item) in file.service.iter().enumerate() { + builder.service(&scope, false, &[FILE_SERVICE, index as i32], item); + } + symbols.append(&mut builder.symbols); + } + if symbols.is_empty() { + return Err(format!( + "descriptor set declares no Protobuf API of package '{package}'" + )); + } + symbols.sort_by(|a, b| a.id.cmp(&b.id)); + let shard = DocIr { + language: LANGUAGE.to_owned(), + package: package.to_owned(), + symbols, + }; + documentation_ir::validate_shard(&shard).map_err(|error| format!("invalid IR: {error:?}"))?; + Ok(shard) +} + +#[cfg(test)] +mod lib_tests; diff --git a/tools/docs/protobuf/extract/src/lib_tests.rs b/tools/docs/protobuf/extract/src/lib_tests.rs new file mode 100644 index 0000000000..51af61590c --- /dev/null +++ b/tools/docs/protobuf/extract/src/lib_tests.rs @@ -0,0 +1,361 @@ +use super::*; +use prost_types::source_code_info::Location; +use prost_types::{ + EnumValueDescriptorProto, FieldDescriptorProto, MessageOptions, MethodDescriptorProto, + SourceCodeInfo, +}; + +const P: &str = "protobuf:shop:"; + +fn location(path: &[i32], line: i32, comment: &str) -> Location { + Location { + path: path.to_vec(), + span: vec![line, 0, 1], + leading_comments: if comment.is_empty() { + None + } else { + Some(comment.to_owned()) + }, + ..Location::default() + } +} + +fn field(name: &str, number: i32, ty: Type, type_name: &str) -> FieldDescriptorProto { + FieldDescriptorProto { + name: Some(name.to_owned()), + number: Some(number), + label: Some(Label::Optional as i32), + r#type: Some(ty as i32), + type_name: (!type_name.is_empty()).then(|| type_name.to_owned()), + ..FieldDescriptorProto::default() + } +} + +fn file() -> FileDescriptorProto { + let mut tags = field("tags", 2, Type::String, ""); + tags.label = Some(Label::Repeated as i32); + let attrs = FieldDescriptorProto { + label: Some(Label::Repeated as i32), + ..field("attrs", 3, Type::Message, ".shop.v1.Order.AttrsEntry") + }; + let entry = DescriptorProto { + name: Some("AttrsEntry".to_owned()), + field: vec![ + field("key", 1, Type::String, ""), + field("value", 2, Type::Int32, ""), + ], + options: Some(MessageOptions { + map_entry: Some(true), + ..MessageOptions::default() + }), + ..DescriptorProto::default() + }; + let order = DescriptorProto { + name: Some("Order".to_owned()), + field: vec![ + field("id", 1, Type::String, ""), + tags, + attrs, + field("status", 4, Type::Enum, ".shop.v1.Status"), + ], + nested_type: vec![ + entry, + DescriptorProto { + name: Some("Line".to_owned()), + ..DescriptorProto::default() + }, + ], + enum_type: vec![EnumDescriptorProto { + name: Some("Kind".to_owned()), + value: vec![EnumValueDescriptorProto { + name: Some("KIND_A".to_owned()), + number: Some(0), + ..EnumValueDescriptorProto::default() + }], + ..EnumDescriptorProto::default() + }], + ..DescriptorProto::default() + }; + let status = EnumDescriptorProto { + name: Some("Status".to_owned()), + value: vec![ + EnumValueDescriptorProto { + name: Some("STATUS_UNKNOWN".to_owned()), + number: Some(0), + ..EnumValueDescriptorProto::default() + }, + EnumValueDescriptorProto { + name: Some("STATUS_OPEN".to_owned()), + number: Some(1), + ..EnumValueDescriptorProto::default() + }, + ], + ..EnumDescriptorProto::default() + }; + let service = ServiceDescriptorProto { + name: Some("Shop".to_owned()), + method: vec![ + MethodDescriptorProto { + name: Some("Get".to_owned()), + input_type: Some(".shop.v1.Order".to_owned()), + output_type: Some(".shop.v1.Order".to_owned()), + ..MethodDescriptorProto::default() + }, + MethodDescriptorProto { + name: Some("Watch".to_owned()), + input_type: Some(".shop.v1.Order".to_owned()), + output_type: Some(".shop.v1.Order".to_owned()), + client_streaming: Some(true), + server_streaming: Some(true), + ..MethodDescriptorProto::default() + }, + ], + ..ServiceDescriptorProto::default() + }; + FileDescriptorProto { + name: Some("shop/v1/shop.proto".to_owned()), + package: Some("shop.v1".to_owned()), + message_type: vec![order], + enum_type: vec![status], + service: vec![service], + source_code_info: Some(SourceCodeInfo { + location: vec![ + location(&[4, 0], 5, " An order.\n\n Second paragraph.\n"), + location(&[4, 0, 2, 0], 6, " The order id.\n"), + location(&[4, 0, 2, 2], 8, " Free-form attributes.\n"), + location(&[4, 0, 3, 1], 12, " One line of an order.\n"), + location(&[4, 0, 4, 0], 14, " Order kinds.\n"), + location(&[4, 0, 4, 0, 2, 0], 15, " First kind.\n"), + location(&[5, 0], 20, " Order status.\n"), + location(&[5, 0, 2, 1], 22, " The order is open.\n"), + location(&[6, 0], 26, " The shop API.\n"), + location(&[6, 0, 2, 0], 27, " Reads one order.\n"), + location(&[6, 0, 2, 1], 29, " Streams orders.\n"), + ], + }), + ..FileDescriptorProto::default() + } +} + +fn options() -> Options { + let mut options = Options { + package: "shop".to_owned(), + out: "out".to_owned(), + descriptor: "in".to_owned(), + ..Options::default() + }; + options.srcs.insert( + "shop/v1/shop.proto".to_owned(), + "api/shop/v1/shop.proto".to_owned(), + ); + options +} + +fn encode(files: Vec) -> Vec { + FileDescriptorSet { file: files }.encode_to_vec() +} + +fn convert_one() -> DocIr { + convert(&encode(vec![file()]), &options()).unwrap() +} + +fn symbol<'a>(shard: &'a DocIr, qualified: &str) -> &'a Symbol { + let id = format!("{P}{qualified}"); + shard + .symbols + .iter() + .find(|s| s.id == id) + .unwrap_or_else(|| panic!("{id} is missing: {:?}", ids(shard))) +} + +fn ids(shard: &DocIr) -> Vec<&str> { + shard.symbols.iter().map(|s| s.id.as_str()).collect() +} + +fn members(symbol: &Symbol) -> Vec<&str> { + symbol + .relations + .as_ref() + .map(|r| r.member_of.iter().map(String::as_str).collect()) + .unwrap_or_default() +} + +#[test] +fn every_kind_maps_with_its_comment_and_line() { + let shard = convert_one(); + let order = symbol(&shard, "shop.v1.Order"); + assert_eq!(order.kind, SymbolKind::Class as i32); + assert_eq!(order.doc_markdown, "An order.\n\nSecond paragraph."); + assert_eq!(order.signature_text, "message Order"); + let source = order.source.as_ref().unwrap(); + assert_eq!(source.file, "api/shop/v1/shop.proto"); + assert_eq!(source.line, 6); + assert!(!source.generated); + assert!(members(order).is_empty()); + + let id = symbol(&shard, "shop.v1.Order.id"); + assert_eq!(id.kind, SymbolKind::Field as i32); + assert_eq!(id.signature_text, "string id = 1;"); + assert_eq!(id.doc_markdown, "The order id."); + assert_eq!(members(id), vec![format!("{P}shop.v1.Order")]); + + let tags = symbol(&shard, "shop.v1.Order.tags"); + assert_eq!(tags.signature_text, "repeated string tags = 2;"); + + let status = symbol(&shard, "shop.v1.Status"); + assert_eq!(status.kind, SymbolKind::Enum as i32); + let open = symbol(&shard, "shop.v1.Status.STATUS_OPEN"); + assert_eq!(open.kind, SymbolKind::EnumVariant as i32); + assert_eq!(open.signature_text, "STATUS_OPEN = 1;"); + assert_eq!(open.doc_markdown, "The order is open."); + assert_eq!(members(open), vec![format!("{P}shop.v1.Status")]); + + let service = symbol(&shard, "shop.v1.Shop"); + assert_eq!(service.kind, SymbolKind::Interface as i32); + let get = symbol(&shard, "shop.v1.Shop.Get"); + assert_eq!(get.kind, SymbolKind::Method as i32); + assert_eq!( + get.signature_text, + "rpc Get(shop.v1.Order) returns (shop.v1.Order);" + ); + assert_eq!(get.doc_markdown, "Reads one order."); + assert_eq!(get.params[0].r#type, "shop.v1.Order"); +} + +#[test] +fn nested_types_and_enums_are_members_of_their_message() { + let shard = convert_one(); + let line = symbol(&shard, "shop.v1.Order.Line"); + assert_eq!(line.kind, SymbolKind::Class as i32); + assert_eq!(line.doc_markdown, "One line of an order."); + assert_eq!(members(line), vec![format!("{P}shop.v1.Order")]); + let kind = symbol(&shard, "shop.v1.Order.Kind"); + assert_eq!(kind.kind, SymbolKind::Enum as i32); + assert_eq!(members(kind), vec![format!("{P}shop.v1.Order")]); + let value = symbol(&shard, "shop.v1.Order.Kind.KIND_A"); + assert_eq!(value.doc_markdown, "First kind."); + assert_eq!(members(value), vec![format!("{P}shop.v1.Order.Kind")]); +} + +#[test] +fn map_fields_show_as_maps_and_entry_messages_are_hidden() { + let shard = convert_one(); + let attrs = symbol(&shard, "shop.v1.Order.attrs"); + assert_eq!(attrs.signature_text, "map attrs = 3;"); + assert_eq!(attrs.doc_markdown, "Free-form attributes."); + assert!(!ids(&shard).iter().any(|id| id.contains("AttrsEntry"))); +} + +#[test] +fn streaming_rpcs_are_marked() { + let shard = convert_one(); + let watch = symbol(&shard, "shop.v1.Shop.Watch"); + assert_eq!( + watch.signature_text, + "rpc Watch(stream shop.v1.Order) returns (stream shop.v1.Order);" + ); + assert_eq!( + watch.returns.as_ref().unwrap().r#type, + "stream shop.v1.Order" + ); +} + +#[test] +fn missing_comments_are_flagged_incomplete_not_invented() { + let shard = convert_one(); + let status_field = symbol(&shard, "shop.v1.Order.status"); + assert!(status_field.doc_markdown.is_empty()); + assert_eq!(status_field.extensions[0].key, KEY_INCOMPLETE); + assert!( + !symbol(&shard, "shop.v1.Order") + .extensions + .iter() + .any(|e| e.key == KEY_INCOMPLETE) + ); +} + +#[test] +fn a_descriptor_without_source_info_is_rejected() { + let mut bare = file(); + bare.source_code_info = None; + let error = convert(&encode(vec![bare]), &options()).unwrap_err(); + assert!(error.contains("--include_source_info"), "{error}"); +} + +#[test] +fn a_file_outside_the_sources_is_rejected() { + let mut other = options(); + other.srcs.clear(); + other + .srcs + .insert("x.proto".to_owned(), "x.proto".to_owned()); + let error = convert(&encode(vec![file()]), &other).unwrap_err(); + assert!(error.contains("shop/v1/shop.proto"), "{error}"); +} + +#[test] +fn trailing_comments_document_an_item_without_a_leading_one() { + let mut trailing = file(); + let info = trailing.source_code_info.as_mut().unwrap(); + info.location.push(Location { + path: vec![4, 0, 2, 3], + span: vec![9, 0, 1], + trailing_comments: Some(" The status.\n".to_owned()), + ..Location::default() + }); + let shard = convert(&encode(vec![trailing]), &options()).unwrap(); + assert_eq!( + symbol(&shard, "shop.v1.Order.status").doc_markdown, + "The status." + ); +} + +#[test] +fn symbols_are_sorted_and_the_shard_validates() { + let shard = convert_one(); + let mut sorted = ids(&shard); + sorted.sort(); + assert_eq!(ids(&shard), sorted); + documentation_ir::validate_shard(&shard).unwrap(); +} + +#[test] +fn generated_sources_are_marked() { + let mut generated = options(); + generated.generated.insert("shop/v1/shop.proto".to_owned()); + let shard = convert(&encode(vec![file()]), &generated).unwrap(); + assert!( + symbol(&shard, "shop.v1.Order") + .source + .as_ref() + .unwrap() + .generated + ); +} + +#[test] +fn parse_collects_sources_and_rejects_bad_flags() { + let args = |list: &[&str]| list.iter().map(|s| (*s).to_owned()).collect::>(); + let parsed = parse(&args(&[ + "--descriptor", + "d", + "--package", + "p", + "--out", + "o", + "--src", + "a.proto=x/a.proto", + "--generated", + "a.proto", + ])) + .unwrap(); + assert_eq!(parsed.srcs["a.proto"], "x/a.proto"); + assert!(parsed.generated.contains("a.proto")); + assert!(parse(&args(&["--bogus", "1"])).is_err()); + assert!(parse(&args(&["--src", "nopath"])).is_err()); + assert!( + parse(&args(&["--package", "p"])) + .unwrap_err() + .contains("--descriptor") + ); +} diff --git a/tools/docs/protobuf/extract/src/main.rs b/tools/docs/protobuf/extract/src/main.rs new file mode 100644 index 0000000000..2dc92da4e6 --- /dev/null +++ b/tools/docs/protobuf/extract/src/main.rs @@ -0,0 +1,38 @@ +//! Command line entry for the Protobuf docs converter. + +#![cfg_attr( + not(test), + deny( + clippy::expect_used, + clippy::unwrap_used, + clippy::unreachable, + clippy::todo + ) +)] + +use std::process::ExitCode; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + if args.iter().any(|a| a == "-h" || a == "--help") { + println!("{}", dx_docs_protobuf_extract::usage()); + return ExitCode::SUCCESS; + } + let options = match dx_docs_protobuf_extract::parse(&args) { + Ok(options) => options, + Err(detail) => { + eprintln!( + "protobuf_extract: {detail}\n{}", + dx_docs_protobuf_extract::usage() + ); + return ExitCode::from(2); + } + }; + match dx_docs_protobuf_extract::run(&options) { + Ok(()) => ExitCode::SUCCESS, + Err(detail) => { + eprintln!("protobuf_extract: {detail}"); + ExitCode::FAILURE + } + } +} diff --git a/tools/docs/protobuf/fixture/BUILD.bazel b/tools/docs/protobuf/fixture/BUILD.bazel new file mode 100644 index 0000000000..7de11381b4 --- /dev/null +++ b/tools/docs/protobuf/fixture/BUILD.bazel @@ -0,0 +1,39 @@ +load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("//quality:fixtures.bzl", "real_source_target") +load("//tools/docs/protobuf:defs.bzl", "protobuf_doc_shard") +load("//tools/visibility:visibility.bzl", "VIS_TOOLS") + +package(default_visibility = VIS_TOOLS) + +exports_files( + [ + "common.proto", + "shop.proto", + ], + visibility = VIS_TOOLS, +) + +proto_library( + name = "common_proto", + srcs = ["common.proto"], + deps = ["@protobuf//:timestamp_proto"], +) + +proto_library( + name = "shop_proto", + srcs = ["shop.proto"], + deps = [":common_proto"], +) + +protobuf_doc_shard( + name = "shop_docs", + testonly = True, + package = "shop", + proto = ":shop_proto", +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/tools/docs/protobuf/fixture/common.proto b/tools/docs/protobuf/fixture/common.proto new file mode 100644 index 0000000000..38b414d610 --- /dev/null +++ b/tools/docs/protobuf/fixture/common.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package shop.v1; + +import "google/protobuf/timestamp.proto"; + +// How an order is going. +enum Status { + // No status was set. + STATUS_UNSPECIFIED = 0; + // The order waits for payment. + STATUS_OPEN = 1; + // The order was paid. + STATUS_PAID = 2; +} + +// A moment on the shop's clock. +message Stamp { + // When the moment happened. + google.protobuf.Timestamp at = 1; +} diff --git a/tools/docs/protobuf/fixture/shop.proto b/tools/docs/protobuf/fixture/shop.proto new file mode 100644 index 0000000000..cb65ffd8d4 --- /dev/null +++ b/tools/docs/protobuf/fixture/shop.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package shop.v1; + +import "tools/docs/protobuf/fixture/common.proto"; + +// A customer order. +message Order { + // One line of an order. + message Line { + // The product's SKU. + string sku = 1; + // How many units were bought. + int32 quantity = 2; + } + + // The order id. + string id = 1; + // Where the order stands. + Status status = 2; + // The lines of the order. + repeated Line lines = 3; + // Free-form labels keyed by name. + map labels = 4; + // When the order was placed. + Stamp placed = 5; +} + +// A request for one order. +message GetOrderRequest { + // The id of the order to read. + string id = 1; +} + +// The order API. +service Shop { + // Reads one order. + rpc GetOrder(GetOrderRequest) returns (Order); + // Streams every change to the orders a client watches. + rpc Watch(stream GetOrderRequest) returns (stream Order); +} diff --git a/tools/docs/protobuf/shard_test.rs b/tools/docs/protobuf/shard_test.rs new file mode 100644 index 0000000000..59901bf520 --- /dev/null +++ b/tools/docs/protobuf/shard_test.rs @@ -0,0 +1,82 @@ +//! Checks that the fixture shard carries the comments of the fixture's .proto files. + +use std::path::PathBuf; + +use documentation_ir::proto::{DocIr, Symbol, SymbolKind}; +use documentation_ir::{decode_shard, validate_shard}; + +const PREFIX: &str = "protobuf:shop:"; + +fn runfile(var: &str) -> PathBuf { + let rel = std::env::var(var).unwrap_or_else(|_| panic!("{var} must name a runfile")); + dx_testing::resolve_runfiles(&rel) +} + +fn shard() -> DocIr { + let bytes = std::fs::read(runfile("DX_FIXTURE_SHARD")).expect("read the shard"); + decode_shard(&bytes).expect("the shard decodes") +} + +fn symbol<'a>(shard: &'a DocIr, qualified: &str) -> &'a Symbol { + let id = format!("{PREFIX}{qualified}"); + shard + .symbols + .iter() + .find(|s| s.id == id) + .unwrap_or_else(|| panic!("{id} is missing")) +} + +#[test] +fn the_fixture_shard_documents_every_item_with_its_comment() { + let shard = shard(); + validate_shard(&shard).expect("validate_shard passes"); + assert_eq!(shard.language, "protobuf"); + assert_eq!(shard.package, "shop"); + // Comments are lost when protoc reads the library's own descriptor set, so an empty doc + // text anywhere means the build fed it back in. + for symbol in &shard.symbols { + assert!(!symbol.doc_markdown.is_empty(), "{} has no doc", symbol.id); + } + for (qualified, kind, doc) in [ + ("shop.v1.Order", SymbolKind::Class, "A customer order."), + ( + "shop.v1.Order.Line", + SymbolKind::Class, + "One line of an order.", + ), + ("shop.v1.Order.id", SymbolKind::Field, "The order id."), + ("shop.v1.Shop", SymbolKind::Interface, "The order API."), + ( + "shop.v1.Shop.GetOrder", + SymbolKind::Method, + "Reads one order.", + ), + ] { + let found = symbol(&shard, qualified); + assert_eq!(found.kind, kind as i32, "{qualified}"); + assert_eq!(found.doc_markdown, doc, "{qualified}"); + } + let map = symbol(&shard, "shop.v1.Order.labels"); + assert_eq!(map.signature_text, "map labels = 4;"); + assert!(!shard.symbols.iter().any(|s| s.id.contains("LabelsEntry"))); + let watch = symbol(&shard, "shop.v1.Shop.Watch"); + assert_eq!( + watch.signature_text, + "rpc Watch(stream shop.v1.GetOrderRequest) returns (stream shop.v1.Order);" + ); + let status = symbol(&shard, "shop.v1.Order.status"); + assert_eq!(status.signature_text, "shop.v1.Status status = 2;"); +} + +#[test] +fn the_imported_proto_is_not_documented_by_the_importing_shard() { + let shard = shard(); + assert!(!shard.symbols.iter().any(|s| s.id.contains("Stamp"))); + let source = symbol(&shard, "shop.v1.Order") + .source + .clone() + .expect("source"); + assert_eq!(source.file, "tools/docs/protobuf/fixture/shop.proto"); + assert_eq!(source.line, 8); + assert!(!source.generated); +} diff --git a/tools/docs/support_matrix/facts_tests.bzl b/tools/docs/support_matrix/facts_tests.bzl index a5abd2734d..7c80cfde80 100644 --- a/tools/docs/support_matrix/facts_tests.bzl +++ b/tools/docs/support_matrix/facts_tests.bzl @@ -15,6 +15,7 @@ load("//tools/docs/jvm:defs.bzl", "java_doc_shard", "kotlin_doc_shard", "scala_d load("//tools/docs/perl:defs.bzl", "perl_doc_shard") load("//tools/docs/php:defs.bzl", "php_doc_shard") load("//tools/docs/powershell:defs.bzl", "powershell_doc_shard") +load("//tools/docs/protobuf:defs.bzl", "protobuf_doc_shard") load("//tools/docs/python:defs.bzl", "python_doc_shard") load("//tools/docs/ruby:defs.bzl", "ruby_doc_shard") load("//tools/docs/site:site.bzl", "docs_site") @@ -43,6 +44,7 @@ _SHARD_RULES = { "perl_doc_shard": perl_doc_shard, "php_doc_shard": php_doc_shard, "powershell_doc_shard": powershell_doc_shard, + "protobuf_doc_shard": protobuf_doc_shard, "python_doc_shard": python_doc_shard, "ruby_doc_shard": ruby_doc_shard, "docs_site": docs_site, diff --git a/tools/docs/support_matrix/languages.bzl b/tools/docs/support_matrix/languages.bzl index ed9577c184..542d9834c6 100644 --- a/tools/docs/support_matrix/languages.bzl +++ b/tools/docs/support_matrix/languages.bzl @@ -607,7 +607,7 @@ LANGUAGES = [ _language( "protobuf", deps = "none: schemas only, no packages", - doc = NO_DOC, + doc = struct(tool = "protoc", shard = "protobuf_doc_shard"), dead_code = ["none"], ), _language(