docs: Avro schema docs shard from the doc attribute and idl2schemata (closes #9936)

BUILD.bazel

@@ -76,6 +76,7 @@ load("//quality:json_schema.bzl", "json_schema", "json_schemas")
 # gazelle:exclude tools/docs/perl/fixture
 # gazelle:exclude tools/docs/php/fixture
 # gazelle:exclude tools/docs/protobuf/fixture
+# gazelle:exclude tools/docs/avro/fixture
 # gazelle:exclude tools/docs/go/godoc
 # gazelle:exclude cli/depcheck/testdata
 # gazelle:exclude go/tools/analysis

MODULE.bazel

@@ -1837,6 +1837,7 @@ crate.from_cargo(
         "//tools/docs/go/extract:Cargo.toml",
         "//tools/docs/php/extract:Cargo.toml",
         "//tools/docs/ruby/extract:Cargo.toml",
+        "//tools/docs/avro/extract:Cargo.toml",
         "//tools/docs/dart/extract:Cargo.toml",
         "//tools/docs/julia/extract:Cargo.toml",
         "//tools/docs/ocaml/extract:Cargo.toml",

cargo-bazel-lock.json

@@ -1,5 +1,5 @@
 {
-  "checksum": "c978dbd2ef70f7bebc134112a424e06fc4aae969ec64256d80463c455ca43c44",
+  "checksum": "72951dade25a7dda429098a64a357be578a2a81401ca32287b43d5d4d4d18989",
   "conditions": {
     "aarch64-apple-darwin": [
       "aarch64-apple-darwin"
@@ -1490,6 +1490,33 @@
       ],
       "version": "1.5.1"
     },
+    "avro_extract 0.0.0": {
+      "common_attrs": {
+        "deps": [
+          {
+            "id": "serde_json 1.0.151",
+            "target": "serde_json"
+          }
+        ],
+        "edition": "2024"
+      },
+      "library_target_name": "dx_docs_avro_extract",
+      "license": "Apache-2.0",
+      "license_ids": [
+        "Apache-2.0"
+      ],
+      "name": "avro_extract",
+      "repository": null,
+      "targets": [
+        {
+          "Library": {
+            "crate_name": "dx_docs_avro_extract",
+            "crate_root": "src/lib.rs"
+          }
+        }
+      ],
+      "version": "0.0.0"
+    },
     "base16ct 0.2.0": {
       "common_attrs": {
         "crate_features": [
@@ -28771,6 +28798,7 @@
   ],
   "workspace_members": {
     "appimage_packer 0.0.0": "appimage/appimage_packer",
+    "avro_extract 0.0.0": "tools/docs/avro/avro_extract",
     "bazelisk_wrapper 0.0.0": "tools/bazelisk_wrapper",
     "cc_context 0.0.0": "cc/cc_context",
     "cc_extract 0.0.0": "tools/docs/cc/cc_extract",

docs/cli/commands/docs.md

@@ -107,6 +107,7 @@ tools/dx docs //lib:site
 | C# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `csharp_doc_shard` | `library`, `package` |
 | F# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `fsharp_doc_shard` | `library`, `package` |
 | Protobuf | `@rules_dx//tools/docs/protobuf:defs.bzl` | `protobuf_doc_shard` | `proto`, `package` |
+| Avro | `@rules_dx//tools/docs/avro:defs.bzl` | `avro_doc_shard` | `avro`, `package` |
 
 Rust, Go, Python, C, C++, CUDA, Fortran, Java, Kotlin, Scala, D, Haskell, Dart, C#, F# and Perl libraries in scope that have
 no shard get one automatically. A C# library needs `nullable = "disable"` and
@@ -125,4 +126,12 @@ service an interface and an rpc a method. The comment above an item, or after
 it on the same line, is its doc text. A `map<K, V>` field shows as a map and
 its generated entry message is left out. Streaming rpcs say `stream`.
 
+An `avro_library` in scope that has no shard gets one automatically. Avro
+shards read the `doc` attribute straight from the schemas: a record is a class,
+a field a field, an enum an enum and its symbols variants, a fixed type a type
+alias and a protocol message a method. Ids start with the schema namespace. An
+`.avdl` file is first turned into schemas by the pinned avro-tools
+(`idl2schemata`) on the hermetic JDK. Avro enum symbols carry no doc of their
+own, so they are marked incomplete.
+
 A source error fails the build and writes no pages. Bazel's exit code is kept.

docs/support-matrix.md

@@ -80,7 +80,7 @@ Each cell names the tools `dx lint`, `dx format`, or `dx typecheck` runs, and th
 | Name | Lint | Format | Typecheck | API docs | Coverage | Dead code |
 | --- | --- | --- | --- | --- | --- | --- |
 | Astro | `eslint`, `typos` | `prettier` | `tsc` | `astro-compiler` | `jest` | `eslint @typescript-eslint/no-unused-vars` |
-| Avro | `avro_tools`, `typos` | `prettier` | `avro_tools` | no | n/a | none |
+| Avro | `avro_tools`, `typos` | `prettier` | `avro_tools` | `avro-tools` | n/a | none |
 | C and C++ | `clang_tidy`, `typos` | `clang_format` | `clang_syntax` | `doxygen` | `gcov`, `llvm-cov` | `clang_tidy misc-unused-parameters, misc-unused-using-decls` |
 | C# | `roslyn`, `typos` | `csharpier` | `csc` | `csharp doc driver` | `coverlet` | `roslyn IDE0051, IDE0052, IDE0060, IDE0005` |
 | CUDA | `clang_tidy`, `typos` | `clang_format` | `nvcc_syntax` | `doxygen` | no: cuda_test has no dx coverage wiring | none |

families/generation/README.md

@@ -12,5 +12,11 @@ A workspace that only uses the code generation rules needs nothing more than
 `protobuf_doc_shard` in `@rules_dx//tools/docs/protobuf:defs.bzl` documents one
 `proto_library`. It runs the protoc of the registered proto toolchain with
 `--include_source_info` and reads the comments above each message, field, enum,
-enum value, service and rpc from the descriptor set. Avro schemas are not
-covered; their `doc` fields are tracked in #9936.
+enum value, service and rpc from the descriptor set. 
+## Avro docs
+
+`avro_doc_shard` in `@rules_dx//tools/docs/avro:defs.bzl` documents one
+`avro_library`. The `doc` attribute of every record, field, enum, fixed type and
+protocol message is read from the `.avsc` files directly. An `.avdl` file is
+first converted with `idl2schemata` of the pinned avro-tools (`//config:tool_avro`),
+which runs on the hermetic JDK, and then read the same way.

modules/rust.bzl

@@ -60,6 +60,7 @@ RUST_CRATE_MANIFESTS = [
     "//tools/docs/go/extract:Cargo.toml",
     "//tools/docs/php/extract:Cargo.toml",
     "//tools/docs/ruby/extract:Cargo.toml",
+    "//tools/docs/avro/extract:Cargo.toml",
     "//tools/docs/dart/extract:Cargo.toml",
     "//tools/docs/julia/extract:Cargo.toml",
     "//tools/docs/ocaml/extract:Cargo.toml",

quality/testdata/BUILD.bazel

@@ -2008,6 +2008,11 @@ real_docs_subject(
     target = "//tools/docs/protobuf/fixture:shop_proto",
 )
 
+real_docs_subject(
+    name = "docs_avro_library_subject",
+    target = "//tools/docs/avro/fixture:shop_avro",
+)
+
 real_docs_subject(
     name = "docs_ruby_library_subject",
     target = "//tools/docs/ruby/fixture:geometry",
@@ -2054,6 +2059,7 @@ real_docs_tests(
         ":docs_fsharp_ledger_subject",
         ":docs_fsharp_xmldoc_off_subject",
         ":docs_astro_library_subject",
+        ":docs_avro_library_subject",
         ":docs_go_library_subject",
         ":docs_java_library_subject",
         ":docs_javascript_library_subject",

quality/testdata/real_aspect_tests.bzl

@@ -575,6 +575,17 @@ aspect_field field_count=5
 aspect_field has_subject=True
 aspect_field subject_label=//quality/testdata:docs_astro_library_subject
 aspect_field transitive_count=0
+subject //quality/testdata:docs_avro_library_subject
+field docs_shard_count=1
+field docs_shards=tools/docs/avro/fixture/_dx_docs/shop_avro.ir.pb
+field dx_docs_sites=(none)
+field dx_results=shop_avro.ir.pb
+field label=//tools/docs/avro/fixture:shop_avro
+aspect_field aspect_seen=True
+aspect_field field_count=5
+aspect_field has_subject=True
+aspect_field subject_label=//quality/testdata:docs_avro_library_subject
+aspect_field transitive_count=0
 subject //quality/testdata:docs_csharp_ledger_subject
 field docs_shard_count=1
 field docs_shards=tools/docs/dotnet/fixture/csharp/_dx_docs/ledger.ir.pb

tools/docs/aspect.bzl

@@ -23,6 +23,8 @@ load("//libs/exec_os:defs.bzl", "ExecOsInfo")
 load("//libs/starlark:resources.bzl", "dx_heavy_resources")
 load("//tools/docs/ir:shard.bzl", "DocsShardInfo", "repo_relative")
 load("//tools/docs/cc:defs.bzl", "cc_shard_action", "doxygen_available")
+load("//avro/rules:providers.bzl", _AvroInfo = "AvroInfo")
+load("//tools/docs/avro:defs.bzl", "avro_shard_action")
 load("//tools/docs/components:shard.bzl", "COMPONENT_LANGUAGES", "component_config", "component_shard_action")
 load("//tools/docs/dart:defs.bzl", "dart_shard_action")
 load("//tools/docs/d:defs.bzl", "D_TOOLCHAIN", "d_shard_action")
@@ -160,6 +162,14 @@ def _powershell_docs_shard(target, ctx):
     pwsh = ctx.attr._pwsh[platform_common.ToolchainInfo]
     return powershell_shard_action(ctx, sources, manifests[0] if manifests else None, package, "", "_dx_docs/" + ctx.label.name, pwsh, ctx.file._pwsh_script, ctx.executable._pwsh_extract).shard
 
+def _avro_docs_shard(target, ctx):
+    if _AvroInfo not in target or DocsShardInfo in target:
+        return None
+    info = target[_AvroInfo]
+    if not info.direct_srcs:
+        return None
+    return avro_shard_action(ctx, info.direct_srcs, info.transitive_srcs, ctx.attr._avro_tools[DefaultInfo].files_to_run, ctx.label.name, "_dx_docs/" + ctx.label.name, ctx.executable._avro_extract).shard
+
 def _ruby_docs_shard(target, ctx):
     if _RubyFilesInfo not in target or DocsShardInfo in target:
         return None
@@ -411,6 +421,9 @@ def _real_docs_aspect_impl(target, ctx):
     pwsh_shard = _powershell_docs_shard(target, ctx)
     if pwsh_shard:
         parts.append({"dx_results": [pwsh_shard], "dx_docs_shards": [pwsh_shard]})
+    avro_shard = _avro_docs_shard(target, ctx)
+    if avro_shard:
+        parts.append({"dx_results": [avro_shard], "dx_docs_shards": [avro_shard]})
     ruby_shard = _ruby_docs_shard(target, ctx)
     if ruby_shard:
         parts.append({"dx_results": [ruby_shard], "dx_docs_shards": [ruby_shard]})
@@ -451,9 +464,19 @@ def _real_docs_aspect_impl(target, ctx):
 
 real_docs_aspect = aspect(
     implementation = _real_docs_aspect_impl,
-    doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Haskell, Dart, C#, F#, PowerShell, Perl, Protobuf, Ruby, Julia, OCaml, Fortran, PHP, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The Protobuf branch runs the protoc of the proto toolchain with --include_source_info over a proto_library's own .proto srcs and the DocIr extractor (//tools/docs/protobuf/extract:protobuf_extract). The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.",
+    doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Haskell, Dart, C#, F#, PowerShell, Perl, Protobuf, Ruby, Julia, OCaml, Fortran, PHP, Avro, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The Protobuf branch runs the protoc of the proto toolchain with --include_source_info over a proto_library's own .proto srcs and the DocIr extractor (//tools/docs/protobuf/extract:protobuf_extract). The Avro branch reads the `doc` attributes of an avro_library's own .avsc srcs and of the schemas the pinned avro-tools (//config:tool_avro) writes for its .avdl srcs with idl2schemata, with the DocIr extractor (//tools/docs/avro/extract:avro_extract). The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.",
     attrs = {
         "_astro_meta": _component_meta_attr("astro"),
+        "_avro_extract": attr.label(
+            cfg = "exec",
+            default = Label("//tools/docs/avro/extract:avro_extract"),
+            executable = True,
+        ),
+        "_avro_tools": attr.label(
+            cfg = "exec",
+            default = Label("//config:tool_avro"),
+            executable = True,
+        ),
         "_cc_extract": attr.label(
             cfg = "exec",
             default = Label("//tools/docs/cc/extract:cc_extract"),

tools/docs/avro/BUILD.bazel

@@ -0,0 +1,32 @@
+load("//quality:fixtures.bzl", "real_source_target")
+load("//rust/rules:defs.bzl", "rust_test")
+load("//tools/visibility:visibility.bzl", "VIS_TOOLS", "VIS_TOOLS_PUBLIC")
+
+package(default_visibility = VIS_TOOLS_PUBLIC)
+
+exports_files(
+    ["defs.bzl"],
+    visibility = VIS_TOOLS,
+)
+
+rust_test(
+    name = "shard_test",
+    srcs = ["shard_test.rs"],
+    crate_name = "shard_test",
+    crate_root = "shard_test.rs",
+    data = ["//tools/docs/avro/fixture:shop_docs"],
+    env = {"DX_FIXTURE_SHARD": "$(rootpath //tools/docs/avro/fixture:shop_docs)"},
+    deps = [
+        "//tools/docs/ir/ir:documentation_ir",
+        "//tools/testing:dx_testing",
+    ],
+)
+
+real_source_target(
+    name = "corpus_starlark",
+    starlark_srcs = [
+        "BUILD.bazel",
+        "defs.bzl",
+    ],
+    tags = ["corpus"],
+)

tools/docs/avro/defs.bzl

@@ -0,0 +1,113 @@
+"""Avro schema reference shards read from the `doc` attribute of the schemas."""
+
+load("//avro/rules:providers.bzl", "AvroInfo")
+load("//tools/docs/ir:shard.bzl", "DocsShardInfo", "repo_relative")
+
+def avro_shard_action(ctx, srcs, imports, avro_tools, package, stem, extract_tool):
+    """Runs avro-tools on each .avdl and the extractor over every schema of one avro_library.
+
+    Args:
+      ctx: The rule or aspect context.
+      srcs: The .avsc and .avdl files the library lists in its own srcs.
+      imports: Every schema the library reaches, so .avdl imports resolve.
+      avro_tools: The FilesToRunProvider of the pinned avro-tools (//config:tool_avro).
+      package: Package segment of the shard's symbol ids.
+      stem: Output name prefix, relative to the target's package.
+      extract_tool: The DocIr extractor executable.
+
+    Returns:
+      A struct with the DocIr `shard`.
+    """
+    if not srcs:
+        fail("avro_doc_shard " + str(ctx.label) + ": avro_library lists no schema srcs")
+    extract = ctx.actions.args()
+    extract.add("--package", package)
+    inputs = []
+    for src in sorted(srcs, key = lambda f: f.path):
+        source = repo_relative(src.short_path)
+        if not src.is_source:
+            extract.add("--generated", source)
+        if src.extension == "avdl":
+            # idl2schemata writes one .avsc per named type; messages stay in the .avdl.
+            tree = ctx.actions.declare_directory("%s.%s.schemata" % (stem, src.basename))
+            args = ctx.actions.args()
+            args.add_all(["idl2schemata", src, tree.path])
+            ctx.actions.run(
+                executable = avro_tools,
+                arguments = [args],
+                inputs = depset([src], transitive = [imports]),
+                outputs = [tree],
+                mnemonic = "DxDocsAvroIdl",
+                toolchain = None,
+                progress_message = "Reading Avro IDL docs of " + src.short_path,
+            )
+            inputs.append(tree)
+            extract.add("--schema-dir", "%s=%s" % (tree.path, source))
+        else:
+            inputs.append(src)
+            extract.add("--schema", "%s=%s" % (src.path, source))
+    shard = ctx.actions.declare_file(stem + ".ir.pb")
+    extract.add("--out", shard)
+    ctx.actions.run(
+        executable = extract_tool,
+        arguments = [extract],
+        inputs = inputs,
+        outputs = [shard],
+        mnemonic = "DxDocsAvroExtract",
+        toolchain = None,
+        progress_message = "Extracting Avro docs for %{label}",
+    )
+    return struct(shard = shard)
+
+def _avro_doc_shard_impl(ctx):
+    info = ctx.attr.avro[AvroInfo]
+    package = ctx.attr.package or ctx.attr.avro.label.name
+    action = avro_shard_action(
+        ctx,
+        info.direct_srcs,
+        info.transitive_srcs,
+        ctx.attr._avro_tools[DefaultInfo].files_to_run,
+        package,
+        ctx.label.name,
+        ctx.executable._extract,
+    )
+    return [
+        DefaultInfo(files = depset([action.shard])),
+        DocsShardInfo(language = "avro", package = package, shard = action.shard),
+    ]
+
+docs_avro_shard = rule(
+    doc = "Converts one avro_library's records, enums, fixed types and protocol messages into a DocIr shard.",
+    implementation = _avro_doc_shard_impl,
+    attrs = {
+        "avro": attr.label(
+            mandatory = True,
+            providers = [AvroInfo],
+            doc = "The documented avro_library; its own schemas are documented.",
+        ),
+        "package": attr.string(doc = "Package segment of the symbol ids; defaults to the name of avro."),
+        "_avro_tools": attr.label(default = Label("//config:tool_avro"), executable = True, cfg = "exec"),
+        "_extract": attr.label(
+            cfg = "exec",
+            default = Label("//tools/docs/avro/extract:avro_extract"),
+            executable = True,
+        ),
+    },
+    provides = [DocsShardInfo],
+)
+
+def avro_doc_shard(name, avro, package = "", **kwargs):
+    """Declares the DocIr shard of one avro_library.
+
+    Args:
+      name: Name of the shard target.
+      avro: The documented avro_library.
+      package: Package segment of every symbol id; defaults to the name of avro.
+      **kwargs: Common attributes such as tags and visibility.
+    """
+    docs_avro_shard(
+        name = name,
+        avro = avro,
+        package = package,
+        **kwargs
+    )