diff --git a/BUILD.bazel b/BUILD.bazel index 0e53494108..083dec898d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -76,6 +76,7 @@ load("//quality:json_schema.bzl", "json_schema", "json_schemas") # gazelle:exclude tools/docs/perl/fixture # gazelle:exclude tools/docs/php/fixture # gazelle:exclude tools/docs/protobuf/fixture +# gazelle:exclude tools/docs/avro/fixture # gazelle:exclude tools/docs/go/godoc # gazelle:exclude cli/depcheck/testdata # gazelle:exclude go/tools/analysis diff --git a/MODULE.bazel b/MODULE.bazel index 5f193c6730..c24fd1ff9e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1837,6 +1837,7 @@ crate.from_cargo( "//tools/docs/go/extract:Cargo.toml", "//tools/docs/php/extract:Cargo.toml", "//tools/docs/ruby/extract:Cargo.toml", + "//tools/docs/avro/extract:Cargo.toml", "//tools/docs/dart/extract:Cargo.toml", "//tools/docs/julia/extract:Cargo.toml", "//tools/docs/ocaml/extract:Cargo.toml", diff --git a/cargo-bazel-lock.json b/cargo-bazel-lock.json index 8767aeb7d0..2b66c36d7d 100644 --- a/cargo-bazel-lock.json +++ b/cargo-bazel-lock.json @@ -1,5 +1,5 @@ { - "checksum": "c978dbd2ef70f7bebc134112a424e06fc4aae969ec64256d80463c455ca43c44", + "checksum": "72951dade25a7dda429098a64a357be578a2a81401ca32287b43d5d4d4d18989", "conditions": { "aarch64-apple-darwin": [ "aarch64-apple-darwin" @@ -1490,6 +1490,33 @@ ], "version": "1.5.1" }, + "avro_extract 0.0.0": { + "common_attrs": { + "deps": [ + { + "id": "serde_json 1.0.151", + "target": "serde_json" + } + ], + "edition": "2024" + }, + "library_target_name": "dx_docs_avro_extract", + "license": "Apache-2.0", + "license_ids": [ + "Apache-2.0" + ], + "name": "avro_extract", + "repository": null, + "targets": [ + { + "Library": { + "crate_name": "dx_docs_avro_extract", + "crate_root": "src/lib.rs" + } + } + ], + "version": "0.0.0" + }, "base16ct 0.2.0": { "common_attrs": { "crate_features": [ @@ -28771,6 +28798,7 @@ ], "workspace_members": { "appimage_packer 0.0.0": "appimage/appimage_packer", + "avro_extract 0.0.0": "tools/docs/avro/avro_extract", "bazelisk_wrapper 0.0.0": "tools/bazelisk_wrapper", "cc_context 0.0.0": "cc/cc_context", "cc_extract 0.0.0": "tools/docs/cc/cc_extract", diff --git a/cc/context/Cargo.lock b/cc/context/Cargo.lock index 6855a86a1a..89e1e5102c 100644 --- a/cc/context/Cargo.lock +++ b/cc/context/Cargo.lock @@ -269,6 +269,13 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "avro_extract" +version = "0.0.0" +dependencies = [ + "serde_json", +] + [[package]] name = "base16ct" version = "0.2.0" diff --git a/docs/cli/commands/docs.md b/docs/cli/commands/docs.md index 1eb6c9fa17..dfe1cc47ee 100644 --- a/docs/cli/commands/docs.md +++ b/docs/cli/commands/docs.md @@ -107,6 +107,7 @@ tools/dx docs //lib:site | C# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `csharp_doc_shard` | `library`, `package` | | F# | `@rules_dx//tools/docs/dotnet:defs.bzl` | `fsharp_doc_shard` | `library`, `package` | | Protobuf | `@rules_dx//tools/docs/protobuf:defs.bzl` | `protobuf_doc_shard` | `proto`, `package` | +| Avro | `@rules_dx//tools/docs/avro:defs.bzl` | `avro_doc_shard` | `avro`, `package` | Rust, Go, Python, C, C++, CUDA, Fortran, Java, Kotlin, Scala, D, Haskell, Dart, C#, F# and Perl libraries in scope that have no shard get one automatically. A C# library needs `nullable = "disable"` and @@ -125,4 +126,12 @@ service an interface and an rpc a method. The comment above an item, or after it on the same line, is its doc text. A `map` field shows as a map and its generated entry message is left out. Streaming rpcs say `stream`. +An `avro_library` in scope that has no shard gets one automatically. Avro +shards read the `doc` attribute straight from the schemas: a record is a class, +a field a field, an enum an enum and its symbols variants, a fixed type a type +alias and a protocol message a method. Ids start with the schema namespace. An +`.avdl` file is first turned into schemas by the pinned avro-tools +(`idl2schemata`) on the hermetic JDK. Avro enum symbols carry no doc of their +own, so they are marked incomplete. + A source error fails the build and writes no pages. Bazel's exit code is kept. diff --git a/docs/support-matrix.md b/docs/support-matrix.md index cb3d1dd22d..c8c159276c 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -80,7 +80,7 @@ Each cell names the tools `dx lint`, `dx format`, or `dx typecheck` runs, and th | Name | Lint | Format | Typecheck | API docs | Coverage | Dead code | | --- | --- | --- | --- | --- | --- | --- | | Astro | `eslint`, `typos` | `prettier` | `tsc` | `astro-compiler` | `jest` | `eslint @typescript-eslint/no-unused-vars` | -| Avro | `avro_tools`, `typos` | `prettier` | `avro_tools` | no | n/a | none | +| Avro | `avro_tools`, `typos` | `prettier` | `avro_tools` | `avro-tools` | n/a | none | | C and C++ | `clang_tidy`, `typos` | `clang_format` | `clang_syntax` | `doxygen` | `gcov`, `llvm-cov` | `clang_tidy misc-unused-parameters, misc-unused-using-decls` | | C# | `roslyn`, `typos` | `csharpier` | `csc` | `csharp doc driver` | `coverlet` | `roslyn IDE0051, IDE0052, IDE0060, IDE0005` | | CUDA | `clang_tidy`, `typos` | `clang_format` | `nvcc_syntax` | `doxygen` | no: cuda_test has no dx coverage wiring | none | diff --git a/families/generation/README.md b/families/generation/README.md index d662e448f6..73193447b3 100644 --- a/families/generation/README.md +++ b/families/generation/README.md @@ -12,5 +12,11 @@ A workspace that only uses the code generation rules needs nothing more than `protobuf_doc_shard` in `@rules_dx//tools/docs/protobuf:defs.bzl` documents one `proto_library`. It runs the protoc of the registered proto toolchain with `--include_source_info` and reads the comments above each message, field, enum, -enum value, service and rpc from the descriptor set. Avro schemas are not -covered; their `doc` fields are tracked in #9936. +enum value, service and rpc from the descriptor set. +## Avro docs + +`avro_doc_shard` in `@rules_dx//tools/docs/avro:defs.bzl` documents one +`avro_library`. The `doc` attribute of every record, field, enum, fixed type and +protocol message is read from the `.avsc` files directly. An `.avdl` file is +first converted with `idl2schemata` of the pinned avro-tools (`//config:tool_avro`), +which runs on the hermetic JDK, and then read the same way. diff --git a/modules/rust.bzl b/modules/rust.bzl index 419bd8e8c3..3fa4a0f064 100644 --- a/modules/rust.bzl +++ b/modules/rust.bzl @@ -60,6 +60,7 @@ RUST_CRATE_MANIFESTS = [ "//tools/docs/go/extract:Cargo.toml", "//tools/docs/php/extract:Cargo.toml", "//tools/docs/ruby/extract:Cargo.toml", + "//tools/docs/avro/extract:Cargo.toml", "//tools/docs/dart/extract:Cargo.toml", "//tools/docs/julia/extract:Cargo.toml", "//tools/docs/ocaml/extract:Cargo.toml", diff --git a/quality/testdata/BUILD.bazel b/quality/testdata/BUILD.bazel index 204041bf92..0750e879c2 100644 --- a/quality/testdata/BUILD.bazel +++ b/quality/testdata/BUILD.bazel @@ -2008,6 +2008,11 @@ real_docs_subject( target = "//tools/docs/protobuf/fixture:shop_proto", ) +real_docs_subject( + name = "docs_avro_library_subject", + target = "//tools/docs/avro/fixture:shop_avro", +) + real_docs_subject( name = "docs_ruby_library_subject", target = "//tools/docs/ruby/fixture:geometry", @@ -2054,6 +2059,7 @@ real_docs_tests( ":docs_fsharp_ledger_subject", ":docs_fsharp_xmldoc_off_subject", ":docs_astro_library_subject", + ":docs_avro_library_subject", ":docs_go_library_subject", ":docs_java_library_subject", ":docs_javascript_library_subject", diff --git a/quality/testdata/real_aspect_tests.bzl b/quality/testdata/real_aspect_tests.bzl index 327e53ef0f..c1dd077e4c 100644 --- a/quality/testdata/real_aspect_tests.bzl +++ b/quality/testdata/real_aspect_tests.bzl @@ -575,6 +575,17 @@ aspect_field field_count=5 aspect_field has_subject=True aspect_field subject_label=//quality/testdata:docs_astro_library_subject aspect_field transitive_count=0 +subject //quality/testdata:docs_avro_library_subject +field docs_shard_count=1 +field docs_shards=tools/docs/avro/fixture/_dx_docs/shop_avro.ir.pb +field dx_docs_sites=(none) +field dx_results=shop_avro.ir.pb +field label=//tools/docs/avro/fixture:shop_avro +aspect_field aspect_seen=True +aspect_field field_count=5 +aspect_field has_subject=True +aspect_field subject_label=//quality/testdata:docs_avro_library_subject +aspect_field transitive_count=0 subject //quality/testdata:docs_csharp_ledger_subject field docs_shard_count=1 field docs_shards=tools/docs/dotnet/fixture/csharp/_dx_docs/ledger.ir.pb diff --git a/tools/docs/aspect.bzl b/tools/docs/aspect.bzl index 585a8cd355..b76c932ff1 100644 --- a/tools/docs/aspect.bzl +++ b/tools/docs/aspect.bzl @@ -23,6 +23,8 @@ load("//libs/exec_os:defs.bzl", "ExecOsInfo") load("//libs/starlark:resources.bzl", "dx_heavy_resources") load("//tools/docs/ir:shard.bzl", "DocsShardInfo", "repo_relative") load("//tools/docs/cc:defs.bzl", "cc_shard_action", "doxygen_available") +load("//avro/rules:providers.bzl", _AvroInfo = "AvroInfo") +load("//tools/docs/avro:defs.bzl", "avro_shard_action") load("//tools/docs/components:shard.bzl", "COMPONENT_LANGUAGES", "component_config", "component_shard_action") load("//tools/docs/dart:defs.bzl", "dart_shard_action") load("//tools/docs/d:defs.bzl", "D_TOOLCHAIN", "d_shard_action") @@ -160,6 +162,14 @@ def _powershell_docs_shard(target, ctx): pwsh = ctx.attr._pwsh[platform_common.ToolchainInfo] return powershell_shard_action(ctx, sources, manifests[0] if manifests else None, package, "", "_dx_docs/" + ctx.label.name, pwsh, ctx.file._pwsh_script, ctx.executable._pwsh_extract).shard +def _avro_docs_shard(target, ctx): + if _AvroInfo not in target or DocsShardInfo in target: + return None + info = target[_AvroInfo] + if not info.direct_srcs: + return None + return avro_shard_action(ctx, info.direct_srcs, info.transitive_srcs, ctx.attr._avro_tools[DefaultInfo].files_to_run, ctx.label.name, "_dx_docs/" + ctx.label.name, ctx.executable._avro_extract).shard + def _ruby_docs_shard(target, ctx): if _RubyFilesInfo not in target or DocsShardInfo in target: return None @@ -411,6 +421,9 @@ def _real_docs_aspect_impl(target, ctx): pwsh_shard = _powershell_docs_shard(target, ctx) if pwsh_shard: parts.append({"dx_results": [pwsh_shard], "dx_docs_shards": [pwsh_shard]}) + avro_shard = _avro_docs_shard(target, ctx) + if avro_shard: + parts.append({"dx_results": [avro_shard], "dx_docs_shards": [avro_shard]}) ruby_shard = _ruby_docs_shard(target, ctx) if ruby_shard: parts.append({"dx_results": [ruby_shard], "dx_docs_shards": [ruby_shard]}) @@ -451,9 +464,19 @@ def _real_docs_aspect_impl(target, ctx): real_docs_aspect = aspect( implementation = _real_docs_aspect_impl, - doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Haskell, Dart, C#, F#, PowerShell, Perl, Protobuf, Ruby, Julia, OCaml, Fortran, PHP, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The Protobuf branch runs the protoc of the proto toolchain with --include_source_info over a proto_library's own .proto srcs and the DocIr extractor (//tools/docs/protobuf/extract:protobuf_extract). The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.", + doc = "Puts the docs sites, shards and API trees of a target in dx_results and a manifest naming each site tree in dx_docs_sites. Rust, Go, Python, JVM, C, C++, CUDA, D, Haskell, Dart, C#, F#, PowerShell, Perl, Protobuf, Ruby, Julia, OCaml, Fortran, PHP, Avro, TypeScript, JavaScript, Vue, Svelte, Astro and MDX library shards. The PHP branch runs the pinned phpDocumentor (@dx_phpdocumentor) on the pinned PHP and the DocIr extractor (//tools/docs/php/extract:php_extract) over a library's own .php srcs. The Perl branch runs the pinned perl over the POD driver (//tools/docs/perl:pod_dump.pl) and the DocIr extractor (//tools/docs/perl/extract:perl_extract) over a library's own .pm and .pl srcs. The Ruby branch runs the pinned YARD (@dx_yard) and the DocIr extractor (//tools/docs/ruby/extract:ruby_extract) over a library's own .rb srcs go in dx_docs_shards. The Protobuf branch runs the protoc of the proto toolchain with --include_source_info over a proto_library's own .proto srcs and the DocIr extractor (//tools/docs/protobuf/extract:protobuf_extract). The Avro branch reads the `doc` attributes of an avro_library's own .avsc srcs and of the schemas the pinned avro-tools (//config:tool_avro) writes for its .avdl srcs with idl2schemata, with the DocIr extractor (//tools/docs/avro/extract:avro_extract). The TypeScript branch runs the pinned TypeDoc (//quality/tools/javascript/bin:typedoc) and the DocIr extractor (//tools/docs/typescript/extract:typescript_extract) over each non-test ts_project. The JavaScript branch runs the same TypeDoc over a js_library of .js, .mjs, .cjs or .jsx srcs that has exactly one tsconfig.json (with allowJs) in its package. The Fortran branch runs the pinned FORD (//quality/tools/python:ford_dump) and the DocIr extractor (//tools/docs/fortran/extract:fortran_extract) over a library's own Fortran srcs. Rust shards also put their rustdoc JSON and source manifest in rustdoc_json and srcs_manifest.", attrs = { "_astro_meta": _component_meta_attr("astro"), + "_avro_extract": attr.label( + cfg = "exec", + default = Label("//tools/docs/avro/extract:avro_extract"), + executable = True, + ), + "_avro_tools": attr.label( + cfg = "exec", + default = Label("//config:tool_avro"), + executable = True, + ), "_cc_extract": attr.label( cfg = "exec", default = Label("//tools/docs/cc/extract:cc_extract"), diff --git a/tools/docs/avro/BUILD.bazel b/tools/docs/avro/BUILD.bazel new file mode 100644 index 0000000000..3b20bc8b11 --- /dev/null +++ b/tools/docs/avro/BUILD.bazel @@ -0,0 +1,32 @@ +load("//quality:fixtures.bzl", "real_source_target") +load("//rust/rules:defs.bzl", "rust_test") +load("//tools/visibility:visibility.bzl", "VIS_TOOLS", "VIS_TOOLS_PUBLIC") + +package(default_visibility = VIS_TOOLS_PUBLIC) + +exports_files( + ["defs.bzl"], + visibility = VIS_TOOLS, +) + +rust_test( + name = "shard_test", + srcs = ["shard_test.rs"], + crate_name = "shard_test", + crate_root = "shard_test.rs", + data = ["//tools/docs/avro/fixture:shop_docs"], + env = {"DX_FIXTURE_SHARD": "$(rootpath //tools/docs/avro/fixture:shop_docs)"}, + deps = [ + "//tools/docs/ir/ir:documentation_ir", + "//tools/testing:dx_testing", + ], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = [ + "BUILD.bazel", + "defs.bzl", + ], + tags = ["corpus"], +) diff --git a/tools/docs/avro/defs.bzl b/tools/docs/avro/defs.bzl new file mode 100644 index 0000000000..739be6e1a3 --- /dev/null +++ b/tools/docs/avro/defs.bzl @@ -0,0 +1,113 @@ +"""Avro schema reference shards read from the `doc` attribute of the schemas.""" + +load("//avro/rules:providers.bzl", "AvroInfo") +load("//tools/docs/ir:shard.bzl", "DocsShardInfo", "repo_relative") + +def avro_shard_action(ctx, srcs, imports, avro_tools, package, stem, extract_tool): + """Runs avro-tools on each .avdl and the extractor over every schema of one avro_library. + + Args: + ctx: The rule or aspect context. + srcs: The .avsc and .avdl files the library lists in its own srcs. + imports: Every schema the library reaches, so .avdl imports resolve. + avro_tools: The FilesToRunProvider of the pinned avro-tools (//config:tool_avro). + package: Package segment of the shard's symbol ids. + stem: Output name prefix, relative to the target's package. + extract_tool: The DocIr extractor executable. + + Returns: + A struct with the DocIr `shard`. + """ + if not srcs: + fail("avro_doc_shard " + str(ctx.label) + ": avro_library lists no schema srcs") + extract = ctx.actions.args() + extract.add("--package", package) + inputs = [] + for src in sorted(srcs, key = lambda f: f.path): + source = repo_relative(src.short_path) + if not src.is_source: + extract.add("--generated", source) + if src.extension == "avdl": + # idl2schemata writes one .avsc per named type; messages stay in the .avdl. + tree = ctx.actions.declare_directory("%s.%s.schemata" % (stem, src.basename)) + args = ctx.actions.args() + args.add_all(["idl2schemata", src, tree.path]) + ctx.actions.run( + executable = avro_tools, + arguments = [args], + inputs = depset([src], transitive = [imports]), + outputs = [tree], + mnemonic = "DxDocsAvroIdl", + toolchain = None, + progress_message = "Reading Avro IDL docs of " + src.short_path, + ) + inputs.append(tree) + extract.add("--schema-dir", "%s=%s" % (tree.path, source)) + else: + inputs.append(src) + extract.add("--schema", "%s=%s" % (src.path, source)) + shard = ctx.actions.declare_file(stem + ".ir.pb") + extract.add("--out", shard) + ctx.actions.run( + executable = extract_tool, + arguments = [extract], + inputs = inputs, + outputs = [shard], + mnemonic = "DxDocsAvroExtract", + toolchain = None, + progress_message = "Extracting Avro docs for %{label}", + ) + return struct(shard = shard) + +def _avro_doc_shard_impl(ctx): + info = ctx.attr.avro[AvroInfo] + package = ctx.attr.package or ctx.attr.avro.label.name + action = avro_shard_action( + ctx, + info.direct_srcs, + info.transitive_srcs, + ctx.attr._avro_tools[DefaultInfo].files_to_run, + package, + ctx.label.name, + ctx.executable._extract, + ) + return [ + DefaultInfo(files = depset([action.shard])), + DocsShardInfo(language = "avro", package = package, shard = action.shard), + ] + +docs_avro_shard = rule( + doc = "Converts one avro_library's records, enums, fixed types and protocol messages into a DocIr shard.", + implementation = _avro_doc_shard_impl, + attrs = { + "avro": attr.label( + mandatory = True, + providers = [AvroInfo], + doc = "The documented avro_library; its own schemas are documented.", + ), + "package": attr.string(doc = "Package segment of the symbol ids; defaults to the name of avro."), + "_avro_tools": attr.label(default = Label("//config:tool_avro"), executable = True, cfg = "exec"), + "_extract": attr.label( + cfg = "exec", + default = Label("//tools/docs/avro/extract:avro_extract"), + executable = True, + ), + }, + provides = [DocsShardInfo], +) + +def avro_doc_shard(name, avro, package = "", **kwargs): + """Declares the DocIr shard of one avro_library. + + Args: + name: Name of the shard target. + avro: The documented avro_library. + package: Package segment of every symbol id; defaults to the name of avro. + **kwargs: Common attributes such as tags and visibility. + """ + docs_avro_shard( + name = name, + avro = avro, + package = package, + **kwargs + ) diff --git a/tools/docs/avro/extract/BUILD.bazel b/tools/docs/avro/extract/BUILD.bazel new file mode 100644 index 0000000000..a5cdaef4fe --- /dev/null +++ b/tools/docs/avro/extract/BUILD.bazel @@ -0,0 +1,85 @@ +load("@crates//:crates.bzl", "aliases", "all_crate_deps") +load("//quality:fixtures.bzl", "real_source_target") +load("//rust/rules:defs.bzl", "rust_binary", "rust_library", "rust_test") +load("//tools/visibility:visibility.bzl", "VIS_TOOLS", "VIS_TOOLS_PUBLIC") + +package(default_visibility = VIS_TOOLS_PUBLIC) + +exports_files( + [ + "Cargo.toml", + "src/lib.rs", + "src/lib_tests.rs", + "src/main.rs", + ], + visibility = VIS_TOOLS, +) + +rust_library( + name = "dx_docs_avro_extract", + srcs = ["src/lib.rs"], + aliases = aliases( + package_name = "tools/docs/avro/avro_extract", + normal = True, + ), + crate_name = "dx_docs_avro_extract", + crate_root = "src/lib.rs", + deps = all_crate_deps( + package_name = "tools/docs/avro/avro_extract", + normal = True, + ) + [ + "//tools/docs/ir/ir:documentation_ir", + "@crates//:serde_json", + ], +) + +rust_binary( + name = "avro_extract", + srcs = ["src/main.rs"], + aliases = aliases( + package_name = "tools/docs/avro/avro_extract", + normal = True, + ), + crate_name = "avro_extract", + crate_root = "src/main.rs", + deps = all_crate_deps( + package_name = "tools/docs/avro/avro_extract", + normal = True, + ) + [":dx_docs_avro_extract"], +) + +rust_test( + name = "dx_docs_avro_extract_test", + srcs = [ + "src/lib.rs", + "src/lib_tests.rs", + ], + aliases = aliases( + package_name = "tools/docs/avro/avro_extract", + normal = True, + normal_dev = True, + ), + coverage_lib = ":dx_docs_avro_extract", + crate_name = "dx_docs_avro_extract", + crate_root = "src/lib.rs", + deps = all_crate_deps( + package_name = "tools/docs/avro/avro_extract", + normal = True, + normal_dev = True, + ) + [ + "//tools/docs/ir/ir:documentation_ir", + "@crates//:serde_json", + ], +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) + +real_source_target( + name = "corpus_toml", + tags = ["corpus"], + toml_srcs = ["Cargo.toml"], +) diff --git a/tools/docs/avro/extract/Cargo.toml b/tools/docs/avro/extract/Cargo.toml new file mode 100644 index 0000000000..faa142f39c --- /dev/null +++ b/tools/docs/avro/extract/Cargo.toml @@ -0,0 +1,16 @@ +[package] +license = "Apache-2.0" +name = "avro_extract" +version = "0.0.0" +edition = "2024" + +[dependencies] +serde_json = "1" + +[lib] +name = "dx_docs_avro_extract" +path = "src/lib.rs" + +[[bin]] +name = "avro_extract" +path = "src/main.rs" diff --git a/tools/docs/avro/extract/src/lib.rs b/tools/docs/avro/extract/src/lib.rs new file mode 100644 index 0000000000..e67980570d --- /dev/null +++ b/tools/docs/avro/extract/src/lib.rs @@ -0,0 +1,538 @@ +//! Converts Avro schemas (.avsc, .avpr, or the schemas avro-tools writes for .avdl) into one +//! validated DocIr shard, reading the `doc` attribute of every named type, field and message. + +#![cfg_attr( + not(test), + deny( + clippy::expect_used, + clippy::unwrap_used, + clippy::unreachable, + clippy::todo + ) +)] + +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; + +use documentation_ir::proto::{ + DocIr, Extension, Param, Relations, Returns, SourceRef, Symbol, SymbolKind, Visibility, +}; +use serde_json::Value; + +const LANGUAGE: &str = "avro"; +const KEY_INCOMPLETE: &str = "avro.incomplete"; + +/// The inputs of one conversion. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Options { + pub package: String, + pub out: String, + /// Schema file to workspace-relative path of the source it documents. + pub schemas: Vec<(String, String)>, + /// Directory of schemas avro-tools wrote, to workspace-relative path of the .avdl. + pub schema_dirs: Vec<(String, String)>, + /// Workspace-relative paths of sources that are build output. + pub generated: Vec, +} + +/// The usage line shown for a rejected invocation. +pub fn usage() -> &'static str { + "usage: avro_extract --package NAME --out FILE (--schema FILE=SRC | --schema-dir DIR=SRC)... [--generated SRC]..." +} + +fn pair(flag: &str, value: &str) -> Result<(String, String), String> { + match value.split_once('=') { + Some((left, right)) if !left.is_empty() && !right.is_empty() => { + Ok((left.to_owned(), right.to_owned())) + } + _ => Err(format!("{flag} '{value}' is not INPUT=SRC")), + } +} + +/// Parses argv into inputs, or returns the rejection text. +pub fn parse(args: &[String]) -> Result { + let mut options = Options::default(); + let mut index = 0; + while index < args.len() { + let flag = args[index].as_str(); + let Some(value) = args.get(index + 1) else { + return Err(format!("{flag} needs a value")); + }; + if value.is_empty() { + return Err(format!("{flag} must not be empty")); + } + match flag { + "--schema" => options.schemas.push(pair(flag, value)?), + "--schema-dir" => options.schema_dirs.push(pair(flag, value)?), + "--generated" => options.generated.push(value.clone()), + "--package" | "--out" => { + let slot = if flag == "--package" { + &mut options.package + } else { + &mut options.out + }; + if !slot.is_empty() { + return Err(format!("{flag} given more than once")); + } + *slot = value.clone(); + } + _ => return Err(format!("unknown argument '{flag}'")), + } + index += 2; + } + for (flag, value) in [("--package", &options.package), ("--out", &options.out)] { + if value.is_empty() { + return Err(format!("{flag} is required")); + } + } + if options.schemas.is_empty() && options.schema_dirs.is_empty() { + return Err("at least one --schema or --schema-dir is required".to_owned()); + } + Ok(options) +} + +/// Reads, converts, validates and writes one shard, returning the failure text on error. +pub fn run(options: &Options) -> Result<(), String> { + let mut inputs: Vec<(String, String, String)> = Vec::new(); + for (file, source) in &options.schemas { + let text = + fs::read_to_string(file).map_err(|error| format!("cannot read {file}: {error}"))?; + inputs.push((text, source.clone(), file.clone())); + } + for (dir, source) in &options.schema_dirs { + let mut files: Vec<_> = fs::read_dir(dir) + .map_err(|error| format!("cannot read {dir}: {error}"))? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "avsc" || ext == "avpr")) + .collect(); + files.sort(); + for path in files { + let text = fs::read_to_string(&path) + .map_err(|error| format!("cannot read {}: {error}", path.display()))?; + inputs.push((text, source.clone(), path.display().to_string())); + } + } + let shard = convert(&inputs, options)?; + let bytes = documentation_ir::encode_shard(&shard) + .map_err(|error| format!("cannot encode IR: {error:?}"))?; + fs::write(Path::new(&options.out), bytes) + .map_err(|error| format!("cannot write {}: {error}", options.out)) +} + +fn str_of<'a>(value: &'a Value, key: &str) -> &'a str { + value.get(key).and_then(Value::as_str).unwrap_or("") +} + +/// Splits a name into its namespace and simple name, inheriting `enclosing` when it has none. +fn split_name(value: &Value, enclosing: &str) -> (String, String) { + let name = str_of(value, "name"); + if let Some((namespace, simple)) = name.rsplit_once('.') { + return (namespace.to_owned(), simple.to_owned()); + } + match value.get("namespace").and_then(Value::as_str) { + Some(namespace) => (namespace.to_owned(), name.to_owned()), + None => (enclosing.to_owned(), name.to_owned()), + } +} + +fn full_name(namespace: &str, simple: &str) -> String { + if namespace.is_empty() { + simple.to_owned() + } else { + format!("{namespace}.{simple}") + } +} + +/// How a type reads in a signature. +fn type_text(value: &Value, enclosing: &str) -> String { + match value { + Value::String(name) => name.clone(), + Value::Array(members) => members + .iter() + .map(|member| type_text(member, enclosing)) + .collect::>() + .join(" | "), + Value::Object(_) => match str_of(value, "type") { + "array" => format!( + "array<{}>", + type_text(value.get("items").unwrap_or(&Value::Null), enclosing) + ), + "map" => format!( + "map", + type_text(value.get("values").unwrap_or(&Value::Null), enclosing) + ), + "record" | "error" | "enum" | "fixed" => { + let (namespace, simple) = split_name(value, enclosing); + full_name(&namespace, &simple) + } + "" => value + .get("type") + .map_or_else(String::new, |inner| type_text(inner, enclosing)), + primitive => primitive.to_owned(), + }, + _ => String::new(), + } +} + +struct Builder<'a> { + package: &'a str, + source: String, + generated: bool, + /// The lines of the schema text when they map to the source, else empty. + lines: Vec, + symbols: BTreeMap, +} + +impl Builder<'_> { + fn id_of(&self, namespace: &str, simple: &str, member: Option<&str>) -> String { + let mut segments: Vec<&str> = namespace.split('.').filter(|s| !s.is_empty()).collect(); + segments.push(simple); + if let Some(member) = member { + segments.push(member); + } + format!( + "{LANGUAGE}:{}:{}", + self.package, + segments + .iter() + .map(|s| documentation_ir::escape_id_segment(s)) + .collect::>() + .join(".") + ) + } + + /// The 1-based line of the first `"name": ""` at or after `from`, or 0. + fn find_line(&self, name: &str, from: u64) -> u64 { + let quoted = format!("\"{name}\""); + let start = from.saturating_sub(1) as usize; + self.lines + .iter() + .enumerate() + .skip(start) + .find(|(_, line)| line.contains("\"name\"") && line.contains("ed)) + .map_or(0, |(index, _)| index as u64 + 1) + } + + #[allow(clippy::too_many_arguments)] + fn push( + &mut self, + id: String, + display: String, + kind: SymbolKind, + signature: String, + doc: &str, + line: u64, + parent: Option, + params: Vec, + returns: Option, + ) { + let doc = doc.trim().to_owned(); + let mut extensions = Vec::new(); + if doc.is_empty() { + extensions.push(Extension { + key: KEY_INCOMPLETE.to_owned(), + value: b"doc".to_vec(), + }); + } + let symbol = Symbol { + id: id.clone(), + kind: kind as i32, + signature_text: signature, + doc_markdown: doc, + params, + returns, + source: Some(SourceRef { + file: self.source.clone(), + line, + generated: self.generated, + }), + visibility: Visibility::Public as i32, + relations: parent.map(|parent| Relations { + member_of: vec![parent], + implements: Vec::new(), + reexported_as: Vec::new(), + }), + extensions, + display_name: display, + ..Symbol::default() + }; + // A type imported by several files appears in each; keep the copy that has docs. + match self.symbols.get(&id) { + Some(existing) if !existing.doc_markdown.is_empty() || symbol.doc_markdown.is_empty() => {} + _ => { + self.symbols.insert(id, symbol); + } + } + } + + fn walk(&mut self, value: &Value, enclosing: &str) { + match value { + Value::Array(members) => { + for member in members { + self.walk(member, enclosing); + } + } + Value::Object(_) => { + if value.get("protocol").is_some() { + self.protocol(value, enclosing); + return; + } + match str_of(value, "type") { + "record" | "error" => self.record(value, enclosing), + "enum" => self.enumeration(value, enclosing), + "fixed" => self.fixed(value, enclosing), + "array" => self.walk(value.get("items").unwrap_or(&Value::Null), enclosing), + "map" => self.walk(value.get("values").unwrap_or(&Value::Null), enclosing), + "" => self.walk(value.get("type").unwrap_or(&Value::Null), enclosing), + _ => {} + } + } + _ => {} + } + } + + fn record(&mut self, value: &Value, enclosing: &str) { + let (namespace, simple) = split_name(value, enclosing); + if simple.is_empty() { + return; + } + let id = self.id_of(&namespace, &simple, None); + let line = self.find_line(&simple, 1); + let keyword = if str_of(value, "type") == "error" { + "error" + } else { + "record" + }; + self.push( + id.clone(), + full_name(&namespace, &simple), + SymbolKind::Class, + format!("{keyword} {simple}"), + str_of(value, "doc"), + line, + None, + Vec::new(), + None, + ); + let fields = value.get("fields").and_then(Value::as_array); + for field in fields.into_iter().flatten() { + let name = str_of(field, "name"); + if name.is_empty() { + continue; + } + let ty = field.get("type").unwrap_or(&Value::Null); + let mut signature = format!("{} {name}", type_text(ty, &namespace)); + if let Some(default) = field.get("default") { + signature.push_str(&format!(" = {default}")); + } + let field_line = if line == 0 { 0 } else { self.find_line(name, line) }; + self.push( + self.id_of(&namespace, &simple, Some(name)), + format!("{}.{name}", full_name(&namespace, &simple)), + SymbolKind::Field, + signature, + str_of(field, "doc"), + field_line, + Some(id.clone()), + Vec::new(), + None, + ); + self.walk(ty, &namespace); + } + } + + fn enumeration(&mut self, value: &Value, enclosing: &str) { + let (namespace, simple) = split_name(value, enclosing); + if simple.is_empty() { + return; + } + let id = self.id_of(&namespace, &simple, None); + let line = self.find_line(&simple, 1); + self.push( + id.clone(), + full_name(&namespace, &simple), + SymbolKind::Enum, + format!("enum {simple}"), + str_of(value, "doc"), + line, + None, + Vec::new(), + None, + ); + let symbols = value.get("symbols").and_then(Value::as_array); + for symbol in symbols.into_iter().flatten().filter_map(Value::as_str) { + // Avro has no per-symbol doc attribute, so a symbol is documented by its enum. + self.push( + self.id_of(&namespace, &simple, Some(symbol)), + format!("{}.{symbol}", full_name(&namespace, &simple)), + SymbolKind::EnumVariant, + symbol.to_owned(), + "", + line, + Some(id.clone()), + Vec::new(), + None, + ); + } + } + + fn fixed(&mut self, value: &Value, enclosing: &str) { + let (namespace, simple) = split_name(value, enclosing); + if simple.is_empty() { + return; + } + let size = value.get("size").and_then(Value::as_u64).unwrap_or(0); + let line = self.find_line(&simple, 1); + self.push( + self.id_of(&namespace, &simple, None), + full_name(&namespace, &simple), + SymbolKind::TypeAlias, + format!("fixed {simple}({size})"), + str_of(value, "doc"), + line, + None, + Vec::new(), + None, + ); + } + + fn protocol(&mut self, value: &Value, enclosing: &str) { + let name = str_of(value, "protocol"); + let namespace = value + .get("namespace") + .and_then(Value::as_str) + .unwrap_or(enclosing) + .to_owned(); + let (namespace, simple) = match name.rsplit_once('.') { + Some((ns, simple)) => (ns.to_owned(), simple.to_owned()), + None => (namespace, name.to_owned()), + }; + if simple.is_empty() { + return; + } + let id = self.id_of(&namespace, &simple, None); + self.push( + id.clone(), + full_name(&namespace, &simple), + SymbolKind::Interface, + format!("protocol {simple}"), + str_of(value, "doc"), + 0, + None, + Vec::new(), + None, + ); + for ty in value.get("types").and_then(Value::as_array).into_iter().flatten() { + self.walk(ty, &namespace); + } + let messages = value.get("messages").and_then(Value::as_object); + for (message_name, message) in messages.into_iter().flatten() { + let request = message.get("request").and_then(Value::as_array); + let params: Vec = request + .into_iter() + .flatten() + .map(|param| Param { + name: str_of(param, "name").to_owned(), + r#type: type_text(param.get("type").unwrap_or(&Value::Null), &namespace), + doc: str_of(param, "doc").to_owned(), + }) + .collect(); + let response = type_text(message.get("response").unwrap_or(&Value::Null), &namespace); + let mut signature = format!( + "{message_name}({}) returns {response}", + params + .iter() + .map(|p| format!("{} {}", p.r#type, p.name)) + .collect::>() + .join(", ") + ); + let errors: Vec = message + .get("errors") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|error| type_text(error, &namespace)) + .filter(|text| text != "string") + .collect(); + if !errors.is_empty() { + signature.push_str(&format!(" throws {}", errors.join(", "))); + } + if message.get("one-way").and_then(Value::as_bool) == Some(true) { + signature.push_str(" (one-way)"); + } + self.push( + self.id_of(&namespace, &simple, Some(message_name)), + format!("{}.{message_name}", full_name(&namespace, &simple)), + SymbolKind::Method, + signature, + str_of(message, "doc"), + 0, + Some(id.clone()), + params, + Some(Returns { + r#type: response, + doc: String::new(), + }), + ); + for ty in message.get("request").and_then(Value::as_array).into_iter().flatten() { + self.walk(ty.get("type").unwrap_or(&Value::Null), &namespace); + } + } + } +} + +/// Converts schema texts, each with the source it documents and its file name, into one shard. +pub fn convert( + inputs: &[(String, String, String)], + options: &Options, +) -> Result { + let package = options.package.as_str(); + if package.is_empty() || package.contains(':') || package.chars().any(char::is_whitespace) { + return Err(format!("invalid package '{package}'")); + } + let mut symbols: BTreeMap = BTreeMap::new(); + for (text, source, file) in inputs { + let value: Value = serde_json::from_str(text) + .map_err(|error| format!("{file} is not valid JSON: {error}"))?; + // Line numbers only mean something when the schema text is the source itself. + let own_text = file == source || source.ends_with(".avsc") || source.ends_with(".avpr"); + let mut builder = Builder { + package, + source: source.clone(), + generated: options.generated.contains(source), + lines: if own_text { + text.lines().map(str::to_owned).collect() + } else { + Vec::new() + }, + symbols: BTreeMap::new(), + }; + builder.walk(&value, ""); + for (id, symbol) in builder.symbols { + match symbols.get(&id) { + Some(existing) + if !existing.doc_markdown.is_empty() || symbol.doc_markdown.is_empty() => {} + _ => { + symbols.insert(id, symbol); + } + } + } + } + if symbols.is_empty() { + return Err(format!( + "the schemas declare no named Avro type of package '{package}'" + )); + } + let shard = DocIr { + language: LANGUAGE.to_owned(), + package: package.to_owned(), + symbols: symbols.into_values().collect(), + }; + documentation_ir::validate_shard(&shard).map_err(|error| format!("invalid IR: {error:?}"))?; + Ok(shard) +} + +#[cfg(test)] +mod lib_tests; diff --git a/tools/docs/avro/extract/src/lib_tests.rs b/tools/docs/avro/extract/src/lib_tests.rs new file mode 100644 index 0000000000..3828d1e0e5 --- /dev/null +++ b/tools/docs/avro/extract/src/lib_tests.rs @@ -0,0 +1,202 @@ +use super::*; + +const P: &str = "avro:shop:"; + +fn options() -> Options { + Options { + package: "shop".to_owned(), + out: "out.ir.pb".to_owned(), + ..Options::default() + } +} + +fn shard_of(schemas: &[&str]) -> DocIr { + let inputs: Vec<(String, String, String)> = schemas + .iter() + .map(|text| ((*text).to_owned(), "shop.avsc".to_owned(), "shop.avsc".to_owned())) + .collect(); + convert(&inputs, &options()).expect("converts") +} + +fn symbol<'a>(shard: &'a DocIr, qualified: &str) -> &'a Symbol { + let id = format!("{P}{qualified}"); + shard + .symbols + .iter() + .find(|s| s.id == id) + .unwrap_or_else(|| panic!("{id} is missing")) +} + +const ORDER: &str = r#"{ + "type": "record", + "name": "Order", + "namespace": "shop.v1", + "doc": "A customer order.", + "fields": [ + {"name": "id", "type": "string", "doc": "The order id."}, + {"name": "tags", "type": {"type": "array", "items": "string"}}, + {"name": "note", "type": ["null", "string"], "default": null, "doc": "Free text."}, + {"name": "status", "type": { + "type": "enum", "name": "Status", "doc": "Order state.", "symbols": ["OPEN", "SHIPPED"] + }}, + {"name": "digest", "type": {"type": "fixed", "name": "Digest", "size": 16, "doc": "A hash."}}, + {"name": "attrs", "type": {"type": "map", "values": "long"}} + ] +}"#; + +#[test] +fn every_kind_maps_to_its_symbol_kind_with_its_doc() { + let shard = shard_of(&[ORDER]); + for (qualified, kind, doc) in [ + ("shop.v1.Order", SymbolKind::Class, "A customer order."), + ("shop.v1.Order.id", SymbolKind::Field, "The order id."), + ("shop.v1.Status", SymbolKind::Enum, "Order state."), + ("shop.v1.Digest", SymbolKind::TypeAlias, "A hash."), + ] { + let found = symbol(&shard, qualified); + assert_eq!(found.kind, kind as i32, "{qualified}"); + assert_eq!(found.doc_markdown, doc, "{qualified}"); + } + let id = symbol(&shard, "shop.v1.Order.id"); + assert_eq!( + id.relations.as_ref().map(|r| r.member_of.clone()), + Some(vec![format!("{P}shop.v1.Order")]) + ); + let variant = symbol(&shard, "shop.v1.Status.OPEN"); + assert_eq!(variant.kind, SymbolKind::EnumVariant as i32); + assert_eq!( + variant.relations.as_ref().map(|r| r.member_of.clone()), + Some(vec![format!("{P}shop.v1.Status")]) + ); +} + +#[test] +fn signatures_show_types_defaults_and_sizes() { + let shard = shard_of(&[ORDER]); + assert_eq!(symbol(&shard, "shop.v1.Order").signature_text, "record Order"); + assert_eq!( + symbol(&shard, "shop.v1.Order.tags").signature_text, + "array tags" + ); + assert_eq!( + symbol(&shard, "shop.v1.Order.note").signature_text, + "null | string note = null" + ); + assert_eq!( + symbol(&shard, "shop.v1.Order.attrs").signature_text, + "map attrs" + ); + assert_eq!( + symbol(&shard, "shop.v1.Order.status").signature_text, + "shop.v1.Status status" + ); + assert_eq!( + symbol(&shard, "shop.v1.Digest").signature_text, + "fixed Digest(16)" + ); +} + +#[test] +fn a_missing_doc_gives_an_empty_doc_marked_incomplete() { + let shard = shard_of(&[ORDER]); + let tags = symbol(&shard, "shop.v1.Order.tags"); + assert_eq!(tags.doc_markdown, ""); + assert!(tags.extensions.iter().any(|e| e.key == KEY_INCOMPLETE)); + let documented = symbol(&shard, "shop.v1.Order.id"); + assert!(documented.extensions.is_empty()); +} + +#[test] +fn a_protocol_message_is_a_method_of_its_protocol() { + let protocol = r#"{ + "protocol": "Shop", "namespace": "shop.v1", "doc": "The order API.", + "types": [{"type": "error", "name": "NoSuchOrder", "doc": "Raised for an unknown id.", + "fields": [{"name": "id", "type": "string"}]}], + "messages": { + "getOrder": {"doc": "Reads one order.", + "request": [{"name": "id", "type": "string", "doc": "The id."}], + "response": "string", "errors": ["NoSuchOrder"]} + } + }"#; + let shard = shard_of(&[protocol]); + let shop = symbol(&shard, "shop.v1.Shop"); + assert_eq!(shop.kind, SymbolKind::Interface as i32); + assert_eq!(shop.doc_markdown, "The order API."); + let method = symbol(&shard, "shop.v1.Shop.getOrder"); + assert_eq!(method.kind, SymbolKind::Method as i32); + assert_eq!(method.doc_markdown, "Reads one order."); + assert_eq!( + method.signature_text, + "getOrder(string id) returns string throws NoSuchOrder" + ); + assert_eq!(method.params.len(), 1); + assert_eq!(symbol(&shard, "shop.v1.NoSuchOrder").kind, SymbolKind::Class as i32); +} + +#[test] +fn namespaces_come_from_the_name_the_field_or_the_enclosing_type() { + let schema = r#"{"type": "record", "name": "a.b.Outer", "doc": "d", "fields": [ + {"name": "inner", "type": {"type": "record", "name": "Inner", "doc": "i", "fields": []}}, + {"name": "other", "type": {"type": "fixed", "name": "Id", "namespace": "z", "size": 4, "doc": "f"}} + ]}"#; + let shard = shard_of(&[schema]); + symbol(&shard, "a.b.Outer"); + symbol(&shard, "a.b.Inner"); + symbol(&shard, "z.Id"); +} + +#[test] +fn a_type_without_a_namespace_has_no_prefix() { + let schema = r#"{"type": "enum", "name": "Color", "doc": "c", "symbols": ["RED"]}"#; + let shard = shard_of(&[schema]); + symbol(&shard, "Color"); + symbol(&shard, "Color.RED"); +} + +#[test] +fn the_documented_copy_of_a_repeated_type_wins() { + let bare = r#"{"type": "enum", "name": "Color", "symbols": ["RED"]}"#; + let documented = r#"{"type": "enum", "name": "Color", "doc": "Paint.", "symbols": ["RED"]}"#; + for order in [[bare, documented], [documented, bare]] { + let shard = shard_of(&order); + assert_eq!(symbol(&shard, "Color").doc_markdown, "Paint."); + } +} + +#[test] +fn lines_point_at_the_name_in_the_source() { + let shard = shard_of(&[ORDER]); + assert_eq!(symbol(&shard, "shop.v1.Order").source.as_ref().map(|s| s.line), Some(3)); + assert_eq!(symbol(&shard, "shop.v1.Order.id").source.as_ref().map(|s| s.line), Some(7)); +} + +#[test] +fn bad_input_is_rejected_with_a_reason() { + let inputs = vec![("{".to_owned(), "x.avsc".to_owned(), "x.avsc".to_owned())]; + let error = convert(&inputs, &options()).expect_err("malformed JSON"); + assert!(error.contains("not valid JSON"), "{error}"); + let inputs = vec![("\"string\"".to_owned(), "x.avsc".to_owned(), "x.avsc".to_owned())]; + let error = convert(&inputs, &options()).expect_err("no named type"); + assert!(error.contains("no named Avro type"), "{error}"); + let mut bad = options(); + bad.package = "a b".to_owned(); + let error = convert(&[], &bad).expect_err("package"); + assert!(error.contains("invalid package"), "{error}"); +} + +#[test] +fn parse_reads_flags_and_rejects_bad_ones() { + let args = |list: &[&str]| list.iter().map(|s| (*s).to_owned()).collect::>(); + let parsed = parse(&args(&[ + "--package", "shop", "--out", "o", "--schema", "a.avsc=src/a.avsc", "--schema-dir", + "tree=src/b.avdl", "--generated", "src/b.avdl", + ])) + .expect("parses"); + assert_eq!(parsed.schemas, vec![("a.avsc".to_owned(), "src/a.avsc".to_owned())]); + assert_eq!(parsed.schema_dirs.len(), 1); + assert_eq!(parsed.generated, vec!["src/b.avdl".to_owned()]); + assert!(parse(&args(&["--package", "p", "--out", "o"])).is_err()); + assert!(parse(&args(&["--package", "p", "--out", "o", "--schema", "nope"])).is_err()); + assert!(parse(&args(&["--bogus", "x"])).is_err()); + assert!(parse(&args(&["--package"])).is_err()); +} diff --git a/tools/docs/avro/extract/src/main.rs b/tools/docs/avro/extract/src/main.rs new file mode 100644 index 0000000000..97f844a195 --- /dev/null +++ b/tools/docs/avro/extract/src/main.rs @@ -0,0 +1,38 @@ +//! Command line entry for the Avro docs converter. + +#![cfg_attr( + not(test), + deny( + clippy::expect_used, + clippy::unwrap_used, + clippy::unreachable, + clippy::todo + ) +)] + +use std::process::ExitCode; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + if args.iter().any(|a| a == "-h" || a == "--help") { + println!("{}", dx_docs_avro_extract::usage()); + return ExitCode::SUCCESS; + } + let options = match dx_docs_avro_extract::parse(&args) { + Ok(options) => options, + Err(detail) => { + eprintln!( + "avro_extract: {detail}\n{}", + dx_docs_avro_extract::usage() + ); + return ExitCode::from(2); + } + }; + match dx_docs_avro_extract::run(&options) { + Ok(()) => ExitCode::SUCCESS, + Err(detail) => { + eprintln!("avro_extract: {detail}"); + ExitCode::FAILURE + } + } +} diff --git a/tools/docs/avro/fixture/BUILD.bazel b/tools/docs/avro/fixture/BUILD.bazel new file mode 100644 index 0000000000..74ad512371 --- /dev/null +++ b/tools/docs/avro/fixture/BUILD.bazel @@ -0,0 +1,35 @@ +load("//avro/rules:defs.bzl", "avro_library") +load("//quality:fixtures.bzl", "real_source_target") +load("//tools/docs/avro:defs.bzl", "avro_doc_shard") +load("//tools/visibility:visibility.bzl", "VIS_TOOLS") + +package(default_visibility = VIS_TOOLS) + +exports_files( + [ + "service.avdl", + "shop.avsc", + ], + visibility = VIS_TOOLS, +) + +avro_library( + name = "shop_avro", + srcs = [ + "service.avdl", + "shop.avsc", + ], +) + +avro_doc_shard( + name = "shop_docs", + testonly = True, + avro = ":shop_avro", + package = "shop", +) + +real_source_target( + name = "corpus_starlark", + starlark_srcs = ["BUILD.bazel"], + tags = ["corpus"], +) diff --git a/tools/docs/avro/fixture/service.avdl b/tools/docs/avro/fixture/service.avdl new file mode 100644 index 0000000000..ee4332fc42 --- /dev/null +++ b/tools/docs/avro/fixture/service.avdl @@ -0,0 +1,20 @@ +/** The order API. */ +@namespace("shop.v1") +protocol Shop { + /** Raised when an order id is unknown. */ + error NoSuchOrder { + /** The id that was asked for. */ + string id; + } + + /** One line of an order. */ + record Line { + /** What was bought. */ + string sku; + /** How many were bought. */ + int quantity; + } + + /** Reads one order. */ + string getOrder(/** The order id. */ string id) throws NoSuchOrder; +} diff --git a/tools/docs/avro/fixture/shop.avsc b/tools/docs/avro/fixture/shop.avsc new file mode 100644 index 0000000000..9caefc2676 --- /dev/null +++ b/tools/docs/avro/fixture/shop.avsc @@ -0,0 +1,11 @@ +{ + "type": "record", + "name": "Order", + "namespace": "shop.v1", + "doc": "A customer order.", + "fields": [ + {"name": "id", "type": "string", "doc": "The order id."}, + {"name": "status", "type": {"type": "enum", "name": "Status", "doc": "Where an order is in its life.", "symbols": ["OPEN", "SHIPPED"]}, "doc": "The current state."}, + {"name": "digest", "type": {"type": "fixed", "name": "Digest", "size": 16, "doc": "A 128 bit content hash."}, "doc": "Hash of the order lines."} + ] +} diff --git a/tools/docs/avro/shard_test.rs b/tools/docs/avro/shard_test.rs new file mode 100644 index 0000000000..aef939589f --- /dev/null +++ b/tools/docs/avro/shard_test.rs @@ -0,0 +1,88 @@ +//! Checks that the fixture shard carries the `doc` text of the fixture's .avsc and .avdl files. + +use std::path::PathBuf; + +use documentation_ir::proto::{DocIr, Symbol, SymbolKind}; +use documentation_ir::{decode_shard, validate_shard}; + +const PREFIX: &str = "avro:shop:"; + +fn runfile(var: &str) -> PathBuf { + let rel = std::env::var(var).unwrap_or_else(|_| panic!("{var} must name a runfile")); + dx_testing::resolve_runfiles(&rel) +} + +fn shard() -> DocIr { + let bytes = std::fs::read(runfile("DX_FIXTURE_SHARD")).expect("read the shard"); + decode_shard(&bytes).expect("the shard decodes") +} + +fn symbol<'a>(shard: &'a DocIr, qualified: &str) -> &'a Symbol { + let id = format!("{PREFIX}{qualified}"); + shard + .symbols + .iter() + .find(|s| s.id == id) + .unwrap_or_else(|| panic!("{id} is missing")) +} + +#[test] +fn the_fixture_shard_documents_the_avsc_and_the_avdl() { + let shard = shard(); + validate_shard(&shard).expect("validate_shard passes"); + assert_eq!(shard.language, "avro"); + assert_eq!(shard.package, "shop"); + // Enum symbols have no doc attribute in Avro; every other item in the fixture has one. + for symbol in &shard.symbols { + if symbol.kind != SymbolKind::EnumVariant as i32 { + assert!(!symbol.doc_markdown.is_empty(), "{} has no doc", symbol.id); + } + } + for (qualified, kind, doc, file) in [ + ( + "shop.v1.Order", + SymbolKind::Class, + "A customer order.", + "tools/docs/avro/fixture/shop.avsc", + ), + ( + "shop.v1.Order.id", + SymbolKind::Field, + "The order id.", + "tools/docs/avro/fixture/shop.avsc", + ), + ( + "shop.v1.Status", + SymbolKind::Enum, + "Where an order is in its life.", + "tools/docs/avro/fixture/shop.avsc", + ), + ( + "shop.v1.Digest", + SymbolKind::TypeAlias, + "A 128 bit content hash.", + "tools/docs/avro/fixture/shop.avsc", + ), + ( + "shop.v1.Line.sku", + SymbolKind::Field, + "What was bought.", + "tools/docs/avro/fixture/service.avdl", + ), + ( + "shop.v1.NoSuchOrder", + SymbolKind::Class, + "Raised when an order id is unknown.", + "tools/docs/avro/fixture/service.avdl", + ), + ] { + let found = symbol(&shard, qualified); + assert_eq!(found.kind, kind as i32, "{qualified}"); + assert_eq!(found.doc_markdown, doc, "{qualified}"); + let source = found.source.as_ref().expect("source"); + assert_eq!(source.file, file, "{qualified}"); + assert!(!source.generated, "{qualified}"); + } + let status = symbol(&shard, "shop.v1.Order.status"); + assert_eq!(status.signature_text, "shop.v1.Status status"); +} diff --git a/tools/docs/support_matrix/facts_tests.bzl b/tools/docs/support_matrix/facts_tests.bzl index 93fe9fa453..dd39af3b74 100644 --- a/tools/docs/support_matrix/facts_tests.bzl +++ b/tools/docs/support_matrix/facts_tests.bzl @@ -8,6 +8,7 @@ load("//tools/docs/cc:defs.bzl", "cc_doc_shard", "cuda_doc_shard") load("//tools/docs/components:defs.bzl", "astro_doc_shard", "mdx_doc_shard", "svelte_doc_shard", "vue_doc_shard") load("//tools/docs/dart:defs.bzl", "dart_doc_shard") load("//tools/docs/d:defs.bzl", "d_doc_shard") +load("//tools/docs/avro:defs.bzl", "avro_doc_shard") load("//tools/docs/dotnet:defs.bzl", "csharp_doc_shard", "fsharp_doc_shard") load("//tools/docs/go:defs.bzl", "go_doc_shard") load("//tools/docs/javascript:defs.bzl", "javascript_doc_shard") @@ -28,6 +29,7 @@ load(":languages.bzl", "BENCH", "DEPENDENCIES", "DOC_TOOLS", "FUZZ", "PLATFORMS" # Loading each symbol proves the shard rule exists; the names are what DOC_TOOLS may point at. _SHARD_RULES = { + "avro_doc_shard": avro_doc_shard, "astro_doc_shard": astro_doc_shard, "cc_doc_shard": cc_doc_shard, "csharp_doc_shard": csharp_doc_shard, diff --git a/tools/docs/support_matrix/languages.bzl b/tools/docs/support_matrix/languages.bzl index e67ca72b3a..645b0758cc 100644 --- a/tools/docs/support_matrix/languages.bzl +++ b/tools/docs/support_matrix/languages.bzl @@ -370,7 +370,7 @@ LANGUAGES = [ _language( "avro", deps = "none: schemas only, no packages", - doc = NO_DOC, + doc = struct(tool = "avro-tools", shard = "avro_doc_shard"), dead_code = ["none"], ), _language(