starlark: pins.json and dx_pins extension; ruby and powershell first (part of #10144)

MODULE.bazel

@@ -824,7 +824,33 @@ ruby_coverage = use_extension("//ruby/coverage:repos.bzl", "ruby_coverage")
 use_repo(ruby_coverage, "dx_ruby_bench_gems", "dx_ruby_coverage_gems")
 
 ruby_tools = use_extension("//ruby/toolchain:repos.bzl", "ruby_tools")
-use_repo(ruby_tools, "dx_sorbet", "rules_ruby")
+use_repo(ruby_tools, "rules_ruby")
+
+dx_pins = use_extension("//libs/starlark:toolchain.bzl", "dx_pins")
+dx_pins.pin(
+    name = "dx_sorbet",
+    aliases = {"windows_arm64": "windows_x86_64"},
+    build = "//ruby/toolchain:sorbet.build",
+    pins = "//ruby/toolchain:pins.json",
+)
+dx_pins.pin(
+    name = "dx_powershell_toolchains",
+    build = "//powershell/toolchain:pwsh.build",
+    pins = "//powershell/toolchain:pins.json",
+    toolchain_type = "//powershell/toolchain/type:toolchain_type",
+)
+use_repo(
+    dx_pins,
+    "dx_powershell_toolchains",
+    "dx_powershell_toolchains__linux_arm64",
+    "dx_powershell_toolchains__linux_x86_64",
+    "dx_powershell_toolchains__macos_arm64",
+    "dx_powershell_toolchains__macos_x86_64",
+    "dx_powershell_toolchains__windows_arm64",
+    "dx_powershell_toolchains__windows_x86_64",
+    "dx_pins_data",
+    "dx_sorbet",
+)
 
 # The pinned Ruby is declared in families/ruby; deploy tests read its files.
 ruby = use_extension("//ruby:extensions.bzl", "ruby")
@@ -1183,17 +1209,7 @@ use_repo(elixir, "dx_benchee", "dx_credo", "dx_elixir", "rules_elixir")
 bazel_dep(name = "rules_batch", version = "0.1.1")
 
 powershell = use_extension("//powershell/toolchain:extensions.bzl", "powershell")
-use_repo(
-    powershell,
-    "dx_powershell_toolchains",
-    "dx_powershell_toolchains__linux_arm64",
-    "dx_powershell_toolchains__linux_x86_64",
-    "dx_powershell_toolchains__macos_arm64",
-    "dx_powershell_toolchains__macos_x86_64",
-    "dx_powershell_toolchains__windows_arm64",
-    "dx_powershell_toolchains__windows_x86_64",
-    "rules_powershell",
-)
+use_repo(powershell, "rules_powershell")
 
 ocaml = use_extension("//ocaml/toolchain:extensions.bzl", "ocaml")
 ocaml.lock(

libs/starlark/BUILD.bazel

@@ -1,5 +1,6 @@
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("//quality:fixtures.bzl", "real_source_target")
+load(":pins_tests.bzl", "pins_tests")
 load(":platform.bzl", "platform_probe")
 
 bool_flag(
@@ -26,6 +27,7 @@ exports_files([
     "node_test.bzl",
     "packaging.bzl",
     "plan_shard.bzl",
+    "pins_tests.bzl",
     "platform.bzl",
     "repo.bzl",
     "resources.bzl",
@@ -55,6 +57,7 @@ real_source_target(
         "jammy_runtime.bzl",
         "msvc_crt.bzl",
         "packaging.bzl",
+        "pins_tests.bzl",
         "plan_shard.bzl",
         "platform.bzl",
         "repo.bzl",
@@ -72,6 +75,8 @@ real_source_target(
     visibility = ["//tools/ci:__pkg__"],
 )
 
+pins_tests(name = "pins_test")
+
 platform_probe(
     name = "platform_probe",
     visibility = ["//visibility:public"],

libs/starlark/pins_tests.bzl

@@ -0,0 +1,33 @@
+"""The checks every pins.json shares."""
+
+load("@dx_pins_data//:pins.bzl", "PINS")
+load(":defs.bzl", "expect_true", "starlark_test")
+load(":toolchain.bzl", "pin_checks")
+
+def _archive(pin):
+    return struct(
+        archive = pin["urls"][0].rpartition("/")[2],
+        sha256 = pin["sha256"],
+        urls = pin["urls"],
+    )
+
+def _checks():
+    checks = [expect_true("pins are read", len(PINS) > 0)]
+    for name, pins in PINS.items():
+        cells = {cell: _archive(pin) for cell, pin in pins["cells"].items() if "urls" in pin}
+        checks += [
+            expect_true(name + " has cells", len(cells) > 0),
+            expect_true(name + " binaries are set", all([pin["binary"] != "" for pin in pins["cells"].values()])),
+        ] + [
+            check
+            for check in pin_checks(cells, pins["version"])
+        ]
+    return checks
+
+def pins_tests(name):
+    """Declares the test that runs pin_checks over every pins.json."""
+    starlark_test(
+        name = name,
+        mode = "load",
+        checks = _checks(),
+    )

libs/starlark/tests/cells_tests.bzl

@@ -1,5 +1,6 @@
 """Shared cell setting tests."""
 
+load("@dx_pins_data//:pins.bzl", "PINS")
 load("//d/toolchain:pins.bzl", "LDC_CELLS")
 load("//erlang/toolchain:pins.bzl", erlang_cells = "CELLS")
 load("//fortran/toolchain:pins.bzl", "FORTRAN_CELLS")
@@ -10,9 +11,7 @@ load("//lua/toolchain:pins.bzl", "LUALS_CELLS")
 load("//modules:cuda.bzl", "CUDA_CELLS")
 load("//ocaml/toolchain:pins.bzl", "OCAML_CELLS")
 load("//php/toolchain:pins.bzl", "PHP_CELLS")
-load("//powershell/toolchain:pins.bzl", "PWSH_CELLS")
 load("//qt/sdk:pins.bzl", qt_cells = "CELLS")
-load("//ruby/toolchain:pins.bzl", "SORBET_CELLS")
 
 _DEFAULT = "//conditions:default"
 
@@ -29,9 +28,9 @@ _ALL_CELLS = {
     "lua": LUALS_CELLS,
     "ocaml": OCAML_CELLS,
     "php": PHP_CELLS,
-    "powershell": PWSH_CELLS,
+    "powershell": PINS["dx_powershell_toolchains"]["cells"],
     "qt": qt_cells,
-    "ruby": SORBET_CELLS,
+    "ruby": PINS["dx_sorbet"]["cells"],
 }
 
 _OSES = ["linux", "macos", "windows"]

libs/starlark/toolchain.bzl

@@ -2,6 +2,7 @@
 
 load(":defs.bzl", "expect_equal", "expect_true")
 load(":host.bzl", "host_cell")
+load(":repo.bzl", "gem_extract", "hub_repo")
 
 _HOST_CELLS = [
     "linux_arm64",
@@ -12,18 +13,35 @@ _HOST_CELLS = [
     "windows_x86_64",
 ]
 
+def _fetch(rctx, tool, pin, build):
+    binary = pin["binary"]
+    if "stub" in pin:
+        rctx.file(binary, pin["stub"], executable = True)
+    elif pin.get("type") == "gem":
+        gem_extract(
+            rctx,
+            url = pin["urls"],
+            sha256 = pin["sha256"],
+            canonical_id = "dx-gem:" + pin["urls"][0],
+            output = ".",
+            required = [binary],
+        )
+    else:
+        rctx.download_and_extract(url = pin["urls"], sha256 = pin["sha256"], stripPrefix = pin.get("strip_prefix", ""))
+        if not rctx.path(binary).exists:
+            fail("{} archive {} has no {}".format(tool, pin["urls"][0], binary))
+        if pin.get("executable"):
+            rctx.file(binary, rctx.read(binary), executable = True)
+    rctx.file("REPO.bazel", "")
+    rctx.file("BUILD.bazel", build.replace("{binary}", binary))
+    return rctx.repo_metadata(reproducible = True)
+
 def _host_archive_impl(rctx):
     cell = rctx.attr.aliases.get(host_cell(rctx), host_cell(rctx))
-    cells = json.decode(rctx.attr.cells)
-    pin = cells.get(cell)
+    pin = json.decode(rctx.attr.cells).get(cell)
     if pin == None:
         fail("no pinned {} archive for {}".format(rctx.attr.tool, cell))
-    rctx.download_and_extract(url = pin["urls"], sha256 = pin["sha256"], stripPrefix = pin.get("strip_prefix", ""))
-    if not rctx.path(pin["binary"]).exists:
-        fail("{} archive {} has no {}".format(rctx.attr.tool, pin["urls"][0], pin["binary"]))
-    rctx.file("REPO.bazel", "")
-    rctx.file("BUILD.bazel", rctx.attr.build.replace("{binary}", pin["binary"]))
-    return rctx.repo_metadata(reproducible = True)
+    return _fetch(rctx, rctx.attr.tool, pin, rctx.attr.build)
 
 _host_archive = repository_rule(
     implementation = _host_archive_impl,
@@ -36,6 +54,19 @@ _host_archive = repository_rule(
     },
 )
 
+def _cell_archive_impl(rctx):
+    return _fetch(rctx, rctx.attr.tool, json.decode(rctx.attr.pin), rctx.attr.build)
+
+_cell_archive = repository_rule(
+    implementation = _cell_archive_impl,
+    doc = "Downloads the pinned archive of one cell and writes its BUILD file.",
+    attrs = {
+        "build": attr.string(mandatory = True, doc = "BUILD file content; {binary} is replaced by the cell's binary path."),
+        "pin": attr.string(mandatory = True, doc = "JSON object of the cell's urls, sha256, binary and optional strip_prefix."),
+        "tool": attr.string(mandatory = True, doc = "Tool name used in failure messages."),
+    },
+)
+
 def host_archive(name, tool, cells, build, aliases = {}):
     """Declares a repository holding the pinned archive of the host cell.
 
@@ -110,3 +141,69 @@ def hub_checks(hub, toolchain_type, count, forbidden = []):
         expect_true("hub names the toolchain type", toolchain_type in hub),
         expect_equal("hub toolchain count", hub.count("toolchain("), count),
     ] + [expect_true("hub does not name " + repo, repo not in hub) for repo in forbidden]
+
+_OS_CONSTRAINTS = {"linux": "@platforms//os:linux", "macos": "@platforms//os:macos", "windows": "@platforms//os:windows"}
+
+_CPU_CONSTRAINTS = {"arm64": "@platforms//cpu:aarch64", "x86_64": "@platforms//cpu:x86_64"}
+
+def cell_repo(name, cell):
+    """Returns the name of the repository holding one cell of a pin."""
+    return name + "__" + cell
+
+def pins_hub(name, cells, toolchain_type):
+    """Returns the hub BUILD file of a pin: one toolchain() per cell.
+
+    Args:
+        name: Pin name.
+        cells: Cell names.
+        toolchain_type: Label string of the toolchain type.
+    """
+    return "\n".join([
+        toolchain_target(
+            name = cell_repo(name, cell),
+            exec_compatible_with = [_OS_CONSTRAINTS[cell.partition("_")[0]], _CPU_CONSTRAINTS[cell.partition("_")[2]]],
+            toolchain = "@{}//:toolchain".format(cell_repo(name, cell)),
+            toolchain_type = toolchain_type,
+            visibility = ["//visibility:public"],
+        )
+        for cell in sorted(cells)
+    ])
+
+def _dx_pins_impl(mctx):
+    data = {}
+    for mod in mctx.modules:
+        for tag in mod.tags.pin:
+            pins = json.decode(mctx.read(tag.pins))
+            build = mctx.read(tag.build)
+            data[tag.name] = pins
+            if tag.toolchain_type == None:
+                _host_archive(
+                    name = tag.name,
+                    aliases = tag.aliases,
+                    build = build,
+                    cells = json.encode(pins["cells"]),
+                    tool = tag.name,
+                )
+                continue
+            for cell, pin in pins["cells"].items():
+                _cell_archive(name = cell_repo(tag.name, cell), build = build, pin = json.encode(pin), tool = tag.name)
+            hub_repo(name = tag.name, files = {"BUILD.bazel": pins_hub(tag.name, pins["cells"].keys(), str(tag.toolchain_type))})
+    hub_repo(name = "dx_pins_data", files = {"BUILD.bazel": "", "pins.bzl": "PINS = " + repr(data) + "\n"})
+    return mctx.extension_metadata(reproducible = True)
+
+dx_pins = module_extension(
+    implementation = _dx_pins_impl,
+    doc = "Creates the repositories of every pins.json: one per cell and a toolchain hub, or one host-selected repository.",
+    tag_classes = {
+        "pin": tag_class(
+            doc = "One pins.json. With a toolchain_type it makes a repository per cell and a hub of toolchain() targets; without, one repository of the host cell.",
+            attrs = {
+                "aliases": attr.string_dict(doc = "Host cell to the pinned cell that serves it."),
+                "build": attr.label(mandatory = True, allow_single_file = True, doc = "BUILD file template; {binary} is replaced by the cell's binary path."),
+                "name": attr.string(mandatory = True, doc = "Hub, or sole, repository name."),
+                "pins": attr.label(mandatory = True, allow_single_file = True, doc = "JSON file of version and cells."),
+                "toolchain_type": attr.label(doc = "Toolchain type the hub registers; unset for a host-selected repository."),
+            },
+        ),
+    },
+)

powershell/toolchain/BUILD.bazel

@@ -7,8 +7,8 @@ exports_files([
     "extensions.bzl",
     "gallery.bzl",
     "metadata_tests.bzl",
-    "pins.bzl",
-    "repos.bzl",
+    "pins.json",
+    "pwsh.build",
 ])
 
 powershell_metadata_tests(name = "metadata")
@@ -20,8 +20,6 @@ real_source_target(
         "extensions.bzl",
         "gallery.bzl",
         "metadata_tests.bzl",
-        "pins.bzl",
-        "repos.bzl",
     ],
     tags = ["corpus"],
 )

powershell/toolchain/extensions.bzl

@@ -1,22 +1,19 @@
 """The powershell module extension."""
 
-load("//libs/starlark:repo.bzl", "hub_repo", "pinned_archive")
-load(":pins.bzl", "PWSH_CELLS", "RULES_POWERSHELL_SHA256", "RULES_POWERSHELL_URL")
-load(":repos.bzl", "HUB_NAME", "hub_build", "pwsh_repo", "repo_name")
+load("//libs/starlark:repo.bzl", "pinned_archive")
+
+_VERSION = "0.3.1"
 
 def _powershell_impl(mctx):
     pinned_archive(
         name = "rules_powershell",
         patches = [Label("//patches/rules_powershell:toolchain_type_alias.patch")],
-        sha256 = RULES_POWERSHELL_SHA256,
-        urls = [RULES_POWERSHELL_URL],
+        sha256 = "8ef875aa1dad8dc0f8f568547d2283f3d4b686b8e50374d6cdda4c5688fe6b7f",
+        urls = ["https://github.com/periareon/rules_powershell/releases/download/{0}/rules_powershell-{0}.tar.gz".format(_VERSION)],
     )
-    for cell in sorted(PWSH_CELLS.keys()):
-        pwsh_repo(name = repo_name(cell), cell = cell)
-    hub_repo(name = HUB_NAME, files = {"BUILD.bazel": hub_build()})
     return mctx.extension_metadata(reproducible = True)
 
 powershell = module_extension(
     implementation = _powershell_impl,
-    doc = "Creates rules_powershell, the pinned PowerShell builds and the toolchain hub.",
+    doc = "Creates the patched rules_powershell.",
 )

powershell/toolchain/metadata_tests.bzl

@@ -1,17 +1,15 @@
-"""PowerShell toolchain pin and hub tests."""
+"""PowerShell toolchain hub tests."""
 
-load("//libs/starlark:defs.bzl", "expect_contains", "expect_equal", "expect_true", "starlark_test")
-load("//libs/starlark:toolchain.bzl", "hub_checks")
-load(":pins.bzl", "PWSH_CELLS", "PWSH_VERSION", "RULES_POWERSHELL_SHA256")
-load(":repos.bzl", "hub_build", "repo_name")
+load("@dx_pins_data//:pins.bzl", "PINS")
+load("//libs/starlark:defs.bzl", "expect_contains", "expect_equal", "starlark_test")
+load("//libs/starlark:toolchain.bzl", "cell_repo", "hub_checks", "pins_hub")
 
 def _checks():
-    hub = hub_build()
-    return hub_checks(hub, "//powershell/toolchain/type:toolchain_type", len(PWSH_CELLS), ["@rules_powershell"]) + [
-        expect_true("pwsh digests are sha256", all([len(c.sha256) == 64 for c in PWSH_CELLS.values()])),
-        expect_true("artifacts name the version", all([PWSH_VERSION in c.artifact for c in PWSH_CELLS.values()])),
-        expect_equal("rules_powershell digest", len(RULES_POWERSHELL_SHA256), 64),
-        expect_contains("hub names each platform repo", hub, "@" + repo_name("linux_x86_64") + "//:toolchain"),
+    cells = PINS["dx_powershell_toolchains"]["cells"].keys()
+    hub = pins_hub("dx_powershell_toolchains", cells, "//powershell/toolchain/type:toolchain_type")
+    return hub_checks(hub, "//powershell/toolchain/type:toolchain_type", len(cells), ["@rules_powershell"]) + [
+        expect_equal("cells are the six host cells", len(cells), 6),
+        expect_contains("hub names each platform repo", hub, "@" + cell_repo("dx_powershell_toolchains", "linux_x86_64") + "//:toolchain"),
     ]
 
 def powershell_metadata_tests(name):