fix: preserve conditional APIs and DocC navigation

README.md

@@ -37,6 +37,15 @@ products from each tagged manifest and merges multi-module documentation.
 Source archives are downloaded at the tag's full commit SHA and removed after
 conversion. No repository checkout is copied into the build cache.
 
+A DocC catalog entry may select `traits` from its tagged package, including
+`default` when it needs the default APIs. Both compilation and symbol extraction
+use that selection. Omitted traits keep the package defaults. Symbol extraction
+includes extensions to external types so those APIs remain linkable in DocC.
+Generated breadcrumbs omit external-type containers for which DocC emits no
+page, while keeping the package's actual extension members. The link checker
+distinguishes tutorial chapter labels from navigable tutorial pages; explicit
+content links still require a real destination.
+
 For landings without their own identity, the builder adds the repository's
 current `Logo.png` and derives the named DocC color from the organization's
 `DESIGN.md`. Existing package directives remain authoritative. A synthesized

Scripts/build-site

@@ -21,7 +21,7 @@ from site_support import (
 
 from site_docc import (
     PIPELINE_SOURCE as DOCC_PIPELINE_SOURCE, add_identity, archive_source,
-    design_colors, identity_records, merged_identity,
+    design_colors, identity_records, merged_identity, normalize_extension_hierarchy,
 )
 
 SCRIPT_SOURCE = Path(__file__).read_text()
@@ -57,6 +57,19 @@ def library_targets(manifest, selection):
     return names
 
 
+def documentation_traits(manifest, selection):
+    if selection is None:
+        return []
+    if (not isinstance(selection, list) or not selection
+            or any(not isinstance(name, str) for name in selection)
+            or len(selection) != len(set(selection))):
+        raise SiteError("Documentation traits must be a non-empty list of distinct names")
+    available = {trait["name"] for trait in manifest.get("traits", [])} | {"default"}
+    if not set(selection) <= available:
+        raise SiteError("Documentation traits must be declared by the tagged package")
+    return ["--traits", ",".join(selection)]
+
+
 def build_documentation(config, project, tag, cache, output, toolchain, jobs, color, icon):
     repo = project["repository"]
     fingerprint = hashlib.sha256(
@@ -87,12 +100,13 @@ def build_documentation(config, project, tag, cache, output, toolchain, jobs, co
         print(f"{repo}: extracting {len(modules)} modules at {tag['name']}", flush=True)
         swift_options = ["--build-system", "native", "--disable-index-store", "-Xswiftc", "-gnone",
                          "--scratch-path", str(scratch)]
+        swift_options += documentation_traits(manifest, project["documentation"].get("traits"))
         if jobs is not None:
             swift_options += ["--jobs", str(jobs)]
         # SwiftPM's symbol extraction includes generated test-runner modules.
         command(["swift", "build", *swift_options, "--build-tests"], cwd=source)
         command(["swift", "package", *swift_options, "dump-symbol-graph",
-                 "--minimum-access-level", "public"], cwd=source)
+                 "--minimum-access-level", "public", "--emit-extension-block-symbols"], cwd=source)
         graph_directories = [path for path in scratch.glob("*/symbolgraph") if path.is_dir()]
         if not graph_directories:
             graph_directories = [path for path in scratch.glob("*/*/symbolgraph") if path.is_dir()]
@@ -145,6 +159,7 @@ def build_documentation(config, project, tag, cache, output, toolchain, jobs, co
                      "--output-path", str(archive), "--synthesized-landing-page-name", repo])
         if len(archives) > 1:
             merged_identity(archive, repo, color, icon)
+        normalize_extension_hierarchy(archive)
         if published.exists():
             shutil.rmtree(published)
         command(["xcrun", "docc", "process-archive", "transform-for-static-hosting", str(archive),

Scripts/site_checks.py

@@ -9,6 +9,8 @@
 import re
 from urllib.parse import unquote, urlsplit
 
+from site_docc import referenced_values
+
 
 class HTML(HTMLParser):
     def __init__(self):
@@ -45,6 +47,17 @@ def json_anchors(value):
     return result
 
 
+def chapter_labels(document):
+    # Tutorial chapters label groups in the navigation; their projects are links.
+    chapters = {chapter.get("reference") for chapter in (document.get("hierarchy") or {}).get("modules", [])}
+    content = {key: value for key, value in document.items() if key not in {"hierarchy", "references"}}
+    used = referenced_values(content)
+    for identifier, reference in document.get("references", {}).items():
+        if identifier not in chapters:
+            used.update(referenced_values(reference))
+    return chapters - used
+
+
 class Links:
     def __init__(self, root, site_url):
         self.root = root.resolve()
@@ -117,7 +130,10 @@ def scan(self):
                 continue
             for source in sorted((directory / "data").rglob("*.json")):
                 value = json.loads(source.read_text())
-                for reference in value.get("references", {}).values():
+                labels = chapter_labels(value)
+                for identifier, reference in value.get("references", {}).items():
+                    if identifier in labels:
+                        continue
                     if isinstance(reference.get("url"), str):
                         self.check(reference["url"], source, directory.name)
                     for variant in reference.get("variants", []):

Scripts/site_docc.py

@@ -176,3 +176,58 @@ def identity_records(archive, modules, merged, repository=None):
             raise SiteError(f"DocC landing has no icon variants: {landing}")
         records.append({"landing": landing, "color": color, "icons": assets})
     return records
+
+
+def referenced_values(value):
+    if isinstance(value, str):
+        return {value}
+    if isinstance(value, dict):
+        return set().union(*(referenced_values(child) for child in value.values()))
+    if isinstance(value, list):
+        return set().union(*(referenced_values(child) for child in value))
+    return set()
+
+
+def normalize_extension_hierarchy(archive):
+    """Keep breadcrumbs on rendered pages when DocC omits external-type containers."""
+    from urllib.parse import unquote, urlsplit
+
+    data = archive / "data"
+    changed = 0
+    for path in data.rglob("*.json"):
+        document = json.loads(path.read_text())
+        references = document.get("references", {})
+        hierarchy = (document.get("hierarchy") or {}).get("paths", [])
+        ancestors = set(value for branch in hierarchy for value in branch)
+
+        def has_page(reference):
+            route = unquote(urlsplit(reference.get("url", "")).path).lstrip("/")
+            target = (data / (route + ".json")).resolve()
+            return target.is_relative_to(data.resolve()) and target.is_file()
+
+        omitted = set()
+        for identifier in ancestors:
+            reference = references.get(identifier, {})
+            is_extension = any(fragment.get("kind") == "keyword" and fragment.get("text") == "extension"
+                               for fragment in reference.get("fragments", []))
+            if not is_extension or has_page(reference):
+                continue
+            omitted.add(identifier)
+            module = identifier.rsplit("/", 1)[0]
+            parent = references.get(module, {})
+            if module in ancestors and parent.get("role") == "collection" and not has_page(parent):
+                omitted.add(module)
+        if not omitted:
+            continue
+        document["hierarchy"]["paths"] = [[value for value in branch if value not in omitted]
+                                           for branch in hierarchy]
+        other_content = {key: value for key, value in document.items() if key != "references"}
+        uses = referenced_values(other_content)
+        for identifier, reference in references.items():
+            if identifier not in omitted:
+                uses.update(referenced_values(reference))
+        for identifier in omitted - uses:
+            references.pop(identifier, None)
+        path.write_text(json.dumps(document, ensure_ascii=False) + "\n")
+        changed += 1
+    return changed

Tests/test_pipeline.py

@@ -12,6 +12,7 @@
 SCRIPTS = Path(__file__).resolve().parent.parent / "Scripts"
 sys.path.insert(0, str(SCRIPTS))
 from site_checks import Links
+from site_docc import normalize_extension_hierarchy
 from site_support import contributor_names, latest_release
 
 loader = importlib.machinery.SourceFileLoader("build_site", str(SCRIPTS / "build-site"))
@@ -22,7 +23,9 @@
 
 class LinkTests(unittest.TestCase):
     def setUp(self):
-        self.temporary = tempfile.TemporaryDirectory()
+        fixtures = SCRIPTS.parent / ".build/test-fixtures"
+        fixtures.mkdir(parents=True, exist_ok=True)
+        self.temporary = tempfile.TemporaryDirectory(dir=fixtures)
         self.addCleanup(self.temporary.cleanup)
         self.root = Path(self.temporary.name)
 
@@ -75,8 +78,78 @@ def test_merged_documentation_landing_and_dotted_symbols(self):
         self.write("package/data/documentation/module/member.name.json", json.dumps({"sections": [{"anchor": "usage"}]}))
         self.assertEqual(Links(self.root, "https://example.org/").scan(), [])
 
+    def test_tutorial_chapter_labels_do_not_require_standalone_pages(self):
+        self.write("package/tutorials/example/step/index.html", "")
+        document = {
+            "hierarchy": {"modules": [{"reference": "chapter", "projects": [{"reference": "step"}]}]},
+            "references": {"chapter": {"url": "/tutorials/example/chapter"},
+                           "step": {"url": "/tutorials/example/step"}},
+        }
+        path = self.write("package/data/tutorials/example/step.json", json.dumps(document))
+        self.assertEqual(Links(self.root, "https://example.org/").scan(), [])
+        document["abstract"] = [{"type": "reference", "identifier": "chapter"}]
+        path.write_text(json.dumps(document))
+        self.assertEqual(len(Links(self.root, "https://example.org/").scan()), 1)
+
+    def extension_document(self):
+        return {
+            "hierarchy": {"paths": [["module", "module/External", "module/External/Value", "member"]]},
+            "references": {
+                "module": {"url": "/documentation/module", "role": "collection"},
+                "module/External": {"url": "/documentation/module/external", "role": "collection"},
+                "module/External/Value": {"url": "/documentation/module/external/value", "role": "symbol",
+                                           "fragments": [{"kind": "keyword", "text": "extension"}]},
+                "member": {"url": "/documentation/module/member", "role": "symbol"},
+            },
+        }
+
+    def test_missing_external_containers_are_removed_from_breadcrumbs(self):
+        document = self.extension_document()
+        path = self.write("package/data/documentation/module/member.json", json.dumps(document))
+        self.write("package/documentation/module/index.html", "")
+        self.write("package/documentation/module/member/index.html", "")
+        self.assertEqual(normalize_extension_hierarchy(self.root / "package"), 1)
+        actual = json.loads(path.read_text())
+        self.assertEqual(actual["hierarchy"]["paths"], [["module", "member"]])
+        self.assertEqual(set(actual["references"]), {"module", "member"})
+        self.assertEqual(Links(self.root, "https://example.org/").scan(), [])
+
+    def test_rendered_extension_containers_are_preserved(self):
+        document = self.extension_document()
+        path = self.write("package/data/documentation/module/member.json", json.dumps(document))
+        self.write("package/data/documentation/module/external/value.json", "{}")
+        self.assertEqual(normalize_extension_hierarchy(self.root / "package"), 0)
+        self.assertEqual(json.loads(path.read_text()), document)
+
+    def test_missing_authored_links_and_regular_ancestors_still_fail(self):
+        document = self.extension_document()
+        document["abstract"] = [{"type": "reference", "identifier": "module/External/Value"}]
+        path = self.write("package/data/documentation/module/member.json", json.dumps(document))
+        self.write("package/documentation/module/member/index.html", "")
+        normalize_extension_hierarchy(self.root / "package")
+        actual = json.loads(path.read_text())
+        self.assertIn("module/External/Value", actual["references"])
+        errors = Links(self.root, "https://example.org/").scan()
+        self.assertEqual(len(errors), 2)
+        self.assertTrue(any("/documentation/module/external/value" in error for error in errors))
+        self.assertTrue(any(error.endswith("missing target /documentation/module") for error in errors))
+
 
 class InputTests(unittest.TestCase):
+    def test_documentation_traits_follow_tagged_manifest_and_keep_explicit_defaults(self):
+        manifest = {"traits": [{"name": "Experimental"}]}
+        self.assertEqual(build_site.documentation_traits(manifest, None), [])
+        self.assertEqual(build_site.documentation_traits(manifest, ["Experimental", "default"]),
+                         ["--traits", "Experimental,default"])
+
+    def test_invalid_trait_selection_fails_before_compilation(self):
+        manifest = {"traits": [{"name": "Experimental"}]}
+        for selection in [[], "Experimental", [True], ["Missing"],
+                          ["Experimental", "Experimental"], ["--disable-sandbox"]]:
+            with self.subTest(selection=selection):
+                with self.assertRaises(build_site.SiteError):
+                    build_site.documentation_traits(manifest, selection)
+
     def test_first_published_prerelease_enables_documentation(self):
         class API:
             def pages(self, endpoint):

projects.json

@@ -16,7 +16,7 @@
         {"repository": "swift-userdefault", "logo_path": "Documentation/Assets/Logo.svg", "documentation": {"kind": "docc", "targets": "library-products"}, "install": "swift-package"},
         {"repository": "swift-redux", "logo_path": "Documentation/Assets/Logo.svg", "documentation": {"kind": "docc", "targets": "library-products"}, "install": "swift-package"},
         {"repository": "swift-pdf", "logo_path": "Documentation/Assets/Logo.svg", "documentation": {"kind": "docc", "targets": "library-products"}, "install": "swift-package"},
-        {"repository": "swift-appstoreconnect", "logo_path": "Documentation/Assets/Logo.svg", "documentation": {"kind": "docc", "targets": "library-products"}, "install": "swift-package"},
+        {"repository": "swift-appstoreconnect", "logo_path": "Documentation/Assets/Logo.svg", "documentation": {"kind": "docc", "targets": "library-products", "traits": ["Experimental", "default"]}, "install": "swift-package"},
         {"repository": "swift-json-schema", "logo_path": "Documentation/Assets/Logo.svg", "documentation": {"kind": "docc", "targets": "library-products"}, "install": "swift-package"}
       ]
     },