Expose repos as github resource in 3008.3+

.pre-commit-hooks/make-autodocs.py

@@ -1,5 +1,5 @@
 import ast
-import os.path
+import posixpath
 import subprocess
 from pathlib import Path
 
@@ -24,48 +24,59 @@ def _find_virtualname(path):
                 continue
             break
     else:
+        if path.name == "__init__.py":
+            # Resource type packages carry their connection module there
+            path = path.parent
         virtualname = path.with_suffix("").name
     return virtualname
 
 
-def write_module(rst_path, path, use_virtualname=True):
-    if use_virtualname:
-        virtualname = "``" + _find_virtualname(path) + "``"
-    else:
-        virtualname = make_import_path(path)
-    header_len = len(virtualname)
+def _kind_title(kind):
+    if kind == "utils":
+        return "Utilities"
+    if kind.lower() == "modules":
+        return "Execution Modules"
+    return (kind.rstrip("s") + " modules").title()
+
+
+def write_file(rst_path, contents):
+    if not rst_path.exists() or rst_path.read_text() != contents:
+        print(rst_path)
+        rst_path.write_text(contents)
+        return True
+    return False
+
+
+def write_module(rst_path, header, paths):
+    header_len = len(header)
     # The check-merge-conflict pre-commit hook chokes here:
     # https://github.com/pre-commit/pre-commit-hooks/issues/100
     if header_len == 7:
         header_len += 1
+    automodules = "\n".join(f""".. automodule:: {make_import_path(path)}
+    :members:
+""" for path in paths)
     module_contents = f"""\
-{virtualname}
+{header}
 {'='*header_len}
 
-.. automodule:: {make_import_path(path)}
-    :members:
-"""
-    if not rst_path.exists() or rst_path.read_text() != module_contents:
-        print(rst_path)
-        rst_path.write_text(module_contents)
-        return True
-    return False
+{automodules}"""
+    return write_file(rst_path, module_contents)
 
 
 def write_index(index_rst, import_paths, kind):
-    if kind == "utils":
-        header_text = "Utilities"
-        common_path = os.path.commonpath(tuple(x.replace(".", "/") for x in import_paths)).replace(
-            "/", "."
-        )
+    header_text = _kind_title(kind)
+    if kind in ("utils", "resources"):
+        # These can contain nested modules, so find the common parent
+        common_path = posixpath.commonpath(
+            tuple(x.replace(".", "/") for x in import_paths)
+        ).replace("/", ".")
+
         if any(x == common_path for x in import_paths):
             common_path = common_path[: common_path.rfind(".")]
     else:
-        header_text = (
-            "execution modules" if kind.lower() == "modules" else kind.rstrip("s") + " modules"
-        )
         common_path = import_paths[0][: import_paths[0].rfind(".")]
-    header = f"{'_'*len(header_text)}\n{header_text.title()}\n{'_'*len(header_text)}"
+    header = f"{'_'*len(header_text)}\n{header_text}\n{'_'*len(header_text)}"
     index_contents = f"""\
 .. all-saltext.github.{kind}:
 
@@ -78,11 +89,7 @@ def write_index(index_rst, import_paths, kind):
 
 {chr(10).join(sorted('    '+p[len(common_path)+1:] for p in import_paths))}
 """
-    if not index_rst.exists() or index_rst.read_text() != index_contents:
-        print(index_rst)
-        index_rst.write_text(index_contents)
-        return True
-    return False
+    return write_file(index_rst, index_contents)
 
 
 def make_import_path(path):
@@ -91,31 +98,209 @@ def make_import_path(path):
     return ".".join(path.relative_to(repo_path / "src").with_suffix("").parts)
 
 
+tracked = set(
+    subprocess.check_output(["git", "ls-files", "-z"], text=True, cwd=repo_path).split("\x00")
+)
+
 for path in src_dir.glob("*/*.py"):
-    if path.name != "__init__.py":
+    if path.name != "__init__.py" and path.relative_to(repo_path).as_posix() in tracked:
         kind = path.parent.name
-        if kind != "utils":
+        if kind not in ("utils", "resources"):
             docs_by_kind.setdefault(kind, set()).add(path)
 
 # Utils can have subdirectories, treat them separately
 for path in (src_dir / "utils").rglob("*.py"):
+    if path.relative_to(repo_path).as_posix() not in tracked:
+        continue
     if path.name == "__init__.py" and not path.read_text():
         continue
     docs_by_kind.setdefault("utils", set()).add(path)
 
+# Resource types can be packages containing per-type loader overlay
+# directories (resources/<rtype>/{modules,states,grains,...}), treat
+# them separately
+for path in (src_dir / "resources").rglob("*.py"):
+    if path.relative_to(repo_path).as_posix() not in tracked:
+        continue
+    if path.name == "__init__.py":
+        # Only a resource type package's own __init__.py is a loader
+        # module (the connection module), the other ones are packaging
+        # artifacts.
+        if path.parent.parent != src_dir / "resources" or not path.read_text():
+            continue
+    docs_by_kind.setdefault("resources", set()).add(path)
+
+
+def _group_by_virtualname(paths):
+    grouped = {}
+    for path in sorted(paths):
+        grouped.setdefault(_find_virtualname(path), []).append(path)
+    # Make the module whose file name matches the virtualname the
+    # canonical one, i.e. the first in the group and the one the
+    # rst file is named after.
+    for virtualname, group in grouped.items():
+        for path in group:
+            stem = path.parent.name if path.name == "__init__.py" else path.stem
+            if stem in (virtualname, virtualname + "_mod"):
+                group.remove(path)
+                group.insert(0, path)
+                break
+    return grouped
+
+
+def _overlay_order(overlay_kind):
+    """
+    Order the per-type loader overlay sections like Salt's docs do,
+    unknown kinds last in alphabetical order.
+    """
+    try:
+        return (("modules", "states", "grains").index(overlay_kind), overlay_kind)
+    except ValueError:
+        return (99, overlay_kind)
+
+
+def _process_resources(kind_path, paths):
+    """
+    Document each resource type with its connection module page (headed
+    by its virtualname) as the entry point. Each per-type loader overlay
+    directory (``resources/<rtype>/<loader_dir>/``) gets its own index
+    page ("Execution Modules", "State Modules", ...) hanging off the
+    type page, so the overlay modules nest under a per-kind subheading
+    in the navigation instead of being appended to one flat list.
+
+    There is no kind index: the root document's "Resources" toctree
+    globs ``ref/resources/*``, which picks up exactly the type pages
+    because everything else is written to a ``<rtype>/`` subdirectory
+    (toctree globs do not cross directories).
+    """
+    resources_dir = src_dir / "resources"
+    types = {}
+    for path in sorted(paths):
+        rel = path.relative_to(resources_dir).parts
+        if len(rel) == 1:
+            # Flat connection module
+            types.setdefault(path.stem, {"conn": None, "overlays": {}})["conn"] = path
+        elif len(rel) == 2 and path.name == "__init__.py":
+            # Resource type package's connection module
+            types.setdefault(rel[0], {"conn": None, "overlays": {}})["conn"] = path
+        else:
+            overlays = types.setdefault(rel[0], {"conn": None, "overlays": {}})["overlays"]
+            overlays.setdefault(rel[1], set()).add(path)
+
+    changed = False
+    index_paths = []
+    written_rst_paths = set()
+    kind_path.mkdir(parents=True, exist_ok=True)
+    for rtype, info in sorted(types.items()):
+        conn = info["conn"]
+        blocks = []
+        if conn is not None:
+            header = f"``{_find_virtualname(conn)}``"
+            type_import_path = make_import_path(conn)
+            blocks.append(f".. automodule:: {type_import_path}\n    :members:\n")
+        else:
+            # Overlay modules without a connection module (unusual)
+            header = f"``{rtype}``"
+            type_import_path = f"saltext.github.resources.{rtype}"
+        header_len = len(header)
+        if header_len == 7:
+            header_len += 1
+        blocks.insert(0, f"{header}\n{'='*header_len}\n")
+
+        type_path = kind_path / rtype
+        if info["overlays"]:
+            type_path.mkdir(parents=True, exist_ok=True)
+        overlay_index_paths = []
+        for overlay_kind in sorted(info["overlays"], key=_overlay_order):
+            overlay_import_path = f"saltext.github.resources.{rtype}.{overlay_kind}"
+            entries = []
+            for virtualname, group in _group_by_virtualname(info["overlays"][overlay_kind]).items():
+                import_path = make_import_path(group[0])
+                rst_path = type_path / (import_path + ".rst")
+                written_rst_paths.add(rst_path)
+                changed |= write_module(rst_path, f"``{virtualname}``", group)
+                entries.append(import_path[len(overlay_import_path) + 1 :])
+            # One index page per overlay kind: its title provides the
+            # per-kind subheading the overlay modules nest under in the
+            # navigation.
+            title = _kind_title(overlay_kind)
+            overlay_index_contents = f"""\
+{title}
+{'='*len(title)}
+
+.. currentmodule:: {overlay_import_path}
+
+.. autosummary::
+    :toctree:
+
+{chr(10).join('    ' + entry for entry in sorted(entries))}
+"""
+            overlay_index_rst = type_path / (overlay_import_path + ".rst")
+            written_rst_paths.add(overlay_index_rst)
+            changed |= write_file(overlay_index_rst, overlay_index_contents)
+            overlay_index_paths.append(f"{rtype}/{overlay_import_path}")
+
+        if overlay_index_paths:
+            heading = "Provided Modules"
+            blocks.append(f"{heading}\n{'-'*len(heading)}\n")
+            blocks.append(
+                ".. toctree::\n    :maxdepth: 1\n\n"
+                + "\n".join("    " + p for p in overlay_index_paths)
+                + "\n"
+            )
+
+        type_rst_path = kind_path / (type_import_path + ".rst")
+        written_rst_paths.add(type_rst_path)
+        changed |= write_file(type_rst_path, "\n".join(blocks))
+        index_paths.append(type_import_path)
+    return changed, index_paths, written_rst_paths
+
+
 for kind in docs_by_kind:
     kind_path = doc_dir / "ref" / kind
     index_rst = kind_path / "index.rst"
     import_paths = []
-    for path in sorted(docs_by_kind[kind]):
-        import_path = make_import_path(path)
-        import_paths.append(import_path)
-        rst_path = kind_path / (import_path + ".rst")
-        rst_path.parent.mkdir(parents=True, exist_ok=True)
-        change = write_module(rst_path, path, use_virtualname=kind != "utils")
+    written_rst_paths = set()
+    if kind == "resources":
+        change, import_paths, written_rst_paths = _process_resources(kind_path, docs_by_kind[kind])
         changed_something = changed_something or change
+    else:
+        if kind == "utils":
+            # Utils are documented under their import path, no grouping necessary
+            grouped = {make_import_path(path): [path] for path in sorted(docs_by_kind[kind])}
+        else:
+            grouped = {
+                f"``{virtualname}``": paths
+                for virtualname, paths in _group_by_virtualname(docs_by_kind[kind]).items()
+            }
+        for header, paths in grouped.items():
+            import_path = make_import_path(paths[0])
+            import_paths.append(import_path)
+            rst_path = kind_path / (import_path + ".rst")
+            rst_path.parent.mkdir(parents=True, exist_ok=True)
+            written_rst_paths.add(rst_path)
+            change = write_module(rst_path, header, paths)
+            changed_something = changed_something or change
 
-    write_index(index_rst, import_paths, kind)
+    # Remove stale generated rst files, e.g. after a module has been renamed/
+    # removed or merged into another one's docs by sharing its virtualname.
+    # Resource type docs nest in per-type subdirectories, search recursively.
+    stale_glob = kind_path.rglob if kind == "resources" else kind_path.glob
+    for rst_path in stale_glob(f"saltext.github.{kind}.*.rst"):
+        if rst_path not in written_rst_paths:
+            print(f"Removing stale {rst_path}")
+            rst_path.unlink()
+            changed_something = True
+
+    if kind == "resources":
+        # No kind index: the root document's "Resources" toctree globs
+        # the per-type pages directly.
+        if index_rst.exists():
+            print(f"Removing stale {index_rst}")
+            index_rst.unlink()
+            changed_something = True
+    else:
+        changed_something |= write_index(index_rst, sorted(import_paths), kind)
 
 
 # Ensure pre-commit realizes we did something

changelog/+resource_type.added.md

@@ -0,0 +1 @@
+Added the {py:mod}`github <saltext.github.resources.github>` {ref}`Salt resource type <repo-resources>` for Salt 3008.3+, exposing each GitHub repository as an individually targetable resource with per-repository grains and repository-implicit execution and state modules

changelog/+sdb-profile-values.added.md

@@ -1 +1 @@
-Profile configuration values containing SDB URIs (e.g. `token: sdb://osenv/GITHUB_TOKEN`) are now rendered on access via `config.get`. Salt itself only renders them inside the minion/master configuration during startup, so this makes them work in pillar-sourced profiles as well.
+Profile configuration values containing SDB URIs (e.g. `token: sdb://osenv/GITHUB_TOKEN`) are now rendered on access. Salt itself only renders them inside the minion/master configuration during startup, so this makes them work in pillar-sourced profiles as well.

changelog/+user_owned_repos.added.md

@@ -1 +1 @@
-The new modules support repositories of any owner. Profiles take an {ghconf}`owner` (organization or user; preferred over the legacy `org_name` alias), which defaults to the token's user when unset. Repository listing and creation live on the `github_repo` module (`list`/`create`), and all of its functions and states as well as all functions of the (now strictly organization-scoped) `github_org` module accept a per-call `owner` override, so repositories, teams, members and rulesets of foreign organizations can be managed as well. Due to GitHub API limitations, repository creation is only supported for organizations and the authenticated user, private repositories of other users are not listable, tokens without an associated user account (e.g. GitHub App installation tokens) require an explicit `owner` and teams always belong to an organization
+The new modules support repositories of any owner. Profiles take an {ghconf}`owner` (organization or user; preferred over the legacy `org_name` alias), which defaults to the token's user when unset. Repository listing and creation live on the {py:mod}`github_repo <saltext.github.modules.github_repo>` module ({py:func}`list <saltext.github.modules.github_repo.list_>` / {py:func}`create <saltext.github.modules.github_repo.create>`), and all of its functions and states as well as all functions of the (now strictly organization-scoped) {py:mod}`github_org <saltext.github.modules.github_org>` module accept a per-call `owner` override, so repositories, teams, members and rulesets of foreign organizations can be managed as well. Due to GitHub API limitations, repository creation is only supported for organizations and the authenticated user, private repositories of other users are not listable, tokens without an associated user account (e.g. GitHub App installation tokens) require an explicit `owner` and teams always belong to an organization

docs/index.rst

@@ -12,6 +12,9 @@ Salt Extension for interacting with GitHub:
   query their issues, milestones and pull requests via the
   :py:mod:`github_repo <saltext.github.modules.github_repo>` execution
   and state modules.
+* On Salt 3008.3+, optionally manage each repository as an individually
+  targetable Salt resource with per-repository grains, execution and
+  state modules via the ``github`` :ref:`resource type <repo-resources>`.
 * Works against GitHub Enterprise Server instances via the
   :ghconf:`api_url` profile parameter.
 
@@ -22,6 +25,7 @@ Salt Extension for interacting with GitHub:
 
   topics/installation
   topics/configuration
+  topics/repo_resources
 
 .. toctree::
   :maxdepth: 2
@@ -32,6 +36,14 @@ Salt Extension for interacting with GitHub:
   ref/states/index
   ref/utils/index
 
+.. toctree::
+  :maxdepth: 3
+  :caption: Provided Resources
+  :hidden:
+  :glob:
+
+  ref/resources/*
+
 .. toctree::
   :maxdepth: 2
   :caption: Reference

docs/ref/resources/github/saltext.github.resources.github.modules.actions.rst

@@ -0,0 +1,5 @@
+``actions``
+===========
+
+.. automodule:: saltext.github.resources.github.modules.actions
+    :members:

docs/ref/resources/github/saltext.github.resources.github.modules.deploy_key.rst

@@ -0,0 +1,5 @@
+``deploy_key``
+==============
+
+.. automodule:: saltext.github.resources.github.modules.deploy_key
+    :members:

docs/ref/resources/github/saltext.github.resources.github.modules.environment.rst

@@ -0,0 +1,5 @@
+``environment``
+===============
+
+.. automodule:: saltext.github.resources.github.modules.environment
+    :members:

docs/ref/resources/github/saltext.github.resources.github.modules.issue.rst

@@ -0,0 +1,5 @@
+``issue``
+=========
+
+.. automodule:: saltext.github.resources.github.modules.issue
+    :members:

docs/ref/resources/github/saltext.github.resources.github.modules.label.rst

@@ -0,0 +1,5 @@
+``label``
+=========
+
+.. automodule:: saltext.github.resources.github.modules.label
+    :members:

docs/ref/resources/github/saltext.github.resources.github.modules.milestone.rst

@@ -0,0 +1,5 @@
+``milestone``
+=============
+
+.. automodule:: saltext.github.resources.github.modules.milestone
+    :members: