MIP sync: a top-level directory can publish under its own name

docs/publishing-automation.md

@@ -141,6 +141,21 @@ differently, because their constraints differ:
   package stays inside the `pydevices` wheel, and a section's
   `pypi-extras = { ble = ["bleak>=1.0"] }` becomes
   `pip install "pydevices[ble]"`. An unknown key in a section fails the sync.
+- **A directory at the repository root can publish to MIP under its own
+  name.** A `mip-split.toml` section that is only `top-level = true` publishes
+  `<repo>/<name>/` to `micropython/pydevices/<name>`. pydevices' `midiports`
+  and `usb-*` packages are the first, eleven of them plus `midiports`
+  (pydevices#226). The directory's own `manifest.py` is the single source for
+  its description, `require()` lines and files, since a firmware build freezes
+  the same file; the sync copies the directory without `package.json` and
+  `README.md` and sets only the manifest's `version` to the release's, so the
+  index and the lockfile agree. A section takes no other keys. The sync fails
+  if a listed name isn't a directory with a `manifest.py`, or if a listed
+  package `require()`s a sibling directory the file doesn't list. The reverse,
+  a directory nobody listed, is the source repository's own test to catch
+  (pydevices' `tests/test_mip_split.py`). Available from `publishing-v14`.
+  mip's `scripts/update_package_list.py` expects these packages once the
+  lockfile's pydevices ref is 0.7.6 or later (`PROFILE_PACKAGES_SINCE`).
 - Every publishable entry under `utils/`, plus everything publishable in
   `board_configs/desktop/`, is bundled into `pydevices-desktop`.
 - `pydevices-desktop` depends on `pydevices`, so one install gets the complete

scripts/synchronize_mip_package.py

@@ -205,7 +205,7 @@ def render_pydevices_manifest(name: str, version: str, requirements: tuple[str,
 
 #: Every key a mip-split.toml section may use. An unknown key fails the sync,
 #: so a misspelt `own-package` can't quietly ship a package inside pydevices.
-MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras"})
+MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras", "top-level"})
 
 
 def read_split(source_root: Path) -> dict[str, dict]:
@@ -233,6 +233,8 @@ def read_own_packages(source_root: Path) -> dict[str, tuple[str, ...]]:
     """
     own: dict[str, tuple[str, ...]] = {}
     for package, section in read_split(source_root).items():
+        if section.get("top-level"):
+            continue
         if not section.get("own-package"):
             if "requires" in section:
                 raise SystemExit(
@@ -254,6 +256,8 @@ def read_host_only(source_root: Path) -> dict[str, frozenset[str]]:
 
     host_only: dict[str, frozenset[str]] = {}
     for package, section in declared.items():
+        if section.get("top-level"):
+            continue
         package_dir = source_root / "lib" / package
         if not package_dir.is_dir():
             raise SystemExit(
@@ -310,6 +314,77 @@ def check_no_host_imports(package_dir: Path, host_only: frozenset[str]) -> None:
                 )
 
 
+def read_top_level_packages(source_root: Path) -> list[str]:
+    """Names of the packages that live at the repository root, not in lib/.
+
+    ``midiports`` and the ``usb-*`` packages are top-level directories, each
+    with its own ``manifest.py`` (also what a firmware build freezes) and its
+    own ``package.json`` (what ``mip.install("github:...")`` reads). A section
+    marked ``top-level = true`` publishes that directory to the index under its
+    own name. The manifest is the one source of what the package is, requires
+    and ships, so a section may carry nothing else.
+    """
+    names: list[str] = []
+    for package, section in read_split(source_root).items():
+        if not section.get("top-level"):
+            continue
+        extra = sorted(set(section) - {"top-level"})
+        if extra:
+            raise SystemExit(
+                f"{MIP_SPLIT_FILE} [{package}] is top-level, so it takes no other keys "
+                f"(remove {', '.join(extra)}). Its manifest.py says what it requires and ships."
+            )
+        package_dir = source_root / package
+        if not package_dir.is_dir():
+            raise SystemExit(f"{MIP_SPLIT_FILE} names top-level package {package!r}, which is not a directory at the repository root")
+        if not (package_dir / "manifest.py").is_file():
+            raise SystemExit(f"top-level package {package!r} has no manifest.py")
+        if (source_root / "lib" / package).exists():
+            raise SystemExit(f"{MIP_SPLIT_FILE} [{package}] is top-level but lib/{package} also exists")
+        names.append(package)
+    # A package that requires a sibling directory the file doesn't list would
+    # install, then fail on import. Say so here, in the pull request's own
+    # terms, instead of in the index build.
+    for package in names:
+        manifest = (source_root / package / "manifest.py").read_text(encoding="utf-8")
+        for required in re.findall(r'^\s*require\(\s*["\']([^"\']+)["\']', manifest, re.MULTILINE):
+            if required not in names and (source_root / required / "manifest.py").is_file():
+                raise SystemExit(
+                    f"top-level package {package!r} requires {required!r}, which is a directory "
+                    f"here but is not listed in {MIP_SPLIT_FILE}; add [{required}] with top-level = true"
+                )
+    return names
+
+
+_VERSION_LINE = re.compile(r'^(\s*version\s*=\s*)(["\'])[^"\']*\2', re.MULTILINE)
+
+
+def stamp_version(manifest: str, package: str, version: str) -> str:
+    """The manifest with its metadata version set to the release's.
+
+    A top-level package keeps its own freeze manifest, whose version is the
+    package's own number. In the index every PyDevices package carries the
+    release version, which is what the lockfile and the package list compare.
+    """
+    stamped, count = _VERSION_LINE.subn(lambda m: f'{m.group(1)}"{version}"', manifest)
+    if count != 1:
+        raise SystemExit(f"{package}/manifest.py needs exactly one version=... line in metadata(), found {count}")
+    return stamped
+
+
+def synchronize_top_level(source_root: Path, destination_root: Path, package: str, version: str) -> None:
+    source = source_root / package
+    destination = destination_root / package
+    destination.mkdir()
+    for path in sorted(source.iterdir()):
+        if path.name == "manifest.py":
+            (destination / "manifest.py").write_text(
+                stamp_version(path.read_text(encoding="utf-8"), package, version), encoding="utf-8"
+            )
+        elif path.name != "package.json" and publishable(path):
+            copy_component(path, destination / path.name)
+
+
 def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> None:
     destination_root = mip_root / "micropython" / "pydevices"
     if destination_root.parent != mip_root / "micropython":
@@ -333,6 +408,7 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No
     # thing and gets what it always got (pydevices#30).
     host_only = read_host_only(source_root)
     own_packages = read_own_packages(source_root)
+    top_level = read_top_level_packages(source_root)
     host_components: list[tuple[Path, frozenset[str]]] = []
     for source in sorted(filter(publishable, (source_root / "lib").iterdir()), key=lambda path: path.name):
         names.append(source.stem if source.is_file() else source.name)
@@ -382,6 +458,13 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No
     if len(names) != len(set(names)):
         raise SystemExit("lib/ contains colliding module and package names")
 
+    # Top-level packages are published beside pydevices, not inside it, and
+    # under the directory's own name, which is the name mip.install() takes.
+    for package_name in top_level:
+        if package_name in names or package_name in {"pydevices", "pydevices-desktop"}:
+            raise SystemExit(f"top-level package {package_name!r} collides with a lib/ module or a pydevices package")
+        synchronize_top_level(source_root, destination_root, package_name, version)
+
     (package / "manifest.py").write_text(
         render_pydevices_manifest("pydevices", version, (), tuple(payloads)), encoding="utf-8"
     )

tests/test_synchronize_mip_package.py

@@ -438,6 +438,151 @@ def test_without_the_flag_bledev_ships_inside_pydevices(self) -> None:
             self.assertIn('package("bledev"', (out / "pydevices" / "manifest.py").read_text(encoding="utf-8"))
 
 
+class TopLevelPackageTests(unittest.TestCase):
+    """A directory at the repository root marked top-level publishes under its own name."""
+
+    CORE = textwrap.dedent(
+        """
+        metadata(
+            description="usb.core for a host board.",
+            version="0.1.0",
+        )
+
+        package("usb")
+        """
+    ).lstrip()
+    MSC = textwrap.dedent(
+        """
+        metadata(
+            description="Mount a USB flash drive.",
+            version="0.1.0",
+        )
+
+        require("usb-core")
+        package("usb")
+        """
+    ).lstrip()
+
+    def _package(self, source: Path, name: str, manifest: str, files: dict[str, str]) -> None:
+        pkg = source / name
+        for relative, text in files.items():
+            (pkg / relative).parent.mkdir(parents=True, exist_ok=True)
+            (pkg / relative).write_text(text, encoding="utf-8")
+        (pkg / "manifest.py").write_text(manifest, encoding="utf-8")
+        (pkg / "package.json").write_text('{"urls": []}\n', encoding="utf-8")
+        (pkg / "README.md").write_text("# readme\n", encoding="utf-8")
+
+    def _source(self, root: Path, split: str) -> Path:
+        source = root / "source"
+        write_pydevices_source(source)
+        self._package(source, "usb-core", self.CORE, {"usb/core.py": "# core\n", "usb_host.py": "# host\n"})
+        self._package(source, "usb-host-msc", self.MSC, {"usb/host/msc.py": "# msc\n"})
+        (source / "mip-split.toml").write_text(textwrap.dedent(split), encoding="utf-8")
+        return source
+
+    def _sync(self, root: Path, source: Path) -> subprocess.CompletedProcess[str]:
+        mip = root / "mip"
+        write_lockfile(mip, {"pydevices": "PyDevices/pydevices"})
+        return run_sync(source, mip, source_name="PyDevices/pydevices", profile="pydevices", version="1.2.3")
+
+    BOTH = """
+        [usb-core]
+        top-level = true
+
+        [usb-host-msc]
+        top-level = true
+        """
+
+    def test_each_package_is_published_with_the_release_version(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            result = self._sync(root, self._source(root, self.BOTH))
+            self.assertEqual(result.returncode, 0, result.stderr)
+            out = root / "mip" / "micropython" / "pydevices"
+            msc = (out / "usb-host-msc" / "manifest.py").read_text(encoding="utf-8")
+            self.assertIn('version="1.2.3"', msc)
+            self.assertNotIn("0.1.0", msc)
+            self.assertIn('require("usb-core")', msc)
+            self.assertIn("Mount a USB flash drive.", msc)
+            self.assertTrue((out / "usb-host-msc" / "usb" / "host" / "msc.py").is_file())
+            self.assertTrue((out / "usb-core" / "usb_host.py").is_file())
+            for name in ("usb-core", "usb-host-msc"):
+                self.assertFalse((out / name / "package.json").exists())
+                self.assertFalse((out / name / "README.md").exists())
+
+    def test_the_pydevices_package_does_not_carry_them(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            result = self._sync(root, self._source(root, self.BOTH))
+            self.assertEqual(result.returncode, 0, result.stderr)
+            out = root / "mip" / "micropython" / "pydevices"
+            manifest = (out / "pydevices" / "manifest.py").read_text(encoding="utf-8")
+            self.assertNotIn("usb", manifest)
+            self.assertNotIn("usb", (out / "pydevices-desktop" / "manifest.py").read_text(encoding="utf-8"))
+
+    def test_a_required_sibling_missing_from_the_toml_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            result = self._sync(root, self._source(root, "[usb-host-msc]\ntop-level = true\n"))
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("requires 'usb-core'", result.stderr)
+            self.assertIn("not listed in mip-split.toml", result.stderr)
+
+    def test_a_name_that_is_not_a_directory_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            result = self._sync(root, self._source(root, self.BOTH + "\n[usb-gone]\ntop-level = true\n"))
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("'usb-gone', which is not a directory", result.stderr)
+
+    def test_a_package_without_a_manifest_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(root, self.BOTH)
+            (source / "usb-core" / "manifest.py").unlink()
+            result = self._sync(root, source)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("has no manifest.py", result.stderr)
+
+    def test_other_keys_in_a_top_level_section_fail(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(root, '[usb-core]\ntop-level = true\nrequires = ["aioble"]\n[usb-host-msc]\ntop-level = true\n')
+            result = self._sync(root, source)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("takes no other keys (remove requires)", result.stderr)
+
+    def test_a_manifest_without_exactly_one_version_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(root, self.BOTH)
+            (source / "usb-core" / "manifest.py").write_text('metadata(description="x")\npackage("usb")\n', encoding="utf-8")
+            result = self._sync(root, source)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("exactly one version=", result.stderr)
+
+    def test_a_name_in_lib_fails(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = self._source(root, self.BOTH)
+            write_lib_package(source, "usb-core")
+            result = self._sync(root, source)
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("also exists", result.stderr)
+
+    def test_without_any_the_pydevices_sync_is_unchanged(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            source = root / "source"
+            write_pydevices_source(source)
+            result = self._sync(root, source)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertEqual(
+                sorted(p.name for p in (root / "mip" / "micropython" / "pydevices").iterdir()),
+                ["pydevices", "pydevices-desktop"],
+            )
+
+
 class PypiExtrasTests(unittest.TestCase):
     def test_extras_reach_the_pydevices_pyproject(self) -> None:
         with tempfile.TemporaryDirectory() as tmp: