diff --git a/docs/publishing-automation.md b/docs/publishing-automation.md index 803ccc0..09424bd 100644 --- a/docs/publishing-automation.md +++ b/docs/publishing-automation.md @@ -141,6 +141,21 @@ differently, because their constraints differ: package stays inside the `pydevices` wheel, and a section's `pypi-extras = { ble = ["bleak>=1.0"] }` becomes `pip install "pydevices[ble]"`. An unknown key in a section fails the sync. +- **A directory at the repository root can publish to MIP under its own + name.** A `mip-split.toml` section that is only `top-level = true` publishes + `//` to `micropython/pydevices/`. pydevices' `midiports` + and `usb-*` packages are the first, eleven of them plus `midiports` + (pydevices#226). The directory's own `manifest.py` is the single source for + its description, `require()` lines and files, since a firmware build freezes + the same file; the sync copies the directory without `package.json` and + `README.md` and sets only the manifest's `version` to the release's, so the + index and the lockfile agree. A section takes no other keys. The sync fails + if a listed name isn't a directory with a `manifest.py`, or if a listed + package `require()`s a sibling directory the file doesn't list. The reverse, + a directory nobody listed, is the source repository's own test to catch + (pydevices' `tests/test_mip_split.py`). Available from `publishing-v14`. + mip's `scripts/update_package_list.py` expects these packages once the + lockfile's pydevices ref is 0.7.6 or later (`PROFILE_PACKAGES_SINCE`). - Every publishable entry under `utils/`, plus everything publishable in `board_configs/desktop/`, is bundled into `pydevices-desktop`. - `pydevices-desktop` depends on `pydevices`, so one install gets the complete diff --git a/scripts/synchronize_mip_package.py b/scripts/synchronize_mip_package.py index e421510..9693748 100755 --- a/scripts/synchronize_mip_package.py +++ b/scripts/synchronize_mip_package.py @@ -205,7 +205,7 @@ def render_pydevices_manifest(name: str, version: str, requirements: tuple[str, #: Every key a mip-split.toml section may use. An unknown key fails the sync, #: so a misspelt `own-package` can't quietly ship a package inside pydevices. -MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras"}) +MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras", "top-level"}) def read_split(source_root: Path) -> dict[str, dict]: @@ -233,6 +233,8 @@ def read_own_packages(source_root: Path) -> dict[str, tuple[str, ...]]: """ own: dict[str, tuple[str, ...]] = {} for package, section in read_split(source_root).items(): + if section.get("top-level"): + continue if not section.get("own-package"): if "requires" in section: raise SystemExit( @@ -254,6 +256,8 @@ def read_host_only(source_root: Path) -> dict[str, frozenset[str]]: host_only: dict[str, frozenset[str]] = {} for package, section in declared.items(): + if section.get("top-level"): + continue package_dir = source_root / "lib" / package if not package_dir.is_dir(): raise SystemExit( @@ -310,6 +314,77 @@ def check_no_host_imports(package_dir: Path, host_only: frozenset[str]) -> None: ) +def read_top_level_packages(source_root: Path) -> list[str]: + """Names of the packages that live at the repository root, not in lib/. + + ``midiports`` and the ``usb-*`` packages are top-level directories, each + with its own ``manifest.py`` (also what a firmware build freezes) and its + own ``package.json`` (what ``mip.install("github:...")`` reads). A section + marked ``top-level = true`` publishes that directory to the index under its + own name. The manifest is the one source of what the package is, requires + and ships, so a section may carry nothing else. + """ + names: list[str] = [] + for package, section in read_split(source_root).items(): + if not section.get("top-level"): + continue + extra = sorted(set(section) - {"top-level"}) + if extra: + raise SystemExit( + f"{MIP_SPLIT_FILE} [{package}] is top-level, so it takes no other keys " + f"(remove {', '.join(extra)}). Its manifest.py says what it requires and ships." + ) + package_dir = source_root / package + if not package_dir.is_dir(): + raise SystemExit(f"{MIP_SPLIT_FILE} names top-level package {package!r}, which is not a directory at the repository root") + if not (package_dir / "manifest.py").is_file(): + raise SystemExit(f"top-level package {package!r} has no manifest.py") + if (source_root / "lib" / package).exists(): + raise SystemExit(f"{MIP_SPLIT_FILE} [{package}] is top-level but lib/{package} also exists") + names.append(package) + # A package that requires a sibling directory the file doesn't list would + # install, then fail on import. Say so here, in the pull request's own + # terms, instead of in the index build. + for package in names: + manifest = (source_root / package / "manifest.py").read_text(encoding="utf-8") + for required in re.findall(r'^\s*require\(\s*["\']([^"\']+)["\']', manifest, re.MULTILINE): + if required not in names and (source_root / required / "manifest.py").is_file(): + raise SystemExit( + f"top-level package {package!r} requires {required!r}, which is a directory " + f"here but is not listed in {MIP_SPLIT_FILE}; add [{required}] with top-level = true" + ) + return names + + +_VERSION_LINE = re.compile(r'^(\s*version\s*=\s*)(["\'])[^"\']*\2', re.MULTILINE) + + +def stamp_version(manifest: str, package: str, version: str) -> str: + """The manifest with its metadata version set to the release's. + + A top-level package keeps its own freeze manifest, whose version is the + package's own number. In the index every PyDevices package carries the + release version, which is what the lockfile and the package list compare. + """ + stamped, count = _VERSION_LINE.subn(lambda m: f'{m.group(1)}"{version}"', manifest) + if count != 1: + raise SystemExit(f"{package}/manifest.py needs exactly one version=... line in metadata(), found {count}") + return stamped + + +def synchronize_top_level(source_root: Path, destination_root: Path, package: str, version: str) -> None: + source = source_root / package + destination = destination_root / package + destination.mkdir() + for path in sorted(source.iterdir()): + if path.name == "manifest.py": + (destination / "manifest.py").write_text( + stamp_version(path.read_text(encoding="utf-8"), package, version), encoding="utf-8" + ) + elif path.name != "package.json" and publishable(path): + copy_component(path, destination / path.name) + + def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> None: destination_root = mip_root / "micropython" / "pydevices" if destination_root.parent != mip_root / "micropython": @@ -333,6 +408,7 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No # thing and gets what it always got (pydevices#30). host_only = read_host_only(source_root) own_packages = read_own_packages(source_root) + top_level = read_top_level_packages(source_root) host_components: list[tuple[Path, frozenset[str]]] = [] for source in sorted(filter(publishable, (source_root / "lib").iterdir()), key=lambda path: path.name): names.append(source.stem if source.is_file() else source.name) @@ -382,6 +458,13 @@ def synchronize_pydevices(source_root: Path, mip_root: Path, version: str) -> No if len(names) != len(set(names)): raise SystemExit("lib/ contains colliding module and package names") + # Top-level packages are published beside pydevices, not inside it, and + # under the directory's own name, which is the name mip.install() takes. + for package_name in top_level: + if package_name in names or package_name in {"pydevices", "pydevices-desktop"}: + raise SystemExit(f"top-level package {package_name!r} collides with a lib/ module or a pydevices package") + synchronize_top_level(source_root, destination_root, package_name, version) + (package / "manifest.py").write_text( render_pydevices_manifest("pydevices", version, (), tuple(payloads)), encoding="utf-8" ) diff --git a/tests/test_synchronize_mip_package.py b/tests/test_synchronize_mip_package.py index e771484..a02cbc3 100644 --- a/tests/test_synchronize_mip_package.py +++ b/tests/test_synchronize_mip_package.py @@ -438,6 +438,151 @@ def test_without_the_flag_bledev_ships_inside_pydevices(self) -> None: self.assertIn('package("bledev"', (out / "pydevices" / "manifest.py").read_text(encoding="utf-8")) +class TopLevelPackageTests(unittest.TestCase): + """A directory at the repository root marked top-level publishes under its own name.""" + + CORE = textwrap.dedent( + """ + metadata( + description="usb.core for a host board.", + version="0.1.0", + ) + + package("usb") + """ + ).lstrip() + MSC = textwrap.dedent( + """ + metadata( + description="Mount a USB flash drive.", + version="0.1.0", + ) + + require("usb-core") + package("usb") + """ + ).lstrip() + + def _package(self, source: Path, name: str, manifest: str, files: dict[str, str]) -> None: + pkg = source / name + for relative, text in files.items(): + (pkg / relative).parent.mkdir(parents=True, exist_ok=True) + (pkg / relative).write_text(text, encoding="utf-8") + (pkg / "manifest.py").write_text(manifest, encoding="utf-8") + (pkg / "package.json").write_text('{"urls": []}\n', encoding="utf-8") + (pkg / "README.md").write_text("# readme\n", encoding="utf-8") + + def _source(self, root: Path, split: str) -> Path: + source = root / "source" + write_pydevices_source(source) + self._package(source, "usb-core", self.CORE, {"usb/core.py": "# core\n", "usb_host.py": "# host\n"}) + self._package(source, "usb-host-msc", self.MSC, {"usb/host/msc.py": "# msc\n"}) + (source / "mip-split.toml").write_text(textwrap.dedent(split), encoding="utf-8") + return source + + def _sync(self, root: Path, source: Path) -> subprocess.CompletedProcess[str]: + mip = root / "mip" + write_lockfile(mip, {"pydevices": "PyDevices/pydevices"}) + return run_sync(source, mip, source_name="PyDevices/pydevices", profile="pydevices", version="1.2.3") + + BOTH = """ + [usb-core] + top-level = true + + [usb-host-msc] + top-level = true + """ + + def test_each_package_is_published_with_the_release_version(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + result = self._sync(root, self._source(root, self.BOTH)) + self.assertEqual(result.returncode, 0, result.stderr) + out = root / "mip" / "micropython" / "pydevices" + msc = (out / "usb-host-msc" / "manifest.py").read_text(encoding="utf-8") + self.assertIn('version="1.2.3"', msc) + self.assertNotIn("0.1.0", msc) + self.assertIn('require("usb-core")', msc) + self.assertIn("Mount a USB flash drive.", msc) + self.assertTrue((out / "usb-host-msc" / "usb" / "host" / "msc.py").is_file()) + self.assertTrue((out / "usb-core" / "usb_host.py").is_file()) + for name in ("usb-core", "usb-host-msc"): + self.assertFalse((out / name / "package.json").exists()) + self.assertFalse((out / name / "README.md").exists()) + + def test_the_pydevices_package_does_not_carry_them(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + result = self._sync(root, self._source(root, self.BOTH)) + self.assertEqual(result.returncode, 0, result.stderr) + out = root / "mip" / "micropython" / "pydevices" + manifest = (out / "pydevices" / "manifest.py").read_text(encoding="utf-8") + self.assertNotIn("usb", manifest) + self.assertNotIn("usb", (out / "pydevices-desktop" / "manifest.py").read_text(encoding="utf-8")) + + def test_a_required_sibling_missing_from_the_toml_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + result = self._sync(root, self._source(root, "[usb-host-msc]\ntop-level = true\n")) + self.assertNotEqual(result.returncode, 0) + self.assertIn("requires 'usb-core'", result.stderr) + self.assertIn("not listed in mip-split.toml", result.stderr) + + def test_a_name_that_is_not_a_directory_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + result = self._sync(root, self._source(root, self.BOTH + "\n[usb-gone]\ntop-level = true\n")) + self.assertNotEqual(result.returncode, 0) + self.assertIn("'usb-gone', which is not a directory", result.stderr) + + def test_a_package_without_a_manifest_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source(root, self.BOTH) + (source / "usb-core" / "manifest.py").unlink() + result = self._sync(root, source) + self.assertNotEqual(result.returncode, 0) + self.assertIn("has no manifest.py", result.stderr) + + def test_other_keys_in_a_top_level_section_fail(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source(root, '[usb-core]\ntop-level = true\nrequires = ["aioble"]\n[usb-host-msc]\ntop-level = true\n') + result = self._sync(root, source) + self.assertNotEqual(result.returncode, 0) + self.assertIn("takes no other keys (remove requires)", result.stderr) + + def test_a_manifest_without_exactly_one_version_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source(root, self.BOTH) + (source / "usb-core" / "manifest.py").write_text('metadata(description="x")\npackage("usb")\n', encoding="utf-8") + result = self._sync(root, source) + self.assertNotEqual(result.returncode, 0) + self.assertIn("exactly one version=", result.stderr) + + def test_a_name_in_lib_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = self._source(root, self.BOTH) + write_lib_package(source, "usb-core") + result = self._sync(root, source) + self.assertNotEqual(result.returncode, 0) + self.assertIn("also exists", result.stderr) + + def test_without_any_the_pydevices_sync_is_unchanged(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "source" + write_pydevices_source(source) + result = self._sync(root, source) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + sorted(p.name for p in (root / "mip" / "micropython" / "pydevices").iterdir()), + ["pydevices", "pydevices-desktop"], + ) + + class PypiExtrasTests(unittest.TestCase): def test_extras_reach_the_pydevices_pyproject(self) -> None: with tempfile.TemporaryDirectory() as tmp: