diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..b828a4a3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# YAML is linted with yamllint's default new-lines rule (LF). The Windows +# runners check out with core.autocrlf=true, which turned every LF file into +# CRLF and failed the rule on line 1 of the first file -- except a file that +# already carried a stray CR, which autocrlf leaves alone. Pin the encoding +# instead of guessing at the checkout's, so the same bytes are linted on +# every OS. +*.yml text eol=lf +*.yaml text eol=lf diff --git a/.github/workflows/auto-assign.yml b/.github/workflows/auto-assign.yml index 60639c32..5f438f5a 100644 --- a/.github/workflows/auto-assign.yml +++ b/.github/workflows/auto-assign.yml @@ -30,4 +30,4 @@ jobs: repo: context.repo.repo, issue_number: context.payload.pull_request.number, assignees: [context.payload.pull_request.user.login] - }); + }); diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index e8507e51..a201167d 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -21,14 +21,14 @@ jobs: security-events: write id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: persist-credentials: false - - uses: ossf/scorecard-action@v2.4.0 + - uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif publish_results: true - - uses: github/codeql-action/upload-sarif@v3 + - uses: github/codeql-action/upload-sarif@faaca9a8f6edddba5725ffe5adefdab6669a2eca # v3 with: sarif_file: results.sarif diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ba01756..9a339e52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,26 @@ `cjson` (v1.7.18), `nanopb` (v0.4.9.1), `lvgl` (v9.2.2), `tinyusb` (v0.18.0), and `unity` (v2.6.1). ### Fixed +- **CI on master runs to completion again.** ruff stopped the pipeline at its + first step on findings the merges had introduced (an F811 duplicate import, + W292, E402). `PackageRecipe.to_dict()` -- defined by #111, deleted by #112's + replay of the same file -- is restored; nine `test_index_sync` cases and mypy + had failed without it. It now emits `install_args`, which the original never + did, and hands back copies of its list fields rather than the live lists; + the index sync mapping carries `install_args` through to the cached recipe. + The vendored `core/eos/docs/three-way-alignment.md` is reverted to its pin + (the correction #109 made there is filed upstream as embeddedos-org/eos#149). + The OSSF Scorecard action moved to the ghcr.io-hosted release and is pinned + by commit. yamllint on the Windows legs: YAML is pinned to LF in + `.gitattributes`, so an existing Windows clone needs its files checked out + again once (`git rm --cached -r . && git reset --hard HEAD`, or a re-clone; + see CONTRIBUTING.md). On Python 3.10 and 3.11 `ebuild/plugins/__init__.py` + now type-checks: the `entry_points()` fallback is spelled out with a cast + instead of a `# type: ignore` naming the wrong error code. + (`ebuild/packages/recipe.py`, `ebuild/packages/index_sync.py`, + `ebuild/plugins/__init__.py`, `core/eos/docs/three-way-alignment.md`, + `.github/workflows/scorecard.yml`, `.gitattributes`, `.yamllint.yml`, and + the three lint-fixed test files.) - **`ebuild test` now finds Windows test binaries.** The Ninja edge for a native `type: test` target already carried the platform suffix (`_exe_suffix()` names it `.exe` on Windows), but `ebuild test` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ecda1e24..29dfb670 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,14 @@ This certifies you have the right to submit the code under the MIT license. 6. Commit with DCO sign-off: git commit -s 7. Push and create Pull Request +**Windows contributors:** `.gitattributes` pins `*.yml` and `*.yaml` to LF so +yamllint sees the same bytes on every platform. The attribute governs future +checkouts, not files already sitting in a working tree, and `git add +--renormalize .` rewrites only the index, never the files. After pulling that +change, from a clean tree run `git rm --cached -r . && git reset --hard HEAD` +once (or re-clone) so the YAML files are checked out again as LF; otherwise +yamllint will still see CRLF locally. + ## Coding Standards ### C (ISO C11) diff --git a/core/eos/docs/three-way-alignment.md b/core/eos/docs/three-way-alignment.md index 767c720f..7f9c8c15 100644 --- a/core/eos/docs/three-way-alignment.md +++ b/core/eos/docs/three-way-alignment.md @@ -8,7 +8,7 @@ This document tracks alignment between all three EoS components to ensure they r | Dimension | eos | eboot | ebuild | Status | |-----------|-----|-------|--------|--------| -| Board definitions | 84 board YAMLs in `eos/boards/` (upstream) | 83 board dirs / 138 `eboot_add_board()` calls in `eboot/boards/` (upstream, pinned rev) | `TARGET_ARCH` 14 + `MCU_TO_EBOOT_BOARD` 138 in `ebuild/sdk_generator.py` | ⚠️ Unverified: 84 vs 83 vs 14/138 — three inventories describe the same set and nothing cross-checks them; see the resolver drift note in PR #109 | +| Board definitions | 25 YAML files in `eos/boards/` | 25 board ports in `eboot/boards/` | `MCU_TO_EBOOT_BOARD` + `EOS_BOARD_MAP` in project generator | ✅ Aligned | | Product profiles | 41 profiles in `eos/products/*.h` | — | `PRODUCT_MAP` (41 entries) in project generator | ✅ Aligned | | Platform enum | — | 24 `eos_platform_t` entries in `eos_hal.h` | MCU_DATABASE (100+ MCUs) in hw analyzer | ✅ Aligned | | Peripheral keywords | 33 HAL APIs in `hal.h` + `hal_extended.h` | — | `PERIPHERAL_KEYWORDS` (24 types) + `ComponentDB` (200+ parts) | ✅ Aligned | @@ -118,7 +118,7 @@ Customer Input │ LLMClient (optional) ──► deep analysis │ │ │ │ 2. EosProjectGenerator │ -│ MCU_TO_EBOOT_BOARD (alias of ebuild/sdk_generator.py) ► eboot board dir │ +│ MCU_TO_EBOOT_BOARD ──────────────────► eboot board dir │ │ EOS_BOARD_MAP ──────────────────────► eos board YAML │ │ PRODUCT_MAP (41 entries) ──────────► eos product .h │ │ MULTICORE_MCUS ─────────────────────► multicore config │ diff --git a/ebuild/build/toolchain.py b/ebuild/build/toolchain.py index 70dc134e..c95d81a1 100644 --- a/ebuild/build/toolchain.py +++ b/ebuild/build/toolchain.py @@ -52,6 +52,30 @@ class ResolvedToolchain: } +#: Compiler drivers `compiler:` may name, longest first so that `clang++` +#: is matched before `clang` and `g++` before `cc`. +_COMPILER_DRIVERS = ("clang++", "clang", "g++", "gcc", "c++", "cc") + +#: The C++ driver that goes with each C driver. +_CXX_DRIVER = {"gcc": "g++", "clang": "clang++", "cc": "c++"} + + +def _split_driver(compiler: str) -> tuple: + """Split a compiler name into (prefix, driver). + + ``arm-none-eabi-gcc`` -> ``("arm-none-eabi-", "gcc")``; ``clang`` -> + ``("", "clang")``. A name that carries no recognised driver is used as + the compiler verbatim, with no prefix inferred -- a custom driver then + fails loudly at build time rather than silently becoming host gcc. + """ + for driver in _COMPILER_DRIVERS: + if compiler == driver: + return "", driver + if compiler.endswith("-" + driver): + return compiler[: -len(driver)], driver + return "", compiler + + def resolve_toolchain(toolchain_config) -> ResolvedToolchain: """Resolve a ToolchainConfig into a ResolvedToolchain. @@ -74,15 +98,30 @@ def resolve_toolchain(toolchain_config) -> ResolvedToolchain: extra_cflags = getattr(toolchain_config, "extra_cflags", []) extra_ldflags = getattr(toolchain_config, "extra_ldflags", []) - predef = PREDEFINED_TOOLCHAINS.get(compiler, {}) - if not prefix and predef.get("prefix"): - prefix = predef["prefix"] - if arch == "x86_64" and predef.get("arch"): - arch = predef["arch"] + predef = PREDEFINED_TOOLCHAINS.get(compiler) + if predef is not None: + # `compiler` named one of the toolchains above, which is a shorthand + # for a prefix and an arch. Those toolchains are all GCC. + if not prefix and predef.get("prefix"): + prefix = predef["prefix"] + if arch == "x86_64" and predef.get("arch"): + arch = predef["arch"] + driver = "gcc" + else: + # Otherwise `compiler` names the compiler binary, which is how the + # documentation spells it (`compiler: gcc` with `prefix:` alongside, + # docs/task_cortex_r5_example.md). It used to be read only as a key + # into the table above: anything not in it -- `clang`, `cc`, or the + # `arm-none-eabi-gcc` spelling -- fell through to a bare `gcc`, with + # no warning, so a cross-compile config produced a host binary and + # reported success. + derived_prefix, driver = _split_driver(compiler) + if not prefix: + prefix = derived_prefix return ResolvedToolchain( - cc=f"{prefix}gcc", - cxx=f"{prefix}g++", + cc=f"{prefix}{driver}", + cxx=f"{prefix}{_CXX_DRIVER.get(driver, driver)}", ar=f"{prefix}ar", objcopy=f"{prefix}objcopy", prefix=prefix, diff --git a/ebuild/packages/index_sync.py b/ebuild/packages/index_sync.py index 42f417f5..f273935e 100644 --- a/ebuild/packages/index_sync.py +++ b/ebuild/packages/index_sync.py @@ -343,6 +343,7 @@ def sync( "dependencies": entry.get("dependencies", []), "configure_args": entry.get("configure_args", []), "build_args": entry.get("build_args", []), + "install_args": entry.get("install_args", []), "patches": entry.get("patches", []), } diff --git a/ebuild/packages/recipe.py b/ebuild/packages/recipe.py index 6cbdb382..87cd4585 100644 --- a/ebuild/packages/recipe.py +++ b/ebuild/packages/recipe.py @@ -89,6 +89,37 @@ def validate(self) -> None: f"Must be one of {self.VALID_BUILD_SYSTEMS}." ) + def to_dict(self) -> Dict[str, Any]: + """Convert recipe to dictionary for YAML serialization.""" + data: Dict[str, Any] = { + "package": self.name, + "version": self.version, + } + if self.description: + data["description"] = self.description + if self.license: + data["license"] = self.license + data["url"] = self.url + if self.checksum: + data["checksum"] = self.checksum + data["build"] = self.build_system + # Copies, not the live lists: a caller that appends to what it got + # back must not edit the recipe behind its back. The key order is the + # one index_sync's recipe_dict uses, with install_args after + # build_args; parse_recipe() reads every key by name, so a dump and a + # reload agree field for field regardless of order. + if self.dependencies: + data["dependencies"] = list(self.dependencies) + if self.configure_args: + data["configure_args"] = list(self.configure_args) + if self.build_args: + data["build_args"] = list(self.build_args) + if self.install_args: + data["install_args"] = list(self.install_args) + if self.patches: + data["patches"] = list(self.patches) + return data + def _parse_string_list( raw: Dict[str, Any], diff --git a/tests/ebuild/test_package_recipe.py b/tests/ebuild/test_package_recipe.py index 3c25922d..962de8f2 100644 --- a/tests/ebuild/test_package_recipe.py +++ b/tests/ebuild/test_package_recipe.py @@ -4,8 +4,10 @@ """Tests for ebuild.packages.recipe.""" import pytest +import yaml from ebuild.packages.recipe import ( + PackageRecipe, RecipeError, _parse_recipe, load_recipe_from_string, @@ -115,3 +117,66 @@ def test_depends_alias_must_be_a_list(): with pytest.raises(RecipeError, match="dependencies"): load_recipe_from_string(content) + + +def _fully_populated_recipe() -> PackageRecipe: + """A recipe with every field set, so a round trip has to carry them all.""" + return PackageRecipe( + name="demo", + version="1.2.3", + url="https://example.com/demo-1.2.3.tar.gz", + checksum="sha256:" + "ab" * 32, + build_system="autoconf", + dependencies=["zlib", "openssl"], + patches=["fix-build.patch"], + configure_args=["--enable-static"], + build_args=["VERBOSE=1"], + install_args=["DESTDIR=/tmp/stage"], + description="A demo package", + license="MIT", + ) + + +def test_to_dict_round_trips_every_field(): + """Dumping to YAML and parsing it back must reproduce the recipe exactly. + + to_dict() predates install_args and never emitted it, so a recipe cached + by index_sync came back with install_args == [] while every other field + survived. A field-for-field comparison catches the next one too. + """ + recipe = _fully_populated_recipe() + + reloaded = parse_recipe(yaml.safe_load(yaml.safe_dump(recipe.to_dict()))) + + assert reloaded == recipe + assert reloaded.install_args == ["DESTDIR=/tmp/stage"] + + # Key order is not a correctness property -- parse_recipe() reads every + # key by name, as recipe.py says -- but it is a stability property: the + # cached recipe YAML that index_sync writes is diffed by humans, and this + # keeps install_args next to build_args, where index_sync's recipe_dict + # puts it. If this fails after a deliberate reordering, update both + # emitters together and then this line; it is not a bug in to_dict(). + keys = list(recipe.to_dict()) + assert keys.index("install_args") == keys.index("build_args") + 1 + + +def test_to_dict_returns_copies_not_live_lists(): + """Mutating a list from to_dict() must not reach into the recipe.""" + recipe = _fully_populated_recipe() + + data = recipe.to_dict() + for field_name in ( + "dependencies", + "patches", + "configure_args", + "build_args", + "install_args", + ): + data[field_name].append("injected") + + assert recipe.dependencies == ["zlib", "openssl"] + assert recipe.patches == ["fix-build.patch"] + assert recipe.configure_args == ["--enable-static"] + assert recipe.build_args == ["VERBOSE=1"] + assert recipe.install_args == ["DESTDIR=/tmp/stage"] diff --git a/tests/unit/test_index_sync.py b/tests/unit/test_index_sync.py index 7265fd2d..fa3d2962 100644 --- a/tests/unit/test_index_sync.py +++ b/tests/unit/test_index_sync.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock, patch import pytest +import yaml from click.testing import CliRunner from ebuild.cli.commands import cli @@ -110,6 +111,42 @@ def test_index_sync_success(tmp_path): assert "1.0.0" in content +def test_index_sync_caches_install_args(tmp_path): + """An index entry's install_args must reach the cached recipe YAML. + + The entry-to-recipe mapping listed every list field except this one, and + to_dict() never emitted it, so the cached copy of a package silently lost + the arguments its install step needs. + """ + mgr = IndexSyncManager(index_dir=tmp_path) + + sample_index = [ + { + "name": "staged-pkg", + "version": "2.0.0", + "url": "https://example.com/staged-pkg-2.0.0.tar.gz", + "checksum": "sha256:" + "ab" * 32, + "build_system": "make", + "install_args": ["DESTDIR=/tmp/stage", "PREFIX=/usr"], + } + ] + raw_json = json.dumps(sample_index).encode("utf-8") + + mock_resp = MagicMock() + mock_resp.read.return_value = raw_json + mock_resp.headers = {"Content-Length": str(len(raw_json))} + mock_resp.__enter__.return_value = mock_resp + + with patch("urllib.request.urlopen", return_value=mock_resp): + res = mgr.sync(url="https://example.com/index.json", force=True) + + assert res.package_count == 1 + cached = yaml.safe_load( + (mgr.recipes_dir / "staged-pkg.yaml").read_text(encoding="utf-8") + ) + assert cached["install_args"] == ["DESTDIR=/tmp/stage", "PREFIX=/usr"] + + def test_index_sync_corrupted_json(tmp_path): mgr = IndexSyncManager(index_dir=tmp_path) diff --git a/tests/unit/test_toolchain_compiler_is_used.py b/tests/unit/test_toolchain_compiler_is_used.py new file mode 100644 index 00000000..6d3a5c28 --- /dev/null +++ b/tests/unit/test_toolchain_compiler_is_used.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 EoS Project + +"""`toolchain.compiler` selects the compiler that actually runs. + +resolve_toolchain() used `compiler` only as a key into PREDEFINED_TOOLCHAINS +and then built the tool names as ``f"{prefix}gcc"``. Any value that was not one +of the five toolchain *names* -- `clang`, `cc`, or the `arm-none-eabi-gcc` +spelling -- produced `prefix = ""` and `cc = "gcc"`, with no warning: a +cross-compile configuration built a host binary and reported success. + +These pin the three things that were wrong (a non-gcc compiler was ignored, a +compiler carrying its own prefix was ignored, and neither produced a warning) +without changing what the documented spelling does. +""" + +import unittest + +from ebuild.build.toolchain import PREDEFINED_TOOLCHAINS, resolve_toolchain +from ebuild.core.config import ToolchainConfig + + +class CompilerNamesTheCompiler(unittest.TestCase): + def test_clang_is_clang_and_not_gcc(self): + t = resolve_toolchain(ToolchainConfig(compiler="clang")) + self.assertEqual(t.cc, "clang") + self.assertEqual(t.cxx, "clang++") + + def test_cc_is_cc(self): + t = resolve_toolchain(ToolchainConfig(compiler="cc")) + self.assertEqual(t.cc, "cc") + self.assertEqual(t.cxx, "c++") + + def test_a_compiler_carrying_its_own_prefix_cross_compiles(self): + # docs/architecture.md spells the compiler this way. It used to fall + # through to host gcc, so `ebuild build` emitted a host binary and + # printed "Ready to flash". + t = resolve_toolchain( + ToolchainConfig(compiler="arm-none-eabi-gcc", arch="arm") + ) + self.assertEqual(t.cc, "arm-none-eabi-gcc") + self.assertEqual(t.cxx, "arm-none-eabi-g++") + self.assertEqual(t.prefix, "arm-none-eabi-") + # The binutils have to follow the compiler, or the archive step runs + # the host ar over cross objects. + self.assertEqual(t.ar, "arm-none-eabi-ar") + self.assertEqual(t.objcopy, "arm-none-eabi-objcopy") + + def test_an_unknown_driver_is_used_verbatim_and_infers_no_prefix(self): + # A custom driver must not be guessed at: it is used as given, so it + # fails loudly at build time instead of silently becoming host gcc. + t = resolve_toolchain(ToolchainConfig(compiler="my-special-driver")) + self.assertEqual(t.cc, "my-special-driver") + self.assertEqual(t.prefix, "") + + +class DocumentedSpellingsAreUnchanged(unittest.TestCase): + def test_default_is_host_gcc(self): + t = resolve_toolchain(ToolchainConfig()) + self.assertEqual((t.cc, t.cxx, t.ar, t.arch), ("gcc", "g++", "ar", "x86_64")) + + def test_none_is_host_gcc(self): + t = resolve_toolchain(None) + self.assertEqual((t.cc, t.prefix, t.arch), ("gcc", "", "x86_64")) + + def test_gcc_with_an_explicit_prefix(self): + # docs/task_cortex_r5_example.md: compiler: gcc + prefix: arm-none-eabi- + t = resolve_toolchain( + ToolchainConfig(compiler="gcc", arch="arm", prefix="arm-none-eabi-") + ) + self.assertEqual(t.cc, "arm-none-eabi-gcc") + self.assertEqual(t.arch, "arm") + + def test_every_predefined_toolchain_still_resolves_to_its_prefix(self): + for name, predef in PREDEFINED_TOOLCHAINS.items(): + with self.subTest(toolchain=name): + t = resolve_toolchain(ToolchainConfig(compiler=name)) + self.assertEqual(t.prefix, predef["prefix"]) + self.assertEqual(t.cc, f"{predef['prefix']}gcc") + self.assertEqual(t.arch, predef["arch"]) + + def test_an_explicit_prefix_wins_over_one_derived_from_the_compiler(self): + t = resolve_toolchain( + ToolchainConfig(compiler="aarch64-linux-gnu-gcc", prefix="riscv64-linux-gnu-") + ) + self.assertEqual(t.cc, "riscv64-linux-gnu-gcc") + + +if __name__ == "__main__": + unittest.main()