diff --git a/.github/workflows/coverage-report.yml b/.github/workflows/coverage-report.yml index 52d4eb2d..16e7e2f0 100644 --- a/.github/workflows/coverage-report.yml +++ b/.github/workflows/coverage-report.yml @@ -45,12 +45,31 @@ jobs: tool: cargo-llvm-cov fallback: none - - name: Generate code coverage - run: cargo llvm-cov --locked --all-features --workspace --lcov --output-path lcov.info + - name: Test the isolated provisioning runner + run: python3 -B -m unittest discover -s testinit/testing-server -p 'test_*.py' -v + + - name: Build the isolated provisioning image + run: | + . /etc/os-release + docker build --build-arg "BASE_IMAGE=ubuntu:${VERSION_ID}" -f testinit/Dockerfile.isolated -t azure-init-main-tests:local testinit + + - name: Collect unit and CLI coverage + run: | + cargo llvm-cov clean --workspace + cargo llvm-cov --locked --all-features --workspace --no-report + + - name: Collect real entrypoint coverage + run: cargo llvm-cov --locked --all-features --package azure-init --test testinit --no-report -- --ignored agent_provisioning_in_container + + - name: Generate combined code coverage + run: cargo llvm-cov report --package azure-init --package libazureinit --package libazureinit-kvp --lcov --output-path lcov.info + + - name: Cover compile-time version variants + run: python3 -B testinit/testing-server/version_coverage.py - name: Upload coverage to Codecov uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} - files: lcov.info + files: lcov.info,target/version-override.lcov,target/version-sha.lcov,target/version-package.lcov fail_ci_if_error: true diff --git a/Cargo.toml b/Cargo.toml index 1d752d2f..01a536f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,10 @@ path = "src/main.rs" name = "functional_tests" path = "tests/functional_tests.rs" +[[test]] +name = "testinit" +path = "testinit/tests/provisioning.rs" + [workspace] members = [ "libazureinit", diff --git a/src/main.rs b/src/main.rs index 516c0ae6..11509a90 100644 --- a/src/main.rs +++ b/src/main.rs @@ -275,6 +275,17 @@ fn failure_report(error: &anyhow::Error, vm_id: &str) -> ProvisioningReport { }) } +fn provisioning_error_exit_code(error: &anyhow::Error) -> ExitCode { + let config: u8 = exitcode::CONFIG + .try_into() + .expect("Error code must be less than 256"); + match error.root_cause().downcast_ref::() { + Some(LibError::UserMissing { user: _ }) => ExitCode::from(config), + Some(LibError::NonEmptyPassword) => ExitCode::from(config), + Some(_) | None => ExitCode::FAILURE, + } +} + async fn write_kvp_report( store: &KvpPoolStore, report: &ProvisioningReport, @@ -436,17 +447,7 @@ async fn main() -> ExitCode { } Err(e) => { tracing::error!("Provisioning failed with error: {e:?}"); - - let config: u8 = exitcode::CONFIG - .try_into() - .expect("Error code must be less than 256"); - match e.root_cause().downcast_ref::() { - Some(LibError::UserMissing { user: _ }) => { - ExitCode::from(config) - } - Some(LibError::NonEmptyPassword) => ExitCode::from(config), - Some(_) | None => ExitCode::FAILURE, - } + provisioning_error_exit_code(&e) } } }; @@ -557,6 +558,85 @@ mod tests { const VM_ID: &str = "00000000-0000-0000-0000-000000000001"; + #[rstest] + #[case::imds_preferred(true, true, Some("imds-user"))] + #[case::imds_only(true, false, Some("imds-user"))] + #[case::ovf_fallback(false, true, Some("ovf-user"))] + #[case::missing_sources(false, false, None)] + fn username_source_precedence( + #[case] imds_available: bool, + #[case] ovf_available: bool, + #[case] expected: Option<&str>, + ) { + let metadata: InstanceMetadata = serde_json::from_value(json!({ + "compute": { + "osProfile": { + "adminUsername": "imds-user", + "computerName": "test-host", + "disablePasswordAuthentication": "true" + }, + "publicKeys": [] + } + })) + .unwrap(); + let mut environment = Environment::default(); + environment + .provisioning_section + .linux_prov_conf_set + .username = "ovf-user".into(); + let result = get_username( + imds_available.then_some(&metadata), + ovf_available.then_some(&environment), + ); + match expected { + Some(username) => assert_eq!(result.unwrap(), username), + None => assert!(matches!( + result.unwrap_err().downcast_ref::(), + Some(LibError::UsernameFailure) + )), + } + } + + #[rstest] + #[case::existing("file", Ok(()))] + #[case::missing("missing", Ok(()))] + #[case::directory("directory", Err(io::ErrorKind::IsADirectory))] + fn log_cleanup_preserves_io_results( + #[case] entry: &str, + #[case] expected: Result<(), io::ErrorKind>, + ) { + let dir = TempDir::new().unwrap(); + let log = dir.path().join("azure-init.log"); + match entry { + "file" => std::fs::write(&log, "log").unwrap(), + "directory" => std::fs::create_dir(&log).unwrap(), + _ => {} + } + let mut config = Config::default(); + config.azure_init_log_path.path = log.clone(); + assert_eq!( + clean_log_file(&config).map_err(|error| error.kind()), + expected + ); + assert_eq!(log.exists(), expected.is_err()); + } + + #[rstest] + #[case::missing_user(LibError::UserMissing { user: "missing".into() }.into(), 78)] + #[case::nonempty_password(LibError::NonEmptyPassword.into(), 78)] + #[case::other_typed_error(LibError::UsernameFailure.into(), 1)] + #[case::untyped_error(anyhow::anyhow!("unexpected failure"), 1)] + fn provisioning_error_exit_codes_preserve_context( + #[case] error: anyhow::Error, + #[case] expected: u8, + ) { + let error = error.context("provisioning failed"); + assert_eq!( + format!("{:?}", provisioning_error_exit_code(&error)), + format!("{:?}", ExitCode::from(expected)) + ); + } + #[test] fn clean_reports_subscriber_conflict() { const CHILD: &str = "AZURE_INIT_TEST_MAIN_SUBSCRIBER_CONFLICT"; diff --git a/testinit/Dockerfile.isolated b/testinit/Dockerfile.isolated new file mode 100644 index 00000000..bb132ca0 --- /dev/null +++ b/testinit/Dockerfile.isolated @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=ubuntu:24.04 +FROM ${BASE_IMAGE} + +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 libudev1 \ + && rm -rf /var/lib/apt/lists \ + && mkdir -p /test /work /var/lib/hyperv /etc/sudoers.d /run/azure-init/media \ + && touch /test/isolated-provisioning + +COPY testing-server/container_provisioning.py testing-server/imds_handler.py \ + testing-server/wireserver_handler.py testing-server/utils.py /test/ + +ENTRYPOINT ["python3", "-B", "/test/container_provisioning.py", "--inside"] diff --git a/testinit/README.md b/testinit/README.md index 2ce95eba..42835d5a 100644 --- a/testinit/README.md +++ b/testinit/README.md @@ -1,6 +1,6 @@ # Azure-Init - Testinit Project -A containerized Azure provisioning agent that simulates Azure Instance Metadata Service (IMDS) and WireServer for testing and development environments. +Testinit provides isolated provisioning tests and systemd-based E2E tests. Both use shared mock Azure Instance Metadata Service (IMDS) and WireServer handlers. > [!WARNING] > @@ -11,11 +11,12 @@ A containerized Azure provisioning agent that simulates Azure Instance Metadata ## Overview -This project consists of two main components: -- **Provisioning Agent**: A systemd-based service running `azure-init` binary in a container +This project consists of: +- **Isolated Tests**: The real Cargo-built agent running with stubbed OS commands in disposable, unprivileged containers +- **Provisioning Agent**: A systemd-based service running the `azure-init` binary in a privileged container - **Testing Server**: A mock Azure service providing IMDS and WireServer endpoints -## Architecture +## Systemd Architecture The setup creates two Docker networks with Azure-like IP addresses: - `imds-network` (169.254.0.0/16) - For IMDS communication @@ -25,9 +26,28 @@ The setup creates two Docker networks with Azure-like IP addresses: - Docker and Docker Compose - WSL2 or Linux environment -- The `azure-init` binary and service file +- Python 3, Rust, and libudev development headers for isolated tests -## Quick Start +## Isolated Provisioning Tests + +The [Cargo test](tests/provisioning.rs) runs the real agent with stubbed OS +commands and shared mock services, without the privileged Compose setup. +From the repository root on Ubuntu: + +```bash +python3 -B -m unittest discover -s testinit/testing-server -p 'test_*.py' -v +. /etc/os-release +docker build --build-arg "BASE_IMAGE=ubuntu:${VERSION_ID}" \ + -f testinit/Dockerfile.isolated -t azure-init-main-tests:local testinit +cargo test --locked --all-features --package azure-init --test testinit \ + -- --ignored agent_provisioning_in_container +``` + +This Docker-dependent test is ignored by default. The +[coverage workflow](../.github/workflows/coverage-report.yml) runs it and +uploads combined coverage to Codecov. + +## Systemd E2E Quick Start ### Starting Services @@ -104,6 +124,7 @@ docker compose logs -f testing-server ### Building -The Dockerfile expects: -- `azure-init/target/debug/azure-init` - The main binary (compiled via `cargo build`) -- `azure-init/config/azure-init.service` - The systemd service file +The systemd [Dockerfile](Dockerfile) builds the repository source with +`cargo build` and installs [azure-init.service](../config/azure-init.service). +The isolated [Dockerfile](Dockerfile.isolated) contains only the fixture runtime +and shared mock handlers; its agent binary is mounted by the Cargo test. diff --git a/testinit/testing-server/container_provisioning.py b/testinit/testing-server/container_provisioning.py new file mode 100644 index 00000000..04c3ef36 --- /dev/null +++ b/testinit/testing-server/container_provisioning.py @@ -0,0 +1,371 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Exercise the real agent against controlled services in a disposable container.""" + +import argparse +from http.server import HTTPServer +import json +import os +from pathlib import Path +import pwd +import shutil +import subprocess +import tempfile +import threading +import uuid + +from imds_handler import IMDSHandler +from wireserver_handler import WireServerHandler + + +IMAGE = "azure-init-main-tests:local" +VM_ID = "00000000-0000-0000-0000-000000000000" +USERNAME = "coverage-user" +SCENARIOS = { + "success": (0, None), + "failure": (1, "failed to provision hostname"), + "config-error": (78, "user not found"), + "marker-error": (1, "I/O error"), + "ovf-fallback": (1, "failed to retrieve instance metadata"), + "ovf-missing": (1, "failed to determine username"), + "clean-read-error": (1, "Failed to read directory entry"), + "clean-marker-error": (1, "Failed to clean provisioning marker"), + "clean-race": (0, "No provisioning marker found"), +} +OVF_ENV = """ + + 1.0 + + ovf-usercoverage-host + + + + 1.0 + + +""" + + +def check_mount_table(mtab): + for entry in mtab.read_text().splitlines(): + fields = entry.split() + if not fields or fields[0].startswith("#"): + continue + if len(fields) < 3: + raise RuntimeError(f"invalid mount table entry: {entry!r}") + if fields[2] in ("iso9660", "udf"): + raise RuntimeError(f"fixture must not expose mounted {fields[2]} media") + + +def make_handlers(scenario, requests, service_errors): + class CheckedHandler: + def log_message(self, *_args): + pass + + def handle(self): + try: + super().handle() + except (AssertionError, ValueError) as error: + service_errors.append(str(error)) + raise + + class MetadataHandler(CheckedHandler, IMDSHandler): + _response_position = 0 + _responses = [{ + "status_code": 403 if scenario.startswith("ovf-") else 200, + "headers": {"Content-Type": "application/json"}, + "response": { + "compute": { + "osProfile": { + "adminUsername": USERNAME, + "computerName": "coverage-host", + "disablePasswordAuthentication": "true", + }, + "publicKeys": ( + [{"keyData": "ssh-ed25519 test-key", "path": "/unused"}] + if scenario == "config-error" + else [] + ), + } + }, + }] + + def do_GET(self): + assert self.path.startswith( + "http://169.254.169.254/metadata/instance?" + ), self.path + assert self.headers["Metadata"] == "true" + requests.append(("imds", None)) + super().do_GET() + + class HealthHandler(CheckedHandler, WireServerHandler): + _response_position = 0 + _responses = [{ + "status_code": 201, + "headers": {"Content-Length": "0"}, + "response": "", + }] + + def record_report(self, post_data): + assert self.path == "/provisioning/health", self.path + requests.append(("report", json.loads(post_data))) + + return MetadataHandler, HealthHandler + + +def inside_container(scenario): + if not Path("/test/isolated-provisioning").is_file(): + raise RuntimeError("provisioning fixture must only run in its isolated image") + if Path("/sys/class/dmi/id/product_uuid").exists(): + raise RuntimeError("fixture requires an isolated, empty /sys") + if any(user.pw_name == USERNAME for user in pwd.getpwall()): + raise RuntimeError(f"fixture requires a nonexistent account: {USERNAME}") + check_mount_table(Path("/proc/mounts")) + mtab = Path("/etc/mtab") + mtab.unlink(missing_ok=True) + mtab.write_text(Path("/test/mtab").read_text()) + + requests = [] + service_errors = [] + metadata_handler, health_handler = make_handlers(scenario, requests, service_errors) + + artifacts = Path("/artifacts") + work = Path("/work") + cleaning = scenario.startswith("clean-") + if scenario == "marker-error": + (work / "state").write_text("not a directory") + if cleaning: + (work / "state").mkdir() + marker = work / "state" / "vanished.provisioned" + if scenario == "clean-marker-error": + marker.mkdir() + else: + marker.write_text("complete") + (work / "state" / "keep.txt").write_text("keep") + (work / "ovf-env.xml").write_text(OVF_ENV) + commands = work / "commands.jsonl" + stubs = work / "bin" + stubs.mkdir() + stub = stubs / "command-stub" + stub.write_text( + "#!/usr/bin/python3\n" + "import json, os, shutil, sys\n" + "name = os.path.basename(sys.argv[0])\n" + "with open('/work/commands.jsonl', 'a') as output:\n" + " output.write(json.dumps([name, *sys.argv[1:]]) + '\\n')\n" + "if name == 'getent':\n" + " sys.exit(1)\n" + "if name == 'mount':\n" + " if sys.argv[-2] != '/test/ovf-good':\n" + " sys.exit(17)\n" + " shutil.copyfile('/work/ovf-env.xml', os.path.join(sys.argv[-1], 'ovf-env.xml'))\n" + "if name == 'hostnamectl' and os.environ['TEST_SCENARIO'] == 'failure':\n" + " print('expected hostname failure', file=sys.stderr)\n" + " sys.exit(17)\n" + "if name not in ('hostnamectl', 'useradd', 'usermod', 'passwd', 'mount', 'umount', 'eject'):\n" + " raise RuntimeError('unexpected command: ' + name)\n" + ) + stub.chmod(0o755) + for name in ("hostnamectl", "getent", "useradd", "usermod", "passwd", "mount", "umount", "eject"): + (stubs / name).symlink_to(stub) + + with ( + HTTPServer(("127.0.0.1", 0), metadata_handler) as imds_server, + HTTPServer(("127.0.0.1", 0), health_handler) as health_server, + ): + imds_endpoint = f"http://127.0.0.1:{imds_server.server_port}" + health_endpoint = f"http://127.0.0.1:{health_server.server_port}" + config = work / "azure-init.toml" + config.write_text( + '[ssh]\nconfigure_sshd_password_authentication = false\n' + 'query_sshd_config = false\n' + '[azure_init_data_dir]\npath = "/work/state"\n' + '[azure_init_log_path]\npath = "/work/azure-init.log"\n' + '[imds]\nconnection_timeout_secs = 1\nrequest_timeout_secs = 1\n' + 'retry_interval_secs = 0.01\ntotal_retry_timeout_secs = 3\n' + '[wireserver]\nconnection_timeout_secs = 0.01\n' + 'read_timeout_secs = 1\ntotal_retry_timeout_secs = 3\n' + f'health_endpoint = "{health_endpoint}/provisioning/health"\n' + '[telemetry]\nkvp_diagnostics = true\nkvp_filter = "info"\n' + ) + environment = dict( + os.environ, + PATH=str(stubs), + TEST_SCENARIO=scenario, + HTTP_PROXY=imds_endpoint, + http_proxy=imds_endpoint, + NO_PROXY="127.0.0.1,localhost", + no_proxy="127.0.0.1,localhost", + AZURE_INIT_LOG="info", + ) + if scenario in ("clean-read-error", "clean-race"): + environment["LD_PRELOAD"] = "/test/fs_faults.so" + servers = (imds_server, health_server) + threads = [ + threading.Thread(target=server.serve_forever, daemon=True) + for server in servers + ] + for thread in threads: + thread.start() + try: + result = subprocess.run( + ["/test/azure-init", "--config", str(config), *(["clean"] if cleaning else [])], + env=environment, + capture_output=True, + text=True, + timeout=20, + ) + (artifacts / "stdout").write_text(result.stdout) + (artifacts / "stderr").write_text(result.stderr) + assert not service_errors, service_errors + expected_status, reason = SCENARIOS[scenario] + assert result.returncode == expected_status, result + log = (work / "azure-init.log").read_text() + if cleaning: + assert requests == [], requests + assert not commands.exists() + assert marker.exists() == (scenario != "clean-race") + assert (work / "state" / "keep.txt").read_text() == "keep" + assert reason in log, log + else: + assert [kind for kind, _ in requests] == ["imds", "report"], requests + report = requests[1][1] + (artifacts / "http-report.json").write_text(json.dumps(report)) + marker = work / "state" / f"{VM_ID}.provisioned" + calls = [json.loads(line) for line in commands.read_text().splitlines()] + if scenario == "ovf-fallback": + expected_commands = ["mount", "mount", "umount", "eject", "mount"] + assert log.count("Successfully parsed OVF environment.") == 1, log + elif scenario == "ovf-missing": + expected_commands = ["mount", "mount"] + else: + expected_commands = ( + ["hostnamectl"] + if scenario == "failure" + else ["hostnamectl", "getent", "useradd", "passwd"] + ) + assert [call[0] for call in calls] == expected_commands, calls + if scenario == "success": + assert report == {"state": "Ready"}, report + assert marker.is_file() + assert Path("/etc/sudoers.d/azure-init-user").read_text() == ( + f"{USERNAME} ALL=(ALL) NOPASSWD: ALL\n" + ) + else: + assert report["state"] == "NotReady", report + assert report["details"]["subStatus"] == "ProvisioningFailed", report + assert f"reason={reason}" in report["details"]["description"], report + assert not marker.exists() + if scenario == "marker-error": + assert (work / "state").read_text() == "not a directory" + assert "Failed to mark provisioning complete." in log, log + expected_log = ( + "Provisioning completed successfully" + if scenario == "success" + else "Provisioning failed with error:" + ) + assert expected_log in log, log + shutil.copyfile("/var/lib/hyperv/.kvp_pool_1", artifacts / ".kvp_pool_1") + print(f"Verified {scenario}: actual agent exit {result.returncode}") + finally: + for server in servers: + server.shutdown() + for thread in threads: + thread.join(timeout=5) + assert not thread.is_alive(), "mock server did not shut down" + log = work / "azure-init.log" + if log.exists(): + shutil.copyfile(log, artifacts / "azure-init.log") + for output in artifacts.iterdir(): + output.chmod(0o644) + + +def run_containers(agent, artifacts): + profile_pattern = os.environ.get("LLVM_PROFILE_FILE") + for scenario in SCENARIOS: + output = artifacts / scenario + output.mkdir() + # Container root lacks DAC_OVERRIDE; only this disposable output is writable. + output.chmod(0o777) + with tempfile.TemporaryDirectory(prefix="azure-init-fixture-") as temporary: + empty_sys = Path(temporary) / "sys" + empty_sys.mkdir(mode=0o755) + mtab = Path(temporary) / "mtab" + devices = [] + if scenario.startswith("ovf-"): + devices = ["/test/missing-before", "/test/missing-after"] + if scenario == "ovf-fallback": + devices.insert(1, "/test/ovf-good") + mtab.write_text("".join(f"{device} /media iso9660 ro 0 0\n" for device in devices)) + mtab.chmod(0o644) + command = [ + "docker", "create", "--name", f"azure-init-test-{uuid.uuid4().hex}", + "--network", "none", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--read-only", + "--mount", "type=volume,dst=/etc", + "--tmpfs", "/work:exec,mode=0700", + "--tmpfs", "/var/lib/hyperv:mode=0700", + "--tmpfs", "/etc/sudoers.d:mode=0700", + "--tmpfs", "/run/azure-init/media:mode=0700", + "--mount", f"type=bind,src={agent},dst=/test/azure-init,readonly", + "--mount", f"type=bind,src={empty_sys},dst=/sys,readonly", + "--mount", f"type=bind,src={mtab},dst=/test/mtab,readonly", + "--mount", f"type=bind,src={artifacts / 'fs_faults.so'},dst=/test/fs_faults.so,readonly", + "--mount", f"type=bind,src={output},dst=/artifacts", + ] + if profile_pattern: + command.extend([ + "--env", + f"LLVM_PROFILE_FILE=/artifacts/{Path(profile_pattern).name}", + ]) + command.extend([IMAGE, scenario]) + created = subprocess.run( + command, check=True, capture_output=True, text=True, timeout=30 + ) + container = created.stdout.strip() + try: + result = subprocess.run( + ["docker", "start", "--attach", container], + capture_output=True, text=True, timeout=60, + ) + print(result.stdout, end="") + print(result.stderr, end="") + result.check_returncode() + if profile_pattern: + profiles = list(output.glob("*.profraw")) + assert profiles, "instrumented agent produced no coverage profile" + destination = Path(profile_pattern).parent + for profile in profiles: + shutil.copyfile( + profile, + destination / f"{profile.stem}-{container[:12]}-{scenario}.profraw", + ) + finally: + subprocess.run( + ["docker", "rm", "--force", "--volumes", container], + check=True, capture_output=True, text=True, timeout=15, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--inside", choices=SCENARIOS) + parser.add_argument("--agent", type=Path) + parser.add_argument("--artifacts", type=Path) + args = parser.parse_args() + if args.inside: + inside_container(args.inside) + elif args.agent and args.artifacts: + artifacts = args.artifacts.resolve(strict=True) + faults = artifacts / "fs_faults.so" + subprocess.run( + ["cc", "-shared", "-fPIC", "-Wall", "-Wextra", "-Werror", + str(Path(__file__).with_name("fs_faults.c")), "-o", str(faults), "-ldl"], + check=True, timeout=30, + ) + faults.chmod(0o644) + run_containers(args.agent.resolve(strict=True), artifacts) + else: + parser.error("supply --inside or both --agent and --artifacts") diff --git a/testinit/testing-server/fs_faults.c b/testinit/testing-server/fs_faults.c new file mode 100644 index 00000000..e0b31795 --- /dev/null +++ b/testinit/testing-server/fs_faults.c @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +static void *next_symbol(const char *name) { + int saved_errno = errno; + void *symbol = dlsym(RTLD_NEXT, name); + if (!symbol) { + fprintf(stderr, "Cannot resolve %s: %s\n", name, dlerror()); + _Exit(127); + } + errno = saved_errno; + return symbol; +} + +static bool enabled(const char *scenario) { + int saved_errno = errno; + const char *current = getenv("TEST_SCENARIO"); + bool result = current && strcmp(current, scenario) == 0 && + access("/test/isolated-provisioning", F_OK) == 0; + errno = saved_errno; + return result; +} + +static bool is_state_directory(DIR *directory) { + int saved_errno = errno; + char descriptor[64], path[sizeof("/work/state")]; + snprintf(descriptor, sizeof(descriptor), "/proc/self/fd/%d", dirfd(directory)); + ssize_t length = readlink(descriptor, path, sizeof(path)); + errno = saved_errno; + return length >= 0 && (size_t)length == sizeof("/work/state") - 1 && + memcmp(path, "/work/state", (size_t)length) == 0; +} + +// Inject faults only at the disposable fixture's cleanup paths. +struct dirent64 *readdir64(DIR *directory) { + if (enabled("clean-read-error") && is_state_directory(directory)) { + errno = EIO; + return NULL; + } + struct dirent64 *(*read_next)(DIR *) = + (struct dirent64 *(*)(DIR *))next_symbol("readdir64"); + return read_next(directory); +} + +int unlink(const char *path) { + int (*remove_file)(const char *) = + (int (*)(const char *))next_symbol("unlink"); + int result = remove_file(path); + if (result == 0 && enabled("clean-race") && + strcmp(path, "/work/state/vanished.provisioned") == 0) { + errno = ENOENT; + return -1; + } + return result; +} diff --git a/testinit/testing-server/test_container_provisioning.py b/testinit/testing-server/test_container_provisioning.py new file mode 100644 index 00000000..bf9e58f1 --- /dev/null +++ b/testinit/testing-server/test_container_provisioning.py @@ -0,0 +1,192 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Check shared HTTP handlers, fixture isolation, and the Docker lifecycle.""" + +from contextlib import contextmanager, redirect_stdout +from http.client import HTTPConnection, RemoteDisconnected +import io +import os +from pathlib import Path +import subprocess +import tempfile +import threading +import unittest +from unittest.mock import Mock, patch + +import container_provisioning as fixture + + +class HandlerTests(unittest.TestCase): + @contextmanager + def serve(self, handler): + with fixture.HTTPServer(("127.0.0.1", 0), handler) as server: + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + thread.start() + connection = HTTPConnection(*server.server_address, timeout=5) + try: + yield connection + finally: + connection.close() + server.shutdown() + thread.join(timeout=5) + self.assertFalse(thread.is_alive(), "mock server did not shut down") + + def test_invalid_requests_are_reported(self): + requests, errors = [], [] + metadata, health = fixture.make_handlers("success", requests, errors) + for handler, method, path, body in ( + (metadata, "GET", "/unexpected", ""), + (health, "POST", "/provisioning/health", "not-json"), + ): + with self.subTest(method=method), patch.object(fixture.HTTPServer, "handle_error"): + with self.serve(handler) as connection: + connection.request(method, path, body) + with self.assertRaises(RemoteDisconnected): + connection.getresponse() + self.assertEqual(len(errors), 2) + self.assertEqual(requests, []) + + def test_systemd_health_responses_still_snapshot_kvp(self): + for custom in (False, True): + with self.subTest(custom=custom): + class Handler(fixture.WireServerHandler): + _response_position = 0 + + if custom: + response_file = Path(fixture.__file__).parent / ( + "api-responses/wireserver/one-bad-then-success.xml" + ) + Handler.set_response_file_path(response_file) + Handler.load_responses() + statuses = (500, 200) if custom else (200,) + with ( + patch.object(Handler, "store_kvp_file") as snapshot, + patch("wireserver_handler.time.sleep"), + self.serve(Handler) as connection, + ): + for status in statuses: + connection.request("POST", "/provisioning/health", '{"state":"Ready"}') + response = connection.getresponse() + self.assertEqual(response.status, status) + self.assertEqual(response.read(), b"") + self.assertEqual(snapshot.call_count, len(statuses)) + + +class MountTableTests(unittest.TestCase): + def check_mount_table(self, contents): + with tempfile.TemporaryDirectory() as temporary: + mtab = Path(temporary) / "mtab" + mtab.write_text(contents) + fixture.check_mount_table(mtab) + + def test_allows_empty_and_standard_container_mounts(self): + for contents in ("", "# comment\n", "overlay / overlay rw 0 0\n"): + with self.subTest(contents=contents): + self.check_mount_table(contents) + + def test_rejects_provisioning_media(self): + for filesystem in ("iso9660", "udf"): + with self.subTest(filesystem=filesystem): + with self.assertRaisesRegex(RuntimeError, filesystem): + self.check_mount_table(f"/dev/sr0 /media {filesystem} ro 0 0\n") + + def test_rejects_malformed_mount_entries(self): + with self.assertRaisesRegex(RuntimeError, "invalid mount table"): + self.check_mount_table("incomplete\n") + + +class ContainerRunnerTests(unittest.TestCase): + def run_fixture(self, root, docker, profile_pattern=None): + artifacts = root / "artifacts" + artifacts.mkdir() + with ( + patch.dict(os.environ), + patch.object(fixture.subprocess, "run", side_effect=docker), + redirect_stdout(io.StringIO()), + ): + os.environ.pop("LLVM_PROFILE_FILE", None) + if profile_pattern: + os.environ["LLVM_PROFILE_FILE"] = str(profile_pattern) + fixture.run_containers(Path("/test-agent"), artifacts) + + def test_collects_profiles_and_removes_each_container(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + profiles = root / "profiles" + profiles.mkdir() + calls = [] + output = None + + def docker(command, **_options): + nonlocal output + calls.append(command[1]) + if command[1] == "create": + mount = next(arg for arg in command if arg.endswith("dst=/artifacts")) + output = Path(mount.split("src=", 1)[1].split(",dst=", 1)[0]) + self.assertEqual(output.stat().st_mode & 0o777, 0o777) + self.assertIn("LLVM_PROFILE_FILE=/artifacts/agent-%p-%m.profraw", command) + self.assertEqual(command[command.index("--network") + 1], "none") + self.assertEqual(command[command.index("--cap-drop") + 1], "ALL") + self.assertIn("no-new-privileges", command) + self.assertIn("--read-only", command) + self.assertIn("type=volume,dst=/etc", command) + self.assertIn("/work:exec,mode=0700", command) + self.assertIn("/run/azure-init/media:mode=0700", command) + for mount in command: + if mount.endswith("dst=/sys,readonly"): + source = Path(mount.split("src=", 1)[1].split(",dst=", 1)[0]) + self.assertEqual(list(source.iterdir()), []) + if mount.endswith("dst=/test/mtab,readonly"): + source = Path(mount.split("src=", 1)[1].split(",dst=", 1)[0]) + contents = source.read_text() + self.assertEqual("iso9660" in contents, command[-1].startswith("ovf-")) + self.assertEqual("/test/ovf-good" in contents, command[-1] == "ovf-fallback") + return subprocess.CompletedProcess(command, 0, "container-id\n", "") + if command[1] == "start": + (output / "agent-1-123.profraw").write_bytes(b"synthetic profile") + if command[1] == "rm": + self.assertIn("--volumes", command) + return subprocess.CompletedProcess(command, 0, "", "") + + self.run_fixture(root, docker, profiles / "agent-%p-%m.profraw") + self.assertEqual(calls, ["create", "start", "rm"] * len(fixture.SCENARIOS)) + for scenario in fixture.SCENARIOS: + profile = profiles / f"agent-1-123-container-id-{scenario}.profraw" + self.assertEqual(profile.read_bytes(), b"synthetic profile") + + def test_normal_run_does_not_require_profiles(self): + with tempfile.TemporaryDirectory() as temporary: + docker = Mock(return_value=subprocess.CompletedProcess([], 0, "container-id\n", "")) + self.run_fixture(Path(temporary), docker) + commands = [call.args[0] for call in docker.call_args_list] + self.assertEqual([cmd[1] for cmd in commands], ["create", "start", "rm"] * len(fixture.SCENARIOS)) + self.assertFalse(any( + arg.startswith("LLVM_PROFILE_FILE=") for cmd in commands for arg in cmd + )) + + def test_failures_remove_the_container(self): + completed = subprocess.CompletedProcess([], 0, "container-id\n", "") + for start, error, needs_profile in ( + (subprocess.CompletedProcess([], 1, "", "fixture failed"), subprocess.CalledProcessError, False), + (subprocess.TimeoutExpired("docker start", 60), subprocess.TimeoutExpired, False), + (completed, AssertionError, True), + ): + with self.subTest(error=error.__name__), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + docker = Mock(side_effect=[completed, start, completed]) + profile = root / "agent-%p-%m.profraw" if needs_profile else None + with self.assertRaises(error) as failure: + self.run_fixture(root, docker, profile) + if needs_profile: + self.assertIn("no coverage profile", str(failure.exception)) + self.assertEqual( + [call.args[0][1] for call in docker.call_args_list], + ["create", "start", "rm"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/testinit/testing-server/version_coverage.py b/testinit/testing-server/version_coverage.py new file mode 100644 index 00000000..811927f5 --- /dev/null +++ b/testinit/testing-server/version_coverage.py @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Test compile-time version choices without changing the production function.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + + +ROOT = Path(__file__).resolve().parents[2] +SHA = "0123456789abcdef0123456789abcdef01234567" +GIT_SHIM = """#!/usr/bin/python3 +import os, sys +if sys.argv[1:] == ["describe", "--dirty", "--tags"]: + sys.exit(1) +if sys.argv[1:] == ["rev-parse", "HEAD"]: + sha = os.environ.get("TEST_BUILD_SHA") + if sha: + print(sha) + sys.exit(0) + sys.exit(1) +os.execv(os.environ["TEST_REAL_GIT"], ["git", *sys.argv[1:]]) +""" + + +def main(): + metadata = json.loads(subprocess.check_output( + ["cargo", "metadata", "--locked", "--no-deps", "--format-version=1"], + cwd=ROOT, text=True, + )) + version = next(p["version"] for p in metadata["packages"] if p["name"] == "azure-init") + real_git = shutil.which("git") + if real_git is None: + raise RuntimeError("git is required for version coverage tests") + + for variant, expected in ( + ("override", "coverage-override"), + ("sha", f"{version}-{SHA[:7]}"), + ("package", version), + ): + with tempfile.TemporaryDirectory(prefix=f"azure-init-version-{variant}-") as temporary: + directory = Path(temporary) + environment = dict(os.environ) + for variable in ( + "AZURE_INIT_VERSION", "AZURE_INIT_BUILD_VERSION", + "AZURE_INIT_BUILD_SHA", "LLVM_PROFILE_FILE", + ): + environment.pop(variable, None) + environment["CARGO_LLVM_COV_TARGET_DIR"] = str(directory / "target") + if variant == "override": + environment["AZURE_INIT_VERSION"] = expected + else: + git = directory / "git" + git.write_text(GIT_SHIM) + git.chmod(0o755) + environment.update( + PATH=f"{directory}{os.pathsep}{environment['PATH']}", + TEST_REAL_GIT=real_git, + TEST_BUILD_SHA=SHA if variant == "sha" else "", + ) + result = subprocess.run( + ["cargo", "llvm-cov", "run", "--locked", "--all-features", + "--package", "azure-init", "--bin", "azure-init", "--no-report", + "--", "--version"], + cwd=ROOT, env=environment, check=True, text=True, + stdout=subprocess.PIPE, timeout=600, + ) + assert result.stdout == f"azure-init {expected}\n", result.stdout + report = ROOT / "target" / f"version-{variant}.lcov" + report.parent.mkdir(exist_ok=True) + subprocess.run( + ["cargo", "llvm-cov", "report", "--package", "azure-init", + "--lcov", "--output-path", str(report)], + cwd=ROOT, env=environment, check=True, timeout=60, + ) + print(f"Verified {variant}: {expected}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/testinit/testing-server/wireserver_handler.py b/testinit/testing-server/wireserver_handler.py index d997d1ce..91345a1f 100644 --- a/testinit/testing-server/wireserver_handler.py +++ b/testinit/testing-server/wireserver_handler.py @@ -116,16 +116,16 @@ def write_default_response(self): self.wfile.write(response.encode()) return - def do_POST(self): - """Handle POST requests to WireServer endpoints.""" - content_length = int(self.headers.get("Content-Length", 0)) - post_data = self.rfile.read(content_length) - + def record_report(self, post_data): logger.info(f"WireServer POST request: {self.path}") logger.info(f"POST data: {post_data.decode('utf-8', errors='ignore')}") - self.store_kvp_file() + def do_POST(self): + """Handle POST requests to WireServer endpoints.""" + content_length = int(self.headers.get("Content-Length", 0)) + self.record_report(self.rfile.read(content_length)) + if self._responses is not None: self.write_custom_response() return diff --git a/testinit/tests/provisioning.rs b/testinit/tests/provisioning.rs new file mode 100644 index 00000000..fd34b45a --- /dev/null +++ b/testinit/tests/provisioning.rs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use libazureinit_kvp::{ + KvpPool, KvpPoolStore, PoolMode, ProvisioningReport, + PROVISIONING_REPORT_KEY, +}; +use std::fs; +use std::path::Path; + +#[test] +#[ignore = "requires Docker and the azure-init-main-tests:local image"] +fn agent_provisioning_in_container() { + let artifacts = tempfile::tempdir().unwrap(); + let runner = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("testinit/testing-server/container_provisioning.py"); + assert_cmd::Command::new("python3") + .arg("-B") + .arg(runner) + .arg("--agent") + .arg(assert_cmd::cargo::cargo_bin!("azure-init")) + .arg("--artifacts") + .arg(artifacts.path()) + .timeout(std::time::Duration::from_secs(240)) + .assert() + .success(); + + for (scenario, result) in [ + ("success", Some("success")), + ("failure", Some("error")), + ("config-error", Some("error")), + ("marker-error", Some("error")), + ("ovf-fallback", Some("error")), + ("ovf-missing", Some("error")), + ("clean-read-error", None), + ("clean-marker-error", None), + ("clean-race", None), + ] { + let directory = artifacts.path().join(scenario); + let store = + KvpPoolStore::new_in(KvpPool::Guest, &directory, PoolMode::Safe) + .unwrap(); + let encoded = store.read(PROVISIONING_REPORT_KEY).unwrap(); + let Some(result) = result else { + assert_eq!(encoded, None, "cleanup must not report provisioning"); + continue; + }; + let encoded = encoded.unwrap(); + let report: ProvisioningReport = encoded.parse().unwrap(); + let report = serde_json::to_value(report).unwrap(); + assert_eq!(report["result"], result); + assert_eq!(report["vm_id"], "00000000-0000-0000-0000-000000000000"); + assert_eq!( + store + .dump() + .unwrap() + .iter() + .filter(|(key, _)| key == PROVISIONING_REPORT_KEY) + .count(), + 1 + ); + let http: serde_json::Value = serde_json::from_slice( + &fs::read(directory.join("http-report.json")).unwrap(), + ) + .unwrap(); + if result == "success" { + assert_eq!(http, serde_json::json!({"state": "Ready"})); + } else { + assert_eq!(http["details"]["description"], encoded); + } + } +}