#!/usr/bin/env python3 """Build/provision/test disposable systemd containers without host data mounts.""" import argparse import hashlib import json import os from pathlib import Path import re import shutil import subprocess import time HERE = Path(__file__).resolve().parent PLATFORMS = { "ubuntu": {"image": "sdp-assurance-ubuntu:26.04", "base": "ubuntu:26.04", "dockerfile": "Dockerfile.ubuntu"}, "rhel": {"image": "sdp-assurance-rhel:10.2", "base": "registry.access.redhat.com/ubi10/ubi-init:10.2", "dockerfile": "Dockerfile.rhel"}, "rhel8": {"image": "sdp-assurance-rhel8:8.10", "base": "registry.access.redhat.com/ubi8/ubi-init:8.10", "dockerfile": "Dockerfile.rhel8"}, "rhel9": {"image": "sdp-assurance-rhel9:9.8", "base": "registry.access.redhat.com/ubi9/ubi-init:9.8", "dockerfile": "Dockerfile.rhel9"}, } LABEL = "io.perforce.sdp.disposable" PRIMARY_STAGES = ("offline", "integration", "runtime", "maintenance", "baseline-comparison", "python-installer") DISTRO_STAGES = tuple("distro-" + stage for stage in PRIMARY_STAGES if stage != "python-installer") HOST_STAGE_FILES = tuple(stage + ".log" for stage in PRIMARY_STAGES + DISTRO_STAGES) + ("test-outcomes.json",) # python-installer.log inside the container is the original compilation log, # not a copy of the host verification-stage output. Preserve it for reuse. CONTAINER_STAGE_FILES = tuple(name for name in HOST_STAGE_FILES if name != "python-installer.log") + ("baseline-comparison.json", "distro-baseline-comparison.json", "coverage.json") class Matrix: def __init__(self, args): self.args = args self.docker = args.docker self.results = args.results.resolve() self.results.mkdir(parents=True, exist_ok=True) def command(self, arguments, logfile=None, check=True): command = [self.docker, *arguments] print("docker " + " ".join(arguments), flush=True) if logfile: with logfile.open("w") as output: result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT) else: result = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if check and result.returncode: detail = "see " + str(logfile) if logfile else result.stderr raise RuntimeError("Docker command failed: " + detail) return result def name(self, platform): return self.args.name_prefix + "-" + platform def inspect_owned(self, platform): result = self.command(["inspect", self.name(platform)], check=False) if result.returncode: return None data = json.loads(result.stdout)[0] if data.get("Config", {}).get("Labels", {}).get(LABEL) != "true": raise RuntimeError("Refusing to use an existing container without the disposable label") return data def build(self, platform): folder = self.results / platform folder.mkdir(exist_ok=True) definition = HERE / PLATFORMS[platform]["dockerfile"] self.command(["build", "--pull", "-f", str(definition), "-t", PLATFORMS[platform]["image"], str(HERE)], folder / "image-build.log") info = self.command(["image", "inspect", PLATFORMS[platform]["image"]]) (folder / "image-inspect.json").write_text(info.stdout) (folder / "image-build-context.json").write_text(json.dumps({ "dockerfile": definition.name, "dockerfile_sha256": hashlib.sha256(definition.read_bytes()).hexdigest(), "image": PLATFORMS[platform]["image"], "base": PLATFORMS[platform]["base"], "image_id": json.loads(info.stdout)[0]["Id"], "build_exit_code": 0, }, indent=2) + "\n") def start(self, platform): name = self.name(platform) folder = self.results / platform image_record = folder / "image-inspect.json" if not image_record.is_file(): raise RuntimeError("Build this platform first; its recorded image identity is required") expected_image = json.loads(image_record.read_text())[0]["Id"] data = self.inspect_owned(platform) if data: if data.get("Image") != expected_image: raise RuntimeError("Existing container does not use the recorded built image; use a new name prefix") if not data["State"]["Running"]: self.command(["start", name]) else: # Privileged systemd runs in the dedicated disposable Docker VM/container. # No host directory, credentials, Docker socket or network port is mounted/published. self.command(["run", "--detach", "--name", name, "--label", LABEL + "=true", "--privileged", "--cgroupns=private", "--tmpfs", "/run", "--tmpfs", "/run/lock", "--tmpfs", "/tmp", expected_image]) data = self.inspect_owned(platform) if not data or data.get("Image") != expected_image: raise RuntimeError("Running container image does not match the recorded build") (folder / "image-reproducibility.json").write_text(json.dumps({ "built_image_id": expected_image, "container_image_id": data["Image"], "matches": True, "container": name, }, indent=2) + "\n") for _ in range(60): status = self.command(["exec", name, "systemctl", "is-system-running"], check=False) if status.stdout.strip() in ("running", "degraded"): self.command(["exec", name, "touch", "/run/sdp-container-test"]) return time.sleep(1) raise RuntimeError("systemd did not reach running/degraded state: " + name) def bootstrap(self, platform): if self.inspect_owned(platform) is None: raise RuntimeError("Start the disposable container before bootstrapping") source = self.args.source if source is None or not (source / ".export-manifest.json").is_file(): raise RuntimeError("--source must be a clean export containing .export-manifest.json") baseline = getattr(self.args, "baseline_source", None) if baseline is None or not (baseline / ".export-manifest.json").is_file() or not (baseline / ".baseline-info.json").is_file(): raise RuntimeError("--baseline-source must be an immutable baseline export with manifest and provenance information") folder = self.results / platform folder.mkdir(exist_ok=True) previous = [folder / name for name in HOST_STAGE_FILES if (folder / name).is_file()] if previous: archive = folder / ("previous-validation-" + str(time.time_ns())) archive.mkdir() for path in previous: shutil.move(str(path), archive / path.name) for name in ("source-manifest.json", "baseline-source-manifest.json", "coverage.json", "baseline-comparison.json", "distro-baseline-comparison.json"): if (folder / name).is_file(): shutil.copy2(folder / name, archive / name) name = self.name(platform) # A reset invalidates behavioral evidence from the previous fixture. # The completed Python installer receipt remains valid only when its # installer hash passes python_runtime.sh's independent verification. self.command(["exec", name, "rm", "-f", *["/results/" + item for item in CONTAINER_STAGE_FILES]]) self.command(["exec", name, "rm", "-rf", "/inputs/sdp"]) self.command(["exec", name, "mkdir", "-p", "/inputs/sdp", "/results"]) self.command(["cp", str(source.resolve()) + "/.", name + ":/inputs/sdp"]) self.command(["exec", name, "rm", "-rf", "/inputs/baseline-sdp"]) self.command(["exec", name, "mkdir", "-p", "/inputs/baseline-sdp"]) self.command(["cp", str(baseline.resolve()) + "/.", name + ":/inputs/baseline-sdp"]) self.command(["cp", str(HERE / "bootstrap.sh"), name + ":/inputs/bootstrap.sh"]) command = ["exec", name, "bash", "/inputs/bootstrap.sh", self.args.release] if self.args.resume: command.append("--resume") self.command(command, folder / "bootstrap-console.log") def select_python(self, platform, variant): # Only generated prefixes in our labelled disposable container are selectable. self.command(["exec", self.name(platform), "bash", "/inputs/sdp/test/containers/python_runtime.sh", "select", variant]) def test(self, platform): if self.inspect_owned(platform) is None: raise RuntimeError("Missing disposable container") name = self.name(platform) folder = self.results / platform folder.mkdir(exist_ok=True) root_exec = ["exec", "--env", "PATH=/p4/common/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "--env", "SDP_BASELINE_SOURCE=/inputs/baseline-sdp", "--env", "SDP_CANDIDATE_SOURCE=/inputs/sdp"] user_exec = root_exec + ["--user", "perforce", "--workdir", "/p4/1"] commands = { "offline": root_exec + ["--env", "TMPDIR=/var/tmp", name, "/p4/common/python/bin/python3", "-m", "unittest", "discover", "-s", "/inputs/sdp/test", "-p", "test_*.py", "-v"], "integration": user_exec + [name, "bash", "/p4/sdp/test/sdp_test_harness.sh", "--root", "/srv/sdp-test-integration", "--instance", "1", "--port", "19999", "--os-user", "perforce", "--confirm-disposable"], "runtime": user_exec + [name, "/p4/common/python/bin/python3", "/inputs/sdp/test/containers/runtime_integration.py", "--root", "/srv/sdp-test-integration", "--instance", "1", "--port", "19999", "--os-user", "perforce", "--confirm-disposable"], "maintenance": user_exec + [name, "/p4/common/python/bin/python3", "/inputs/sdp/test/containers/maintenance_integration.py", "--root", "/srv/sdp-test-integration", "--instance", "1", "--port", "19999", "--os-user", "perforce", "--confirm-disposable"], "baseline-comparison": root_exec + ["--env", "TMPDIR=/var/tmp", name, "/p4/common/python/bin/python3", "/inputs/sdp/test/containers/baseline_comparison.py", "--baseline", "/inputs/baseline-sdp", "--candidate", "/inputs/sdp", "--output", "/results/baseline-comparison.json"], "python-installer": root_exec + [name, "bash", "/inputs/sdp/test/containers/python_runtime.sh", "verify"], } outcome_file = folder / "test-outcomes.json" outcomes = json.loads(outcome_file.read_text()) if outcome_file.is_file() else {} selected = getattr(self.args, "test_stage", None) or list(PRIMARY_STAGES + DISTRO_STAGES) try: for kind in selected: distro = kind.startswith("distro-") outcomes[kind] = 1 self.select_python(platform, "distro" if distro else "baseline") command = list(commands[kind.removeprefix("distro-")]) if distro and kind == "distro-baseline-comparison": command[-1] = "/results/distro-baseline-comparison.json" outcomes[kind] = self.command(command, folder / (kind + ".log"), check=False).returncode finally: try: self.select_python(platform, "baseline") outcomes.pop("python-runtime-restore", None) except (OSError, RuntimeError) as error: outcomes["python-runtime-restore"] = 1 raise RuntimeError("Failed to restore the primary Python runtime") from error finally: (folder / "test-outcomes.json").write_text(json.dumps(outcomes, indent=2) + "\n") return any(outcomes.values()) def collect(self, platform): data = self.inspect_owned(platform) if not data: raise RuntimeError("Missing disposable container") folder = self.results / platform folder.mkdir(exist_ok=True) (folder / "container-inspect.json").write_text(json.dumps(data, indent=2) + "\n") # Stage output and exit codes belong to the host runner. A resumed or # manually inspected container may still contain older copies of them. saved_stage_files = {name: (folder / name).read_bytes() if (folder / name).is_file() else None for name in HOST_STAGE_FILES} self.command(["cp", self.name(platform) + ":/results/.", str(folder)], check=False) for name, contents in saved_stage_files.items(): path = folder / name if contents is not None: path.write_bytes(contents) elif path.is_file(): path.unlink() # Never manufacture an unrun stage from stale container output. self.command(["exec", self.name(platform), "journalctl", "-u", "p4d_1.service", "--no-pager"], folder / "service-journal.log", check=False) self.command(["exec", self.name(platform), "systemctl", "show", "p4d_1.service", "-p", "ActiveState", "-p", "SubState", "-p", "MainPID"], folder / "service-final-state.txt", check=False) coverage = HERE / "coverage.py" if coverage.is_file() and self.args.source: result = subprocess.run([os.sys.executable, str(coverage), "--source", str(self.args.source), "--results", str(folder), "--output", str(folder / "coverage.json")]) if result.returncode: raise RuntimeError("Per-script coverage report failed") def cleanup(self, platform): if self.inspect_owned(platform): self.command(["rm", "--force", self.name(platform)]) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("stage", choices=("build", "start", "bootstrap", "test", "collect", "cleanup", "all")) parser.add_argument("--platform", choices=(*PLATFORMS, "both", "all"), default="all") parser.add_argument("--source", type=Path, help="Clean export from export_source.py") parser.add_argument("--baseline-source", type=Path, help="Immutable deployed-archive or submitted baseline export, copied without host mounts") parser.add_argument("--results", type=Path, required=True) parser.add_argument("--release", default="r26.1") parser.add_argument("--test-stage", action="append", choices=PRIMARY_STAGES + DISTRO_STAGES, help="Select one test stage; repeat to select several. Unselected outcomes are preserved.") parser.add_argument("--resume", action="store_true", help="Explicitly reset an existing marked disposable deployment during bootstrap") parser.add_argument("--name-prefix", default="sdp-assurance-matrix") parser.add_argument("--docker", default=shutil.which("docker") or "docker") args = parser.parse_args() if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", args.name_prefix): parser.error("invalid name prefix") matrix = Matrix(args) platforms = list(PLATFORMS) if args.platform == "all" else (["ubuntu", "rhel"] if args.platform == "both" else [args.platform]) failed = False for platform in platforms: try: if args.stage == "all": for stage in ("build", "start", "bootstrap", "test", "collect"): failed = bool(getattr(matrix, stage)(platform)) or failed else: failed = bool(getattr(matrix, args.stage)(platform)) or failed except (OSError, RuntimeError, ValueError) as error: print("ERROR [%s]: %s" % (platform, error), flush=True) failed = True if args.stage not in ("build", "cleanup"): try: matrix.collect(platform) except (OSError, RuntimeError, ValueError): pass return int(failed) if __name__ == "__main__": raise SystemExit(main())