run_matrix.py #1

#!/usr/bin/env python3
"""Build/provision/test disposable systemd containers without host data mounts."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import time

HERE = Path(__file__).resolve().parent
PLATFORMS = {
    "ubuntu": {"image": "sdp-assurance-ubuntu:26.04", "base": "ubuntu:26.04", "dockerfile": "Dockerfile.ubuntu"},
    "rhel": {"image": "sdp-assurance-rhel:10.2", "base": "registry.access.redhat.com/ubi10/ubi-init:10.2", "dockerfile": "Dockerfile.rhel"},
    "rhel8": {"image": "sdp-assurance-rhel8:8.10", "base": "registry.access.redhat.com/ubi8/ubi-init:8.10", "dockerfile": "Dockerfile.rhel8"},
    "rhel9": {"image": "sdp-assurance-rhel9:9.8", "base": "registry.access.redhat.com/ubi9/ubi-init:9.8", "dockerfile": "Dockerfile.rhel9"},
}
LABEL = "io.perforce.sdp.disposable"
PRIMARY_STAGES = ("offline", "integration", "runtime", "maintenance", "baseline-comparison", "python-installer")
DISTRO_STAGES = tuple("distro-" + stage for stage in PRIMARY_STAGES if stage != "python-installer")
HOST_STAGE_FILES = tuple(stage + ".log" for stage in PRIMARY_STAGES + DISTRO_STAGES) + ("test-outcomes.json",)
# python-installer.log inside the container is the original compilation log,
# not a copy of the host verification-stage output. Preserve it for reuse.
CONTAINER_STAGE_FILES = tuple(name for name in HOST_STAGE_FILES if name != "python-installer.log") + ("baseline-comparison.json", "distro-baseline-comparison.json", "coverage.json")


class Matrix:
    def __init__(self, args):
        self.args = args
        self.docker = args.docker
        self.results = args.results.resolve()
        self.results.mkdir(parents=True, exist_ok=True)

    def command(self, arguments, logfile=None, check=True):
        command = [self.docker, *arguments]
        print("docker " + " ".join(arguments), flush=True)
        if logfile:
            with logfile.open("w") as output:
                result = subprocess.run(command, stdout=output, stderr=subprocess.STDOUT)
        else:
            result = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if check and result.returncode:
            detail = "see " + str(logfile) if logfile else result.stderr
            raise RuntimeError("Docker command failed: " + detail)
        return result

    def name(self, platform):
        return self.args.name_prefix + "-" + platform

    def inspect_owned(self, platform):
        result = self.command(["inspect", self.name(platform)], check=False)
        if result.returncode:
            return None
        data = json.loads(result.stdout)[0]
        if data.get("Config", {}).get("Labels", {}).get(LABEL) != "true":
            raise RuntimeError("Refusing to use an existing container without the disposable label")
        return data

    def build(self, platform):
        folder = self.results / platform
        folder.mkdir(exist_ok=True)
        definition = HERE / PLATFORMS[platform]["dockerfile"]
        self.command(["build", "--pull", "-f", str(definition), "-t", PLATFORMS[platform]["image"], str(HERE)], folder / "image-build.log")
        info = self.command(["image", "inspect", PLATFORMS[platform]["image"]])
        (folder / "image-inspect.json").write_text(info.stdout)
        (folder / "image-build-context.json").write_text(json.dumps({
            "dockerfile": definition.name, "dockerfile_sha256": hashlib.sha256(definition.read_bytes()).hexdigest(),
            "image": PLATFORMS[platform]["image"], "base": PLATFORMS[platform]["base"],
            "image_id": json.loads(info.stdout)[0]["Id"], "build_exit_code": 0,
        }, indent=2) + "\n")

    def start(self, platform):
        name = self.name(platform)
        folder = self.results / platform
        image_record = folder / "image-inspect.json"
        if not image_record.is_file():
            raise RuntimeError("Build this platform first; its recorded image identity is required")
        expected_image = json.loads(image_record.read_text())[0]["Id"]
        data = self.inspect_owned(platform)
        if data:
            if data.get("Image") != expected_image:
                raise RuntimeError("Existing container does not use the recorded built image; use a new name prefix")
            if not data["State"]["Running"]:
                self.command(["start", name])
        else:
            # Privileged systemd runs in the dedicated disposable Docker VM/container.
            # No host directory, credentials, Docker socket or network port is mounted/published.
            self.command(["run", "--detach", "--name", name, "--label", LABEL + "=true",
                          "--privileged", "--cgroupns=private", "--tmpfs", "/run", "--tmpfs", "/run/lock", "--tmpfs", "/tmp",
                          expected_image])
        data = self.inspect_owned(platform)
        if not data or data.get("Image") != expected_image:
            raise RuntimeError("Running container image does not match the recorded build")
        (folder / "image-reproducibility.json").write_text(json.dumps({
            "built_image_id": expected_image, "container_image_id": data["Image"],
            "matches": True, "container": name,
        }, indent=2) + "\n")
        for _ in range(60):
            status = self.command(["exec", name, "systemctl", "is-system-running"], check=False)
            if status.stdout.strip() in ("running", "degraded"):
                self.command(["exec", name, "touch", "/run/sdp-container-test"])
                return
            time.sleep(1)
        raise RuntimeError("systemd did not reach running/degraded state: " + name)

    def bootstrap(self, platform):
        if self.inspect_owned(platform) is None:
            raise RuntimeError("Start the disposable container before bootstrapping")
        source = self.args.source
        if source is None or not (source / ".export-manifest.json").is_file():
            raise RuntimeError("--source must be a clean export containing .export-manifest.json")
        baseline = getattr(self.args, "baseline_source", None)
        if baseline is None or not (baseline / ".export-manifest.json").is_file() or not (baseline / ".baseline-info.json").is_file():
            raise RuntimeError("--baseline-source must be an immutable baseline export with manifest and provenance information")
        folder = self.results / platform
        folder.mkdir(exist_ok=True)
        previous = [folder / name for name in HOST_STAGE_FILES if (folder / name).is_file()]
        if previous:
            archive = folder / ("previous-validation-" + str(time.time_ns()))
            archive.mkdir()
            for path in previous:
                shutil.move(str(path), archive / path.name)
            for name in ("source-manifest.json", "baseline-source-manifest.json", "coverage.json", "baseline-comparison.json", "distro-baseline-comparison.json"):
                if (folder / name).is_file():
                    shutil.copy2(folder / name, archive / name)
        name = self.name(platform)
        # A reset invalidates behavioral evidence from the previous fixture.
        # The completed Python installer receipt remains valid only when its
        # installer hash passes python_runtime.sh's independent verification.
        self.command(["exec", name, "rm", "-f", *["/results/" + item for item in CONTAINER_STAGE_FILES]])
        self.command(["exec", name, "rm", "-rf", "/inputs/sdp"])
        self.command(["exec", name, "mkdir", "-p", "/inputs/sdp", "/results"])
        self.command(["cp", str(source.resolve()) + "/.", name + ":/inputs/sdp"])
        self.command(["exec", name, "rm", "-rf", "/inputs/baseline-sdp"])
        self.command(["exec", name, "mkdir", "-p", "/inputs/baseline-sdp"])
        self.command(["cp", str(baseline.resolve()) + "/.", name + ":/inputs/baseline-sdp"])
        self.command(["cp", str(HERE / "bootstrap.sh"), name + ":/inputs/bootstrap.sh"])
        command = ["exec", name, "bash", "/inputs/bootstrap.sh", self.args.release]
        if self.args.resume:
            command.append("--resume")
        self.command(command, folder / "bootstrap-console.log")

    def select_python(self, platform, variant):
        # Only generated prefixes in our labelled disposable container are selectable.
        self.command(["exec", self.name(platform), "bash", "/inputs/sdp/test/containers/python_runtime.sh", "select", variant])

    def test(self, platform):
        if self.inspect_owned(platform) is None:
            raise RuntimeError("Missing disposable container")
        name = self.name(platform)
        folder = self.results / platform
        folder.mkdir(exist_ok=True)
        root_exec = ["exec", "--env", "PATH=/p4/common/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
                     "--env", "SDP_BASELINE_SOURCE=/inputs/baseline-sdp", "--env", "SDP_CANDIDATE_SOURCE=/inputs/sdp"]
        user_exec = root_exec + ["--user", "perforce", "--workdir", "/p4/1"]
        commands = {
            "offline": root_exec + ["--env", "TMPDIR=/var/tmp", name, "/p4/common/python/bin/python3", "-m", "unittest", "discover", "-s", "/inputs/sdp/test", "-p", "test_*.py", "-v"],
            "integration": user_exec + [name, "bash", "/p4/sdp/test/sdp_test_harness.sh", "--root", "/srv/sdp-test-integration", "--instance", "1", "--port", "19999", "--os-user", "perforce", "--confirm-disposable"],
            "runtime": user_exec + [name, "/p4/common/python/bin/python3", "/inputs/sdp/test/containers/runtime_integration.py", "--root", "/srv/sdp-test-integration", "--instance", "1", "--port", "19999", "--os-user", "perforce", "--confirm-disposable"],
            "maintenance": user_exec + [name, "/p4/common/python/bin/python3", "/inputs/sdp/test/containers/maintenance_integration.py", "--root", "/srv/sdp-test-integration", "--instance", "1", "--port", "19999", "--os-user", "perforce", "--confirm-disposable"],
            "baseline-comparison": root_exec + ["--env", "TMPDIR=/var/tmp", name, "/p4/common/python/bin/python3", "/inputs/sdp/test/containers/baseline_comparison.py", "--baseline", "/inputs/baseline-sdp", "--candidate", "/inputs/sdp", "--output", "/results/baseline-comparison.json"],
            "python-installer": root_exec + [name, "bash", "/inputs/sdp/test/containers/python_runtime.sh", "verify"],
        }
        outcome_file = folder / "test-outcomes.json"
        outcomes = json.loads(outcome_file.read_text()) if outcome_file.is_file() else {}
        selected = getattr(self.args, "test_stage", None) or list(PRIMARY_STAGES + DISTRO_STAGES)
        try:
            for kind in selected:
                distro = kind.startswith("distro-")
                outcomes[kind] = 1
                self.select_python(platform, "distro" if distro else "baseline")
                command = list(commands[kind.removeprefix("distro-")])
                if distro and kind == "distro-baseline-comparison":
                    command[-1] = "/results/distro-baseline-comparison.json"
                outcomes[kind] = self.command(command, folder / (kind + ".log"), check=False).returncode
        finally:
            try:
                self.select_python(platform, "baseline")
                outcomes.pop("python-runtime-restore", None)
            except (OSError, RuntimeError) as error:
                outcomes["python-runtime-restore"] = 1
                raise RuntimeError("Failed to restore the primary Python runtime") from error
            finally:
                (folder / "test-outcomes.json").write_text(json.dumps(outcomes, indent=2) + "\n")
        return any(outcomes.values())

    def collect(self, platform):
        data = self.inspect_owned(platform)
        if not data:
            raise RuntimeError("Missing disposable container")
        folder = self.results / platform
        folder.mkdir(exist_ok=True)
        (folder / "container-inspect.json").write_text(json.dumps(data, indent=2) + "\n")
        # Stage output and exit codes belong to the host runner. A resumed or
        # manually inspected container may still contain older copies of them.
        saved_stage_files = {name: (folder / name).read_bytes() if (folder / name).is_file() else None
                             for name in HOST_STAGE_FILES}
        self.command(["cp", self.name(platform) + ":/results/.", str(folder)], check=False)
        for name, contents in saved_stage_files.items():
            path = folder / name
            if contents is not None:
                path.write_bytes(contents)
            elif path.is_file():
                path.unlink()  # Never manufacture an unrun stage from stale container output.
        self.command(["exec", self.name(platform), "journalctl", "-u", "p4d_1.service", "--no-pager"], folder / "service-journal.log", check=False)
        self.command(["exec", self.name(platform), "systemctl", "show", "p4d_1.service", "-p", "ActiveState", "-p", "SubState", "-p", "MainPID"], folder / "service-final-state.txt", check=False)
        coverage = HERE / "coverage.py"
        if coverage.is_file() and self.args.source:
            result = subprocess.run([os.sys.executable, str(coverage), "--source", str(self.args.source), "--results", str(folder), "--output", str(folder / "coverage.json")])
            if result.returncode:
                raise RuntimeError("Per-script coverage report failed")

    def cleanup(self, platform):
        if self.inspect_owned(platform):
            self.command(["rm", "--force", self.name(platform)])


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("stage", choices=("build", "start", "bootstrap", "test", "collect", "cleanup", "all"))
    parser.add_argument("--platform", choices=(*PLATFORMS, "both", "all"), default="all")
    parser.add_argument("--source", type=Path, help="Clean export from export_source.py")
    parser.add_argument("--baseline-source", type=Path, help="Immutable deployed-archive or submitted baseline export, copied without host mounts")
    parser.add_argument("--results", type=Path, required=True)
    parser.add_argument("--release", default="r26.1")
    parser.add_argument("--test-stage", action="append", choices=PRIMARY_STAGES + DISTRO_STAGES, help="Select one test stage; repeat to select several. Unselected outcomes are preserved.")
    parser.add_argument("--resume", action="store_true", help="Explicitly reset an existing marked disposable deployment during bootstrap")
    parser.add_argument("--name-prefix", default="sdp-assurance-matrix")
    parser.add_argument("--docker", default=shutil.which("docker") or "docker")
    args = parser.parse_args()
    if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*", args.name_prefix):
        parser.error("invalid name prefix")
    matrix = Matrix(args)
    platforms = list(PLATFORMS) if args.platform == "all" else (["ubuntu", "rhel"] if args.platform == "both" else [args.platform])
    failed = False
    for platform in platforms:
        try:
            if args.stage == "all":
                for stage in ("build", "start", "bootstrap", "test", "collect"):
                    failed = bool(getattr(matrix, stage)(platform)) or failed
            else:
                failed = bool(getattr(matrix, args.stage)(platform)) or failed
        except (OSError, RuntimeError, ValueError) as error:
            print("ERROR [%s]: %s" % (platform, error), flush=True)
            failed = True
            if args.stage not in ("build", "cleanup"):
                try:
                    matrix.collect(platform)
                except (OSError, RuntimeError, ValueError):
                    pass
    return int(failed)


if __name__ == "__main__":
    raise SystemExit(main())