export_baseline.py #1

#!/usr/bin/env python3
"""Export immutable submitted Perforce revisions for paired regression tests.

Never syncs or changes the workspace. Pending/local bytes are not baseline data.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import io
import json
import marshal
import os
from pathlib import Path, PurePosixPath
import subprocess


def records(arguments):
    result = subprocess.run(["p4", "-G", *arguments], capture_output=True)
    stream = io.BytesIO(result.stdout)
    rows, errors = [], []
    while True:
        try:
            row = marshal.load(stream)
        except EOFError:
            break
        row = {(k.decode() if isinstance(k, bytes) else k):
               (v.decode() if isinstance(v, bytes) else v) for k, v in row.items()}
        if row.get("code") == "error":
            errors.append(row.get("data", str(row)))
        elif row.get("depotFile"):
            rows.append(row)
    if result.returncode or errors:
        raise RuntimeError("Perforce baseline query failed: " + "\n".join(errors) + result.stderr.decode())
    return rows


def relative_path(depot_root, depot_file):
    prefix = depot_root.rstrip("/") + "/"
    if not depot_file.startswith(prefix):
        raise ValueError("File is outside the requested baseline depot")
    relative = PurePosixPath(depot_file[len(prefix):])
    if relative.is_absolute() or not relative.parts or ".." in relative.parts:
        raise ValueError("Invalid baseline path")
    return relative.as_posix()


def export(depot_root, change, destination):
    if not depot_root.startswith("//") or any(c in depot_root for c in "@#*%") or "..." in depot_root:
        raise ValueError("Use an explicit depot directory without revision syntax or wildcards")
    if change <= 0:
        raise ValueError("A positive submitted changelist is required")
    if destination.exists():
        raise ValueError("Baseline destination must be new")
    inventory = records(["files", depot_root.rstrip("/") + "/...@" + str(change)])
    inventory = [row for row in inventory if row.get("action") not in ("delete", "move/delete")]
    if not inventory:
        raise ValueError("No active files at the requested changelist")
    paths = [relative_path(depot_root, row["depotFile"]) for row in inventory]
    if len(set(paths)) != len(paths):
        raise ValueError("Duplicate baseline paths")
    destination.mkdir(parents=True)

    def download(item):
        row, relative = item
        target = destination / relative
        target.parent.mkdir(parents=True, exist_ok=True)
        revision = row["depotFile"] + "#" + str(row["rev"])
        result = subprocess.run(["p4", "print", "-q", revision], capture_output=True)
        if result.returncode:
            raise RuntimeError("Failed to export " + revision + ": " + result.stderr.decode())
        data = result.stdout
        symlink = None
        if row.get("type", "").startswith("symlink"):
            symlink = data.decode().rstrip("\n")
            # An exported baseline must not redirect a test outside its tree.
            if Path(symlink).is_absolute() or not (target.parent / symlink).resolve().is_relative_to(destination.resolve()):
                raise ValueError("Unsafe baseline symlink: " + relative)
            target.symlink_to(symlink)
            data = symlink.encode()
        else:
            target.write_bytes(data)
            target.chmod(0o755 if "x" in row.get("type", "").partition("+")[2] else 0o644)
        return {"path": relative, "depot": row["depotFile"], "haveRev": str(row["rev"]),
                "action": "", "source_kind": "submitted", "baseline_change": change,
                "sha256": hashlib.sha256(data).hexdigest(), "symlink": symlink}

    with ThreadPoolExecutor(max_workers=6) as pool:
        manifest = sorted(pool.map(download, zip(inventory, paths)), key=lambda row: row["path"])
    # Publication of this file marks a complete export; partial downloads cannot
    # be accepted as a baseline by the runner or acceptance gate.
    (destination / ".export-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
    (destination / ".baseline-info.json").write_text(json.dumps({
        "source_kind": "submitted", "depot_root": depot_root, "change": change,
        "production_equivalence": "Unconfirmed until the operator identifies the deployed revision",
        "files": len(manifest)}, indent=2) + "\n")
    return manifest


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("depot_root")
    parser.add_argument("change", type=int)
    parser.add_argument("destination", type=Path)
    args = parser.parse_args()
    try:
        manifest = export(args.depot_root, args.change, args.destination)
    except (OSError, ValueError, RuntimeError) as error:
        parser.exit(1, str(error) + "\n")
    print("Exported %d submitted files at changelist %d" % (len(manifest), args.change))


if __name__ == "__main__":
    main()