file_inventory.py #1

#!/usr/bin/env python3
"""Stream a complete depot inventory and atomically publish its pathname list.

Uses the p4 CLI's marshalled output; P4Python is not required. One query covers
files directly at each depot root, then one query per top-level directory.
The output retains the existing format: one depotFile pathname per line,
including deleted head revisions reported by `p4 files`.
"""

import argparse
import logging
import marshal
import os
from pathlib import Path
import stat
import subprocess
import sys
import tempfile


class InventoryError(RuntimeError):
    def __init__(self, message, returncode=1):
        super().__init__(message)
        self.returncode = returncode if returncode > 0 else 1


def _text(value):
    return value.decode("utf-8", "surrogateescape") if isinstance(value, bytes) else str(value)


def records(binary, args, allow_empty=False):
    """Yield stat records while distinguishing empty queries from failures."""
    with tempfile.TemporaryFile() as diagnostic:
        process = subprocess.Popen([binary, "-G", "-ztag", *args],
                                   stdout=subprocess.PIPE, stderr=diagnostic)
        errors = []
        empty = False
        try:
            # peek distinguishes a clean EOF from a truncated marshalled record.
            while process.stdout.peek(1):
                try:
                    raw = marshal.load(process.stdout)
                except (EOFError, ValueError, TypeError) as exc:
                    raise InventoryError("Invalid or truncated p4 output for " + args[0]) from exc
                if not isinstance(raw, dict):
                    raise InventoryError("Expected p4 dictionary output for " + args[0])
                rec = {_text(key): value for key, value in raw.items()}
                code = _text(rec.get("code", ""))
                if code == "stat":
                    yield rec
                elif code == "error":
                    generic = int(rec.get("generic", 0))
                    severity = int(rec.get("severity", 3))
                    # Perforce EV_EMPTY (0x11) means the query had no matches.
                    # Permission, connection and resource-limit errors never qualify.
                    if allow_empty and generic == 17 and severity <= 2:
                        empty = True
                    else:
                        errors.append(_text(rec.get("data", "p4 query failed")).strip())
                elif code in ("info", "text"):
                    detail = _text(rec.get("data", "")).strip()
                    if detail:
                        logging.info("%s", detail)
                else:
                    raise InventoryError("Unexpected p4 record type: " + code)
            returncode = process.wait()
            diagnostic.seek(0)
            stderr = diagnostic.read().decode("utf-8", "replace").strip()
            if stderr:
                logging.warning("%s", stderr)
            if errors or (returncode != 0 and not (empty and returncode == 1)):
                detail = "; ".join(errors) or stderr or "p4 query failed"
                raise InventoryError(detail, returncode)
        finally:
            process.stdout.close()
            if process.poll() is None:
                process.terminate()
                process.wait()


def _field(record, name):
    if name not in record:
        raise InventoryError("p4 inventory record is missing " + name)
    value = _text(record[name])
    if not value or "\n" in value or "\r" in value:
        raise InventoryError("Inventory names must fit on one line")
    return value


def create_inventory(binary, output):
    output = Path(output)
    temporary = None
    count = 0
    try:
        with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", errors="surrogateescape",
                                         newline="\n", dir=str(output.parent),
                                         prefix="." + output.name + ".", delete=False) as stream:
            temporary = Path(stream.name)
            for depot in records(binary, ["depots"], allow_empty=True):
                depot_path = "//" + _field(depot, "name")
                logging.info("Scanning %s", depot_path)
                # '*' never crosses '/', so these cannot overlap child scans.
                for rec in records(binary, ["files", depot_path + "/*"], allow_empty=True):
                    stream.write(_field(rec, "depotFile") + "\n")
                    count += 1
                for directory in records(binary, ["dirs", "-D", depot_path + "/*"], allow_empty=True):
                    directory_path = _field(directory, "dir").rstrip("/")
                    for rec in records(binary, ["files", directory_path + "/..."], allow_empty=True):
                        stream.write(_field(rec, "depotFile") + "\n")
                        count += 1
            stream.flush()
            os.fsync(stream.fileno())
        if output.exists():
            temporary.chmod(stat.S_IMODE(output.stat().st_mode))
        os.replace(temporary, output)
        temporary = None
    finally:
        if temporary is not None:
            temporary.unlink()
    logging.info("Published %d paths to %s", count, output)
    return count


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--p4", required=True, help="Configured SDP p4 executable")
    parser.add_argument("--output", default="p4_file_list.txt")
    args = parser.parse_args(argv)
    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
    try:
        create_inventory(args.p4, args.output)
    except InventoryError as exc:
        logging.error("%s", exc)
        return exc.returncode
    except (OSError, ValueError) as exc:
        logging.error("Unable to create file inventory: %s", exc)
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())