package main
import (
"bytes"
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/sirupsen/logrus"
)
func TestParseQueueStatus(t *testing.T) {
status, err := parseQueueStatus("File transfers: 13 active/94723 total, bytes: 68.2M active/153.7G total.\n")
if err != nil {
t.Fatal(err)
}
if status.ActiveTransfers != 13 || status.TotalTransfers != 94723 || status.TotalBytes != "153.7G" {
t.Fatalf("unexpected queue status: %#v", status)
}
}
func TestParseQueueStatusRejectsUnexpectedOutput(t *testing.T) {
if _, err := parseQueueStatus("File transfers unavailable"); err == nil {
t.Fatal("parseQueueStatus() succeeded for invalid output")
}
}
func TestParseTrackOutput(t *testing.T) {
track := parseTrackOutput(`--- lapse 1209s
--- usage 40809+83885us 25860680+40io 0+0net 24708k 42135pf
--- db.monitor
--- pages in+out+cached 2+4+4096
--- locks read/write 317/2 rows get+pos+scan put+del 317+0+0 2+0
--- total lock wait+held read/write 3ms+47ms/0ms+0ms
--- lbr Rcs
--- opens+closes+checkins+exists 0+0+0+414533
--- lbr Binary
--- opens+closes+checkins+exists 0+0+0+1196378
`)
if track.Lapse != 1209*time.Second || track.UserCPU != 40809*time.Millisecond || track.SystemCPU != 83885*time.Millisecond {
t.Fatalf("unexpected command metrics: %#v", track)
}
monitor := track.Tables["db.monitor"]
if monitor.ReadWait != 3*time.Millisecond || monitor.ReadHeld != 47*time.Millisecond || monitor.WriteWait != 0 || monitor.WriteHeld != 0 {
t.Fatalf("unexpected db.monitor metrics: %#v", monitor)
}
if rcs := track.Libraries["Rcs"]; rcs.Exists != 414533 {
t.Fatalf("unexpected Rcs metrics: %#v", rcs)
}
if binary := track.Libraries["Binary"]; binary.Exists != 1196378 {
t.Fatalf("unexpected Binary metrics: %#v", binary)
}
}
func TestVersionPrint(t *testing.T) {
change := strings.TrimSuffix(strings.TrimPrefix(VersionChange, "$Change: "), " $")
want := "p4seedreplica, version v1.2.3 (change: " + change + ")\n go version: " + GoVersion
if got := VersionPrint("p4seedreplica", "v1.2.3"); got != want {
t.Fatalf("VersionPrint() = %q, want %q", got, want)
}
}
func TestMissingRevision(t *testing.T) {
for line, want := range map[string]string{
"//depot/a/file#42 - MISSING! change 123": "//depot/a/file#42",
"//depot/b/file#7 - BAD! change 124": "//depot/b/file#7",
"//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5 - edit change 142121 (text) 87710FE48D73BA954E0C585D44DA8F32 MISSING!": "//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5",
} {
if got, ok := missingRevision(line); !ok || got != want {
t.Errorf("missingRevision(%q) = %q, %t", line, got, ok)
}
}
if _, ok := missingRevision("//depot/a/file#42 - ok"); ok {
t.Error("missingRevision() matched a non-missing revision")
}
}
func TestDiscoverRevisions(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"//depot/a/..."*) printf '%s\n' '//depot/a/file#1 - MISSING! change 1' ;;
*"//depot/b/..."*) printf '%s\n' '//depot/b/file#2 - BAD! change 2' ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
filespecs := filepath.Join(directory, "paths.txt")
if err := os.WriteFile(filespecs, []byte("//depot/a/...\n//depot/b/...\n"), 0644); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "revisions.txt")
logger := logrus.New()
logger.SetOutput(os.Stderr)
found, err := discoverRevisions(context.Background(), p4Client{bin: p4bin, logger: logger}, filespecs, output, 2, false)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
if found != 2 || !strings.Contains(string(contents), "//depot/a/file#1") || !strings.Contains(string(contents), "//depot/b/file#2") {
t.Fatalf("found %d revisions, output %q", found, contents)
}
}
func TestDiscoverRevisionsAcceptsMissingRecordsOnStderr(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
printf '%s\n' '//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5 - edit change 142121 (text) 87710FE48D73BA954E0C585D44DA8F32 MISSING!' >&2
exit 1
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
filespecs := filepath.Join(directory, "paths.txt")
if err := os.WriteFile(filespecs, []byte("//streamsAdvertiserPortal/portalUI/...\n"), 0644); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "revisions.txt")
found, err := discoverRevisions(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, filespecs, output, 1, false)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
if found != 1 || string(contents) != "//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5\n" {
t.Fatalf("found %d revisions, output %q", found, contents)
}
}
func TestRunVerifyCommand(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-Ztrack"*) ;;
*) printf 'missing -Ztrack: %s\n' "$*" >&2; exit 1 ;;
esac
printf '%s\n' '//depot/a/file#1 - MISSING! change 1'
printf '%s\n' '--- lapse 2s' >&2
printf '%s\n' '--- usage 3000+4000us 0+0io 0+0net 0k 0pf' >&2
printf '%s\n' '--- db.monitor' >&2
printf '%s\n' '--- total lock wait+held read/write 5ms+6ms/7ms+8ms' >&2
printf '%s\n' '--- lbr Rcs' >&2
printf '%s\n' '--- opens+closes+checkins+exists 1+2+3+4' >&2
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
paths := filepath.Join(directory, "paths.txt")
if err := os.WriteFile(paths, []byte("//depot/a/...\n"), 0644); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "missing.txt")
var stdout, stderr bytes.Buffer
if err := run([]string{"verify", "--paths", paths, "--output", output, "--p4bin", p4bin, "--debug"}, &stdout, &stderr); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
if string(contents) != "//depot/a/file#1\n" || !strings.Contains(stdout.String(), "Wrote 1 missing revisions") || !strings.Contains(stdout.String(), "P4 tracking verify verify: lapse=2s user=3s system=4s") || !strings.Contains(stdout.String(), "P4 tracking verify verify db.monitor: read wait/held=5ms/6ms write wait/held=7ms/8ms") || !strings.Contains(stdout.String(), "P4 tracking verify verify lbr Rcs: opens/closes/checkins/exists=1/2/3/4") || !strings.Contains(stderr.String(), "P4 tracking verify: lapse=2s user=3s system=4s") || !strings.Contains(stderr.String(), "P4 tracking verify lbr Rcs: opens/closes/checkins/exists=1/2/3/4") {
t.Fatalf("stdout=%q, output=%q", stdout.String(), contents)
}
}
func TestRunSeedCommandDryRun(t *testing.T) {
directory := t.TempDir()
input := filepath.Join(directory, "missing.txt")
if err := os.WriteFile(input, []byte("//depot/a/file#1\n"), 0644); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
if err := run([]string{"seed", "--input", input, "--dry-run"}, &stdout, &stderr); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(input + ".seedreplica.json"); err != nil {
t.Fatal(err)
}
}
func TestDiscoverRevisionsRejectsInvalidFilespec(t *testing.T) {
directory := t.TempDir()
filespecs := filepath.Join(directory, "paths.txt")
if err := os.WriteFile(filespecs, []byte("not-a-depot-path\n"), 0644); err != nil {
t.Fatal(err)
}
_, err := discoverRevisions(context.Background(), p4Client{logger: logrus.New()}, filespecs, filepath.Join(directory, "revisions.txt"), 1, false)
if err == nil || !strings.Contains(err.Error(), "invalid depot path") {
t.Fatalf("discoverRevisions() error = %v", err)
}
}
func TestDiscoverRevisionsRefusesExistingOutput(t *testing.T) {
directory := t.TempDir()
filespecs := filepath.Join(directory, "paths.txt")
output := filepath.Join(directory, "revisions.txt")
if err := os.WriteFile(filespecs, []byte("//depot/a/...\n"), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(output, []byte("old data\n"), 0644); err != nil {
t.Fatal(err)
}
if _, err := discoverRevisions(context.Background(), p4Client{logger: logrus.New()}, filespecs, output, 1, false); err == nil {
t.Fatal("discoverRevisions() succeeded without --replace-input")
}
}
func TestGenerateDepotPathsSubdividesTimedOutDepots(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-ztag depots"*) cat <<'EOF'
... depot depot
... type local
... depot streams
... type remote
... depot remote
... type remote
EOF
;;
*"sizes -sh //depot/..."*) sleep 1 ;;
*"dirs -D //depot/*"*) printf '%s\n' '//depot/alpha' '//depot/beta' '//helix-installer/* - no such file(s)./...' ;;
*"sizes -sh //depot/alpha/..."*) : ;;
*"sizes -sh //depot/beta/..."*) : ;;
*) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "depot-paths.txt")
count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, 50*time.Millisecond, 1_000_000, 20)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
got := string(contents)
for _, path := range []string{"//depot/alpha/...", "//depot/beta/..."} {
if !strings.Contains(got, path+"\n") {
t.Errorf("generated paths %q do not contain %q", got, path)
}
}
if count != 2 || strings.Contains(got, "//depot/...\n") || strings.Contains(got, "//remote/") {
t.Fatalf("unexpected generated paths: count=%d, paths=%q", count, got)
}
}
func TestGenerateDepotPathsProcessesDepotsInParallel(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-ztag depots"*) cat <<'EOF'
... depot alpha
... type local
... depot beta
... type local
EOF
;;
*"sizes -sh //alpha/..."*|*"sizes -sh //beta/..."*) sleep 1 ;;
*) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "depot-paths.txt")
started := time.Now()
count, err := generateDepotPathsWithWorkers(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, 2*time.Second, 1_000_000, 20, 2)
if err != nil {
t.Fatal(err)
}
if elapsed := time.Since(started); elapsed >= 1500*time.Millisecond {
t.Fatalf("generated %d paths in %s; depot paths were not processed in parallel", count, elapsed)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
if count != 2 || string(contents) != "//alpha/...\n//beta/...\n" {
t.Fatalf("generated paths: count=%d, paths=%q", count, contents)
}
}
func TestGenerateDepotPathsGroupsStreamsUsingDepotDepth(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-ztag depots"*) cat <<'EOF'
... depot streamsAdvertiserPortal
... type stream
... depth 3
EOF
;;
*"dirs //streamsAdvertiserPortal/*/*/*"*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a' '//streamsAdvertiserPortal/team/product/b' '//streamsAdvertiserPortal/other/product/c' ;;
*"sizes -sh //streamsAdvertiserPortal/team/product/a/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a/... - 600000 files 1 GB' ;;
*"sizes -sh //streamsAdvertiserPortal/team/product/b/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/b/... - 600000 files 1 GB' ;;
*"sizes -sh //streamsAdvertiserPortal/other/product/c/..."*) printf '%s\n' '//streamsAdvertiserPortal/other/product/c/... - 600000 files 1 GB' ;;
*) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "depot-paths.txt")
count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 3)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
want := "//streamsAdvertiserPortal/other/...\n//streamsAdvertiserPortal/team/product/a/...\n//streamsAdvertiserPortal/team/product/b/...\n"
if count != 3 || string(contents) != want {
t.Fatalf("generated paths: count=%d, paths=%q, want %q", count, contents, want)
}
}
func TestGenerateDepotPathsGroupsMultipleStreamsWithinFileLimit(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-ztag depots"*) cat <<'EOF'
... depot streamsAdvertiserPortal
... type stream
... depth 3
EOF
;;
*"dirs //streamsAdvertiserPortal/*/*/*"*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a' '//streamsAdvertiserPortal/team/product/b' '//streamsAdvertiserPortal/other/product/c' ;;
*"sizes -sh //streamsAdvertiserPortal/team/product/a/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a/... - 400000 files 1 GB' ;;
*"sizes -sh //streamsAdvertiserPortal/team/product/b/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/b/... - 400000 files 1 GB' ;;
*"sizes -sh //streamsAdvertiserPortal/other/product/c/..."*) printf '%s\n' '//streamsAdvertiserPortal/other/product/c/... - 400000 files 1 GB' ;;
*) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "depot-paths.txt")
count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 3)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
want := "//streamsAdvertiserPortal/other/...\n//streamsAdvertiserPortal/team/...\n"
if count != 2 || string(contents) != want {
t.Fatalf("generated paths: count=%d, paths=%q, want %q", count, contents, want)
}
}
func TestGenerateDepotPathsBatchesUnrelatedStreamsWithinFileLimit(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-ztag depots"*) cat <<'EOF'
... depot streams
... type stream
... depth 1
EOF
;;
*"dirs //streams/*"*) printf '%s\n' '//streams/a' '//streams/b' '//streams/c' ;;
*"sizes -sh //streams/a/..."*) printf '%s\n' '//streams/a/... - 400000 files 1 GB' ;;
*"sizes -sh //streams/b/..."*) printf '%s\n' '//streams/b/... - 400000 files 1 GB' ;;
*"sizes -sh //streams/c/..."*) printf '%s\n' '//streams/c/... - 400000 files 1 GB' ;;
*) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "depot-paths.txt")
count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 3)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
want := "//streams/a/... //streams/b/...\n//streams/c/...\n"
if count != 2 || string(contents) != want {
t.Fatalf("generated paths: count=%d, paths=%q, want %q", count, contents, want)
}
}
func TestDiscoverRevisionsVerifiesMultiplePathsFromOneLine(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
if [ "$#" -ne 7 ] || [ "$1" != "-Ztrack" ] || [ "$6" != "//depot/a/..." ] || [ "$7" != "//depot/b/..." ]; then
printf 'unexpected p4 arguments: %s\n' "$*" >&2
exit 1
fi
printf '%s\n' '//depot/a/file#1 - MISSING! change 1'
printf '%s\n' '//depot/b/file#2 - MISSING! change 2'
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
filespecs := filepath.Join(directory, "paths.txt")
if err := os.WriteFile(filespecs, []byte("//depot/a/... //depot/b/...\n"), 0644); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "revisions.txt")
found, err := discoverRevisions(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, filespecs, output, 1, false)
if err != nil {
t.Fatal(err)
}
if found != 2 {
t.Fatalf("found %d revisions, want 2", found)
}
}
func TestGenerateDepotPathsIgnoresEmptyStreamDepot(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-ztag depots"*) cat <<'EOF'
... depot streams
... type stream
... streamdepth 2
EOF
;;
*"dirs //streams/*/*"*) : ;;
*) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "depot-paths.txt")
count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 20)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
if count != 0 || len(contents) != 0 {
t.Fatalf("generated paths: count=%d, paths=%q", count, contents)
}
}
func TestGenerateDepotPathsIgnoresStreamDepotWithoutDepth(t *testing.T) {
directory := t.TempDir()
p4bin := filepath.Join(directory, "p4")
script := `#!/bin/sh
case "$*" in
*"-ztag depots"*) cat <<'EOF'
... depot streamsAdvertiserPortal
... type stream
EOF
;;
*) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;;
esac
`
if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil {
t.Fatal(err)
}
output := filepath.Join(directory, "depot-paths.txt")
count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 20)
if err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(output)
if err != nil {
t.Fatal(err)
}
if count != 0 || len(contents) != 0 {
t.Fatalf("generated paths: count=%d, paths=%q", count, contents)
}
}
func TestReadBatchTracksOffsetAndSkipsBlankLines(t *testing.T) {
input := filepath.Join(t.TempDir(), "revisions.txt")
content := "//depot/a#1\n\n//depot/b#2\n//depot/c#3\n"
if err := os.WriteFile(input, []byte(content), 0644); err != nil {
t.Fatal(err)
}
batch, offset, linesRead, eof, err := readBatch(input, 0, 0, 2)
if err != nil {
t.Fatal(err)
}
if eof || linesRead != 3 || len(batch) != 2 || batch[1] != "//depot/b#2" {
t.Fatalf("unexpected first batch: %#v, offset=%d, lines=%d, eof=%t", batch, offset, linesRead, eof)
}
batch, offset, linesRead, eof, err = readBatch(input, offset, linesRead, 2)
if err != nil {
t.Fatal(err)
}
if !eof || linesRead != 4 || len(batch) != 1 || batch[0] != "//depot/c#3" || offset != int64(len(content)) {
t.Fatalf("unexpected second batch: %#v, offset=%d, lines=%d, eof=%t", batch, offset, linesRead, eof)
}
}
func TestCheckpointRoundTrip(t *testing.T) {
input := filepath.Join(t.TempDir(), "revisions.txt")
checkpointPath := input + ".seedreplica.json"
state := checkpoint{Input: input, Offset: 42, LinesRead: 3, Processed: 2, Batches: 1}
if err := writeCheckpoint(checkpointPath, state); err != nil {
t.Fatal(err)
}
loaded, err := loadCheckpoint(checkpointPath, input)
if err != nil {
t.Fatal(err)
}
if loaded.Offset != state.Offset || loaded.Processed != state.Processed || loaded.Batches != state.Batches {
t.Fatalf("checkpoint = %#v, want %#v", loaded, state)
}
}