package main import ( "bytes" "context" "os" "path/filepath" "strings" "testing" "time" "github.com/sirupsen/logrus" ) func TestParseQueueStatus(t *testing.T) { status, err := parseQueueStatus("File transfers: 13 active/94723 total, bytes: 68.2M active/153.7G total.\n") if err != nil { t.Fatal(err) } if status.ActiveTransfers != 13 || status.TotalTransfers != 94723 || status.TotalBytes != "153.7G" { t.Fatalf("unexpected queue status: %#v", status) } } func TestParseQueueStatusRejectsUnexpectedOutput(t *testing.T) { if _, err := parseQueueStatus("File transfers unavailable"); err == nil { t.Fatal("parseQueueStatus() succeeded for invalid output") } } func TestParseTrackOutput(t *testing.T) { track := parseTrackOutput(`--- lapse 1209s --- usage 40809+83885us 25860680+40io 0+0net 24708k 42135pf --- db.monitor --- pages in+out+cached 2+4+4096 --- locks read/write 317/2 rows get+pos+scan put+del 317+0+0 2+0 --- total lock wait+held read/write 3ms+47ms/0ms+0ms --- lbr Rcs --- opens+closes+checkins+exists 0+0+0+414533 --- lbr Binary --- opens+closes+checkins+exists 0+0+0+1196378 `) if track.Lapse != 1209*time.Second || track.UserCPU != 40809*time.Millisecond || track.SystemCPU != 83885*time.Millisecond { t.Fatalf("unexpected command metrics: %#v", track) } monitor := track.Tables["db.monitor"] if monitor.ReadWait != 3*time.Millisecond || monitor.ReadHeld != 47*time.Millisecond || monitor.WriteWait != 0 || monitor.WriteHeld != 0 { t.Fatalf("unexpected db.monitor metrics: %#v", monitor) } if rcs := track.Libraries["Rcs"]; rcs.Exists != 414533 { t.Fatalf("unexpected Rcs metrics: %#v", rcs) } if binary := track.Libraries["Binary"]; binary.Exists != 1196378 { t.Fatalf("unexpected Binary metrics: %#v", binary) } } func TestVersionPrint(t *testing.T) { change := strings.TrimSuffix(strings.TrimPrefix(VersionChange, "$Change: "), " $") want := "p4seedreplica, version v1.2.3 (change: " + change + ")\n go version: " + GoVersion if got := VersionPrint("p4seedreplica", "v1.2.3"); got != want { t.Fatalf("VersionPrint() = %q, want %q", got, want) } } func TestMissingRevision(t *testing.T) { for line, want := range map[string]string{ "//depot/a/file#42 - MISSING! change 123": "//depot/a/file#42", "//depot/b/file#7 - BAD! change 124": "//depot/b/file#7", "//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5 - edit change 142121 (text) 87710FE48D73BA954E0C585D44DA8F32 MISSING!": "//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5", } { if got, ok := missingRevision(line); !ok || got != want { t.Errorf("missingRevision(%q) = %q, %t", line, got, ok) } } if _, ok := missingRevision("//depot/a/file#42 - ok"); ok { t.Error("missingRevision() matched a non-missing revision") } } func TestDiscoverRevisions(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"//depot/a/..."*) printf '%s\n' '//depot/a/file#1 - MISSING! change 1' ;; *"//depot/b/..."*) printf '%s\n' '//depot/b/file#2 - BAD! change 2' ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } filespecs := filepath.Join(directory, "paths.txt") if err := os.WriteFile(filespecs, []byte("//depot/a/...\n//depot/b/...\n"), 0644); err != nil { t.Fatal(err) } output := filepath.Join(directory, "revisions.txt") logger := logrus.New() logger.SetOutput(os.Stderr) found, err := discoverRevisions(context.Background(), p4Client{bin: p4bin, logger: logger}, filespecs, output, 2, false) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } if found != 2 || !strings.Contains(string(contents), "//depot/a/file#1") || !strings.Contains(string(contents), "//depot/b/file#2") { t.Fatalf("found %d revisions, output %q", found, contents) } } func TestDiscoverRevisionsAcceptsMissingRecordsOnStderr(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh printf '%s\n' '//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5 - edit change 142121 (text) 87710FE48D73BA954E0C585D44DA8F32 MISSING!' >&2 exit 1 ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } filespecs := filepath.Join(directory, "paths.txt") if err := os.WriteFile(filespecs, []byte("//streamsAdvertiserPortal/portalUI/...\n"), 0644); err != nil { t.Fatal(err) } output := filepath.Join(directory, "revisions.txt") found, err := discoverRevisions(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, filespecs, output, 1, false) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } if found != 1 || string(contents) != "//streamsAdvertiserPortal/portalUI/main/main/src/utils/request.js#5\n" { t.Fatalf("found %d revisions, output %q", found, contents) } } func TestRunVerifyCommand(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-Ztrack"*) ;; *) printf 'missing -Ztrack: %s\n' "$*" >&2; exit 1 ;; esac printf '%s\n' '//depot/a/file#1 - MISSING! change 1' printf '%s\n' '--- lapse 2s' >&2 printf '%s\n' '--- usage 3000+4000us 0+0io 0+0net 0k 0pf' >&2 printf '%s\n' '--- db.monitor' >&2 printf '%s\n' '--- total lock wait+held read/write 5ms+6ms/7ms+8ms' >&2 printf '%s\n' '--- lbr Rcs' >&2 printf '%s\n' '--- opens+closes+checkins+exists 1+2+3+4' >&2 ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } paths := filepath.Join(directory, "paths.txt") if err := os.WriteFile(paths, []byte("//depot/a/...\n"), 0644); err != nil { t.Fatal(err) } output := filepath.Join(directory, "missing.txt") var stdout, stderr bytes.Buffer if err := run([]string{"verify", "--paths", paths, "--output", output, "--p4bin", p4bin, "--debug"}, &stdout, &stderr); err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } if string(contents) != "//depot/a/file#1\n" || !strings.Contains(stdout.String(), "Wrote 1 missing revisions") || !strings.Contains(stdout.String(), "P4 tracking verify verify: lapse=2s user=3s system=4s") || !strings.Contains(stdout.String(), "P4 tracking verify verify db.monitor: read wait/held=5ms/6ms write wait/held=7ms/8ms") || !strings.Contains(stdout.String(), "P4 tracking verify verify lbr Rcs: opens/closes/checkins/exists=1/2/3/4") || !strings.Contains(stderr.String(), "P4 tracking verify: lapse=2s user=3s system=4s") || !strings.Contains(stderr.String(), "P4 tracking verify lbr Rcs: opens/closes/checkins/exists=1/2/3/4") { t.Fatalf("stdout=%q, output=%q", stdout.String(), contents) } } func TestRunSeedCommandDryRun(t *testing.T) { directory := t.TempDir() input := filepath.Join(directory, "missing.txt") if err := os.WriteFile(input, []byte("//depot/a/file#1\n"), 0644); err != nil { t.Fatal(err) } var stdout, stderr bytes.Buffer if err := run([]string{"seed", "--input", input, "--dry-run"}, &stdout, &stderr); err != nil { t.Fatal(err) } if _, err := os.Stat(input + ".seedreplica.json"); err != nil { t.Fatal(err) } } func TestDiscoverRevisionsRejectsInvalidFilespec(t *testing.T) { directory := t.TempDir() filespecs := filepath.Join(directory, "paths.txt") if err := os.WriteFile(filespecs, []byte("not-a-depot-path\n"), 0644); err != nil { t.Fatal(err) } _, err := discoverRevisions(context.Background(), p4Client{logger: logrus.New()}, filespecs, filepath.Join(directory, "revisions.txt"), 1, false) if err == nil || !strings.Contains(err.Error(), "invalid depot path") { t.Fatalf("discoverRevisions() error = %v", err) } } func TestDiscoverRevisionsRefusesExistingOutput(t *testing.T) { directory := t.TempDir() filespecs := filepath.Join(directory, "paths.txt") output := filepath.Join(directory, "revisions.txt") if err := os.WriteFile(filespecs, []byte("//depot/a/...\n"), 0644); err != nil { t.Fatal(err) } if err := os.WriteFile(output, []byte("old data\n"), 0644); err != nil { t.Fatal(err) } if _, err := discoverRevisions(context.Background(), p4Client{logger: logrus.New()}, filespecs, output, 1, false); err == nil { t.Fatal("discoverRevisions() succeeded without --replace-input") } } func TestGenerateDepotPathsSubdividesTimedOutDepots(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-ztag depots"*) cat <<'EOF' ... depot depot ... type local ... depot streams ... type remote ... depot remote ... type remote EOF ;; *"sizes -sh //depot/..."*) sleep 1 ;; *"dirs -D //depot/*"*) printf '%s\n' '//depot/alpha' '//depot/beta' '//helix-installer/* - no such file(s)./...' ;; *"sizes -sh //depot/alpha/..."*) : ;; *"sizes -sh //depot/beta/..."*) : ;; *) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } output := filepath.Join(directory, "depot-paths.txt") count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, 50*time.Millisecond, 1_000_000, 20) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } got := string(contents) for _, path := range []string{"//depot/alpha/...", "//depot/beta/..."} { if !strings.Contains(got, path+"\n") { t.Errorf("generated paths %q do not contain %q", got, path) } } if count != 2 || strings.Contains(got, "//depot/...\n") || strings.Contains(got, "//remote/") { t.Fatalf("unexpected generated paths: count=%d, paths=%q", count, got) } } func TestGenerateDepotPathsProcessesDepotsInParallel(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-ztag depots"*) cat <<'EOF' ... depot alpha ... type local ... depot beta ... type local EOF ;; *"sizes -sh //alpha/..."*|*"sizes -sh //beta/..."*) sleep 1 ;; *) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } output := filepath.Join(directory, "depot-paths.txt") started := time.Now() count, err := generateDepotPathsWithWorkers(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, 2*time.Second, 1_000_000, 20, 2) if err != nil { t.Fatal(err) } if elapsed := time.Since(started); elapsed >= 1500*time.Millisecond { t.Fatalf("generated %d paths in %s; depot paths were not processed in parallel", count, elapsed) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } if count != 2 || string(contents) != "//alpha/...\n//beta/...\n" { t.Fatalf("generated paths: count=%d, paths=%q", count, contents) } } func TestGenerateDepotPathsGroupsStreamsUsingDepotDepth(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-ztag depots"*) cat <<'EOF' ... depot streamsAdvertiserPortal ... type stream ... depth 3 EOF ;; *"dirs //streamsAdvertiserPortal/*/*/*"*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a' '//streamsAdvertiserPortal/team/product/b' '//streamsAdvertiserPortal/other/product/c' ;; *"sizes -sh //streamsAdvertiserPortal/team/product/a/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a/... - 600000 files 1 GB' ;; *"sizes -sh //streamsAdvertiserPortal/team/product/b/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/b/... - 600000 files 1 GB' ;; *"sizes -sh //streamsAdvertiserPortal/other/product/c/..."*) printf '%s\n' '//streamsAdvertiserPortal/other/product/c/... - 600000 files 1 GB' ;; *) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } output := filepath.Join(directory, "depot-paths.txt") count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 3) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } want := "//streamsAdvertiserPortal/other/...\n//streamsAdvertiserPortal/team/product/a/...\n//streamsAdvertiserPortal/team/product/b/...\n" if count != 3 || string(contents) != want { t.Fatalf("generated paths: count=%d, paths=%q, want %q", count, contents, want) } } func TestGenerateDepotPathsGroupsMultipleStreamsWithinFileLimit(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-ztag depots"*) cat <<'EOF' ... depot streamsAdvertiserPortal ... type stream ... depth 3 EOF ;; *"dirs //streamsAdvertiserPortal/*/*/*"*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a' '//streamsAdvertiserPortal/team/product/b' '//streamsAdvertiserPortal/other/product/c' ;; *"sizes -sh //streamsAdvertiserPortal/team/product/a/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/a/... - 400000 files 1 GB' ;; *"sizes -sh //streamsAdvertiserPortal/team/product/b/..."*) printf '%s\n' '//streamsAdvertiserPortal/team/product/b/... - 400000 files 1 GB' ;; *"sizes -sh //streamsAdvertiserPortal/other/product/c/..."*) printf '%s\n' '//streamsAdvertiserPortal/other/product/c/... - 400000 files 1 GB' ;; *) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } output := filepath.Join(directory, "depot-paths.txt") count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 3) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } want := "//streamsAdvertiserPortal/other/...\n//streamsAdvertiserPortal/team/...\n" if count != 2 || string(contents) != want { t.Fatalf("generated paths: count=%d, paths=%q, want %q", count, contents, want) } } func TestGenerateDepotPathsBatchesUnrelatedStreamsWithinFileLimit(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-ztag depots"*) cat <<'EOF' ... depot streams ... type stream ... depth 1 EOF ;; *"dirs //streams/*"*) printf '%s\n' '//streams/a' '//streams/b' '//streams/c' ;; *"sizes -sh //streams/a/..."*) printf '%s\n' '//streams/a/... - 400000 files 1 GB' ;; *"sizes -sh //streams/b/..."*) printf '%s\n' '//streams/b/... - 400000 files 1 GB' ;; *"sizes -sh //streams/c/..."*) printf '%s\n' '//streams/c/... - 400000 files 1 GB' ;; *) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } output := filepath.Join(directory, "depot-paths.txt") count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 3) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } want := "//streams/a/... //streams/b/...\n//streams/c/...\n" if count != 2 || string(contents) != want { t.Fatalf("generated paths: count=%d, paths=%q, want %q", count, contents, want) } } func TestDiscoverRevisionsVerifiesMultiplePathsFromOneLine(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh if [ "$#" -ne 7 ] || [ "$1" != "-Ztrack" ] || [ "$6" != "//depot/a/..." ] || [ "$7" != "//depot/b/..." ]; then printf 'unexpected p4 arguments: %s\n' "$*" >&2 exit 1 fi printf '%s\n' '//depot/a/file#1 - MISSING! change 1' printf '%s\n' '//depot/b/file#2 - MISSING! change 2' ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } filespecs := filepath.Join(directory, "paths.txt") if err := os.WriteFile(filespecs, []byte("//depot/a/... //depot/b/...\n"), 0644); err != nil { t.Fatal(err) } output := filepath.Join(directory, "revisions.txt") found, err := discoverRevisions(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, filespecs, output, 1, false) if err != nil { t.Fatal(err) } if found != 2 { t.Fatalf("found %d revisions, want 2", found) } } func TestGenerateDepotPathsIgnoresEmptyStreamDepot(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-ztag depots"*) cat <<'EOF' ... depot streams ... type stream ... streamdepth 2 EOF ;; *"dirs //streams/*/*"*) : ;; *) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } output := filepath.Join(directory, "depot-paths.txt") count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 20) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } if count != 0 || len(contents) != 0 { t.Fatalf("generated paths: count=%d, paths=%q", count, contents) } } func TestGenerateDepotPathsIgnoresStreamDepotWithoutDepth(t *testing.T) { directory := t.TempDir() p4bin := filepath.Join(directory, "p4") script := `#!/bin/sh case "$*" in *"-ztag depots"*) cat <<'EOF' ... depot streamsAdvertiserPortal ... type stream EOF ;; *) printf 'unexpected p4 command: %s\n' "$*" >&2; exit 1 ;; esac ` if err := os.WriteFile(p4bin, []byte(script), 0755); err != nil { t.Fatal(err) } output := filepath.Join(directory, "depot-paths.txt") count, err := generateDepotPaths(context.Background(), p4Client{bin: p4bin, logger: logrus.New()}, output, time.Second, 1_000_000, 20) if err != nil { t.Fatal(err) } contents, err := os.ReadFile(output) if err != nil { t.Fatal(err) } if count != 0 || len(contents) != 0 { t.Fatalf("generated paths: count=%d, paths=%q", count, contents) } } func TestReadBatchTracksOffsetAndSkipsBlankLines(t *testing.T) { input := filepath.Join(t.TempDir(), "revisions.txt") content := "//depot/a#1\n\n//depot/b#2\n//depot/c#3\n" if err := os.WriteFile(input, []byte(content), 0644); err != nil { t.Fatal(err) } batch, offset, linesRead, eof, err := readBatch(input, 0, 0, 2) if err != nil { t.Fatal(err) } if eof || linesRead != 3 || len(batch) != 2 || batch[1] != "//depot/b#2" { t.Fatalf("unexpected first batch: %#v, offset=%d, lines=%d, eof=%t", batch, offset, linesRead, eof) } batch, offset, linesRead, eof, err = readBatch(input, offset, linesRead, 2) if err != nil { t.Fatal(err) } if !eof || linesRead != 4 || len(batch) != 1 || batch[0] != "//depot/c#3" || offset != int64(len(content)) { t.Fatalf("unexpected second batch: %#v, offset=%d, lines=%d, eof=%t", batch, offset, linesRead, eof) } } func TestCheckpointRoundTrip(t *testing.T) { input := filepath.Join(t.TempDir(), "revisions.txt") checkpointPath := input + ".seedreplica.json" state := checkpoint{Input: input, Offset: 42, LinesRead: 3, Processed: 2, Batches: 1} if err := writeCheckpoint(checkpointPath, state); err != nil { t.Fatal(err) } loaded, err := loadCheckpoint(checkpointPath, input) if err != nil { t.Fatal(err) } if loaded.Offset != state.Offset || loaded.Processed != state.Processed || loaded.Batches != state.Batches { t.Fatalf("checkpoint = %#v, want %#v", loaded, state) } }