diff --git a/README.md b/README.md index 2fd4dd9..4a9ca25 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # clone -Go library for programs that keep local checkouts of HTTPS Git repositories. It shells out to the `git` binary, which must be on `PATH`, and has no third-party Go dependencies. The package supports Go 1.25 or later. For in-process object parsing or history walking, use a library such as [go-git](https://github.com/go-git/go-git). +Go library for programs that keep local checkouts of HTTPS Git repositories. It shells out to the `git` binary, which must be on `PATH`. The package supports Go 1.25 or later. For in-process object parsing or history walking, use a library such as [go-git](https://github.com/go-git/go-git). ## Install @@ -57,7 +57,9 @@ if err := cache.EnsureCommit(ctx, url, commit); err != nil { ## Read a file from a commit -`Blob` runs `git show :` and reads at most `maxBytes+1`, draining the rest of stdout so Git can exit. The extra byte distinguishes content exactly at the limit from truncated content, and a NUL byte within the returned range marks the blob as binary. Check untrusted input with `ValidCommit` and `SanitizePath` before calling it: +`InspectBlob` runs `git show :` and reads at most `maxBytes+1`. The extra byte distinguishes content exactly at the limit from truncated content. Complete reads use `magic.Detect`; truncated reads use `magic.DetectPrefix` so the result can report that later bytes may change the classification. The returned content is retained for text, binary, and unknown results. + +Both blob functions validate commits and paths before invoking Git. `ValidCommit` and `SanitizePath` are also available when callers need to validate input earlier: ```go path, ok := clone.SanitizePath("cmd/tool/main.go") @@ -65,7 +67,7 @@ if !ok || !clone.ValidCommit(commit) { log.Fatal("invalid commit or path") } -content, binary, truncated, err := clone.Blob( +result, err := clone.InspectBlob( ctx, filepath.Join(cache.Dir(url), "src"), commit, @@ -75,12 +77,17 @@ content, binary, truncated, err := clone.Blob( if err != nil { log.Fatal(err) } -if !binary { - fmt.Printf("%s", content) +if result.Detection.Kind == magic.KindText && + result.Detection.Encoding == "utf-8" { + fmt.Printf("%s", result.Content) } -fmt.Println("truncated:", truncated) +fmt.Println("truncated:", result.Truncated) ``` +`Blob` remains available for callers that only need its original NUL-based +binary flag. It returns nil content when a NUL occurs within the returned range; +a NUL beyond `maxBytes` is not observed. + ## Remote queries `RemoteBranches` returns sorted branch names from `git ls-remote --heads`. It disables terminal prompts and the ambient credential helper so a supplied URL cannot trigger credential lookup. `RemoteHead` returns the SHA advertised for `HEAD` and keeps ambient non-interactive credentials available. diff --git a/blob.go b/blob.go index c97e5ca..5d39538 100644 --- a/blob.go +++ b/blob.go @@ -9,24 +9,67 @@ import ( "math" "os/exec" "strings" + + "github.com/git-pkgs/magic" ) +// BlobResult contains a bounded blob read and its content classification. +type BlobResult struct { + Content []byte + Detection magic.Result + Truncated bool +} + +// InspectBlob reads path from commit in dir and classifies the returned bytes. +// It uses prefix detection when maxBytes truncates the blob. commit and path +// are validated with ValidCommit and SanitizePath before reaching Git. +func InspectBlob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) (BlobResult, error) { + content, truncated, err := readBlob(ctx, dir, commit, blobPath, maxBytes) + if err != nil { + return BlobResult{}, err + } + + var detection magic.Result + if truncated { + detection = magic.DetectPrefix(content) + } else { + detection = magic.Detect(content) + } + + return BlobResult{ + Content: content, + Detection: detection, + Truncated: truncated, + }, nil +} + // Blob reads path from commit in dir. It caps content at maxBytes and reports // whether the blob is binary or was truncated. commit and path are validated // with ValidCommit and SanitizePath before reaching Git. func Blob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) (content []byte, binary, truncated bool, err error) { + content, truncated, err = readBlob(ctx, dir, commit, blobPath, maxBytes) + if err != nil { + return nil, false, false, err + } + if bytes.IndexByte(content, 0) != -1 { + return nil, true, truncated, nil + } + return content, false, truncated, nil +} + +func readBlob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) (content []byte, truncated bool, err error) { if maxBytes < 0 { - return nil, false, false, fmt.Errorf("maxBytes must be non-negative") + return nil, false, fmt.Errorf("maxBytes must be non-negative") } if maxBytes == math.MaxInt64 { - return nil, false, false, fmt.Errorf("maxBytes is too large") + return nil, false, fmt.Errorf("maxBytes is too large") } if !ValidCommit(commit) { - return nil, false, false, fmt.Errorf("invalid commit %q", commit) + return nil, false, fmt.Errorf("invalid commit %q", commit) } clean, ok := SanitizePath(blobPath) if !ok { - return nil, false, false, fmt.Errorf("invalid path %q", blobPath) + return nil, false, fmt.Errorf("invalid path %q", blobPath) } // --end-of-options stops a commit or path that somehow slipped past the @@ -35,12 +78,12 @@ func Blob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) (co cmd := exec.CommandContext(ctx, "git", "-C", dir, "show", "--end-of-options", commit+":"+clean) stdout, err := cmd.StdoutPipe() if err != nil { - return nil, false, false, err + return nil, false, err } var errBuf bytes.Buffer cmd.Stderr = &errBuf if err := cmd.Start(); err != nil { - return nil, false, false, err + return nil, false, err } raw, readErr := io.ReadAll(io.LimitReader(stdout, maxBytes+1)) @@ -60,13 +103,10 @@ func Blob(ctx context.Context, dir, commit, blobPath string, maxBytes int64) (co if message == "" { message = waitErr.Error() } - return nil, false, false, errors.New(message) + return nil, false, errors.New(message) } if readErr != nil { - return nil, false, false, readErr - } - if bytes.IndexByte(raw, 0) != -1 { - return nil, true, truncated, nil + return nil, false, readErr } - return raw, false, truncated, nil + return raw, truncated, nil } diff --git a/blob_test.go b/blob_test.go index 7d5382a..6d4006c 100644 --- a/blob_test.go +++ b/blob_test.go @@ -7,9 +7,11 @@ import ( "path/filepath" "strings" "testing" + + "github.com/git-pkgs/magic" ) -func seedBlobRepository(t *testing.T) (string, string) { +func seedBlobRepository(t testing.TB) (string, string) { t.Helper() requireGit(t) @@ -20,6 +22,10 @@ func seedBlobRepository(t *testing.T) (string, string) { "big.txt": bytes.Repeat([]byte("a"), 128<<10), "binary": {'a', 0, 'b'}, "late-nul": {'a', 'b', 'c', 0}, + "empty": {}, + "png": []byte("\x89PNG\r\n\x1a\n"), + "invalid": {0xff, 'a'}, + "utf16le": {0xff, 0xfe, 'h', 0, 'i', 0}, } for name, content := range files { if err := os.WriteFile(filepath.Join(dir, name), content, 0o644); err != nil { @@ -31,6 +37,183 @@ func seedBlobRepository(t *testing.T) (string, string) { return dir, runGitTest(t, dir, "rev-parse", "HEAD") } +func BenchmarkBlob(b *testing.B) { + dir, commit := seedBlobRepository(b) + b.ReportAllocs() + + for b.Loop() { + content, binary, truncated, err := Blob(context.Background(), dir, commit, "exact.txt", 5) + if err != nil { + b.Fatal(err) + } + if len(content) != 5 || binary || truncated { + b.Fatal("unexpected Blob result") + } + } +} + +func BenchmarkInspectBlob(b *testing.B) { + dir, commit := seedBlobRepository(b) + b.ReportAllocs() + + for b.Loop() { + result, err := InspectBlob(context.Background(), dir, commit, "exact.txt", 5) + if err != nil { + b.Fatal(err) + } + if len(result.Content) != 5 || result.Detection.Kind != magic.KindText || result.Truncated { + b.Fatal("unexpected InspectBlob result") + } + } +} + +func TestInspectBlobClassifiesContent(t *testing.T) { + dir, commit := seedBlobRepository(t) + tests := []struct { + name string + path string + maxBytes int64 + wantContent []byte + wantDetection magic.Result + wantTruncated bool + }{ + { + name: "complete text", + path: "exact.txt", + maxBytes: 5, + wantContent: []byte("12345"), + wantDetection: magic.Result{ + Kind: magic.KindText, + MIME: "text/plain", + Format: "text", + Encoding: "utf-8", + }, + }, + { + name: "truncated text", + path: "big.txt", + maxBytes: 32, + wantContent: bytes.Repeat([]byte("a"), 32), + wantTruncated: true, + wantDetection: magic.Result{ + Kind: magic.KindText, + MIME: "text/plain", + Format: "text", + Encoding: "utf-8", + Reason: magic.ReasonNeedMore, + }, + }, + { + name: "binary signature without NUL", + path: "png", + maxBytes: 8, + wantContent: []byte("\x89PNG\r\n\x1a\n"), + wantDetection: magic.Result{ + Kind: magic.KindBinary, + MIME: "image/png", + Format: "png", + }, + }, + { + name: "invalid UTF-8", + path: "invalid", + maxBytes: 2, + wantContent: []byte{0xff, 'a'}, + wantDetection: magic.Result{ + Kind: magic.KindUnknown, + Reason: magic.ReasonInvalidText, + }, + }, + { + name: "UTF-16LE", + path: "utf16le", + maxBytes: 6, + wantContent: []byte{0xff, 0xfe, 'h', 0, 'i', 0}, + wantDetection: magic.Result{ + Kind: magic.KindText, + MIME: "text/plain", + Format: "text", + Encoding: "utf-16le", + }, + }, + { + name: "early NUL", + path: "binary", + maxBytes: 3, + wantContent: []byte{'a', 0, 'b'}, + wantDetection: magic.Result{Kind: magic.KindBinary}, + }, + { + name: "NUL beyond limit", + path: "late-nul", + maxBytes: 3, + wantContent: []byte("abc"), + wantTruncated: true, + wantDetection: magic.Result{ + Kind: magic.KindText, + MIME: "text/plain", + Format: "text", + Encoding: "utf-8", + Reason: magic.ReasonNeedMore, + }, + }, + { + name: "empty", + path: "empty", + maxBytes: 0, + wantContent: []byte{}, + wantDetection: magic.Result{ + Kind: magic.KindText, + MIME: "text/plain", + Format: "text", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := InspectBlob(context.Background(), dir, commit, tt.path, tt.maxBytes) + if err != nil { + t.Fatalf("InspectBlob: %v", err) + } + if !bytes.Equal(result.Content, tt.wantContent) { + t.Errorf("Content = %q, want %q", result.Content, tt.wantContent) + } + if result.Detection != tt.wantDetection { + t.Errorf("Detection = %#v, want %#v", result.Detection, tt.wantDetection) + } + if result.Truncated != tt.wantTruncated { + t.Errorf("Truncated = %v, want %v", result.Truncated, tt.wantTruncated) + } + }) + } +} + +func TestInspectBlobReportsErrors(t *testing.T) { + dir, commit := seedBlobRepository(t) + tests := []struct { + name string + commit string + path string + maxBytes int64 + contains string + }{ + {"missing path", commit, "missing.txt", 100, "does not exist"}, + {"invalid limit", commit, "exact.txt", -1, "non-negative"}, + {"invalid commit", "HEAD", "exact.txt", 100, "invalid commit"}, + {"invalid path", commit, "../exact.txt", 100, "invalid path"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := InspectBlob(context.Background(), dir, tt.commit, tt.path, tt.maxBytes) + if err == nil || !strings.Contains(err.Error(), tt.contains) { + t.Errorf("error = %v, want error containing %q", err, tt.contains) + } + }) + } +} + func TestBlobReadsTextAtLimit(t *testing.T) { dir, commit := seedBlobRepository(t) content, binary, truncated, err := Blob(context.Background(), dir, commit, "exact.txt", 5) diff --git a/doc.go b/doc.go index 747dad9..df247c3 100644 --- a/doc.go +++ b/doc.go @@ -1,8 +1,7 @@ // Package clone keeps local checkouts of HTTPS Git repositories. It shells -// out to the git binary, which must be on PATH, and has no third-party Go -// dependencies. It provides shallow clone-or-fetch, bounded retries for -// network failures, a persistent cache, and capped reads of files from -// commits. +// out to the git binary, which must be on PATH. It provides shallow +// clone-or-fetch, bounded retries for network failures, a persistent cache, +// and capped reads and content classification for files from commits. // // Applications that need to parse Git objects or walk history in process can // use a library such as github.com/go-git/go-git. diff --git a/go.mod b/go.mod index 52a07bc..530ab32 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/git-pkgs/clone go 1.25.6 + +require github.com/git-pkgs/magic v0.1.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1c7e6dc --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/git-pkgs/magic v0.1.0 h1:xLrqq7CMXB9g5bJnmJyKw17Rvlh0GFiEmO6e5RFsoeY= +github.com/git-pkgs/magic v0.1.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= diff --git a/test_helpers_test.go b/test_helpers_test.go index 3525bb3..af10aa7 100644 --- a/test_helpers_test.go +++ b/test_helpers_test.go @@ -7,7 +7,7 @@ import ( "testing" ) -func requireGit(t *testing.T) { +func requireGit(t testing.TB) { t.Helper() if _, err := exec.LookPath("git"); err != nil { t.Skip("git is not installed") @@ -40,7 +40,7 @@ func subcommand(args []string) string { return "" } -func runGitTest(t *testing.T, dir string, args ...string) string { +func runGitTest(t testing.TB, dir string, args ...string) string { t.Helper() cmd := exec.Command("git", args...) cmd.Dir = dir