feat(internal/fetch,sidekick): add fetch package (#2964)

Move functions related to downloading tarballs and computing checksum to
internal/fetch, so that it can be used by code outside of sidekick.

For #2966

Author: julieqiu

internal/fetch/fetch.go

@@ -0,0 +1,109 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fetch provides utilities for fetching GitHub repository metadata and computing checksums.
+package fetch
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// Endpoints defines the endpoints used to access GitHub.
+type Endpoints struct {
+	// API defines the endpoint used to make API calls.
+	API string
+
+	// Download defines the endpoint to download tarballs.
+	Download string
+}
+
+// Repo represents a GitHub repository name.
+type Repo struct {
+	// Org defines the GitHub organization (or user), that owns the repository.
+	Org string
+
+	// Repo is the name of the repository, such as `googleapis` or `google-cloud-rust`.
+	Repo string
+}
+
+// RepoFromTarballLink extracts the gitHub account and repository (such as
+// `googleapis/googleapis`, or `googleapis/google-cloud-rust`) from the tarball
+// link.
+func RepoFromTarballLink(githubDownload, tarballLink string) (*Repo, error) {
+	urlPath := strings.TrimPrefix(tarballLink, githubDownload)
+	urlPath = strings.TrimPrefix(urlPath, "/")
+	components := strings.Split(urlPath, "/")
+	if len(components) < 2 {
+		return nil, fmt.Errorf("url path for tarball link is missing components")
+	}
+	repo := &Repo{
+		Org:  components[0],
+		Repo: components[1],
+	}
+	return repo, nil
+}
+
+// Sha256 downloads the content from the given URL and returns its SHA256
+// checksum as a hex string.
+func Sha256(query string) (string, error) {
+	response, err := http.Get(query)
+	if err != nil {
+		return "", err
+	}
+	if response.StatusCode >= 300 {
+		return "", fmt.Errorf("http error in download %s", response.Status)
+	}
+	defer response.Body.Close()
+
+	hasher := sha256.New()
+	if _, err := io.Copy(hasher, response.Body); err != nil {
+		return "", err
+	}
+	got := fmt.Sprintf("%x", hasher.Sum(nil))
+	return got, nil
+}
+
+// LatestSha fetches the latest commit SHA from the GitHub API for the given
+// repository URL.
+func LatestSha(query string) (string, error) {
+	client := &http.Client{}
+	request, err := http.NewRequest(http.MethodGet, query, nil)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Accept", "application/vnd.github.VERSION.sha")
+	response, err := client.Do(request)
+	if err != nil {
+		return "", err
+	}
+	if response.StatusCode >= 300 {
+		return "", fmt.Errorf("http error in download %s", response.Status)
+	}
+	defer response.Body.Close()
+	contents, err := io.ReadAll(response.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(contents), nil
+}
+
+// TarballLink constructs a GitHub tarball download URL for the given
+// repository and commit SHA.
+func TarballLink(githubDownload string, repo *Repo, sha string) string {
+	return fmt.Sprintf("%s/%s/%s/archive/%s.tar.gz", githubDownload, repo.Org, repo.Repo, sha)
+}

internal/fetch/fetch_test.go

@@ -0,0 +1,193 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fetch
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+const (
+	testGitHubDn       = "https://localhost:12345"
+	tarballPathTrailer = "/archive/5d5b1bf126485b0e2c972bac41b376438601e266.tar.gz"
+)
+
+func TestRepoFromTarballLink(t *testing.T) {
+	got, err := RepoFromTarballLink(testGitHubDn, testGitHubDn+"/org-name/repo-name"+tarballPathTrailer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := &Repo{
+		Org:  "org-name",
+		Repo: "repo-name",
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("mismatch (-want +got):\n%s", diff)
+	}
+}
+
+func TestRepoFromTarballLinkErrors(t *testing.T) {
+	for _, test := range []struct {
+		tarballLink string
+	}{
+		{tarballLink: "too-short"},
+	} {
+		if got, err := RepoFromTarballLink(testGitHubDn, test.tarballLink); err == nil {
+			t.Errorf("expected an error, got=%v", got)
+		}
+	}
+}
+
+func TestSha256(t *testing.T) {
+	const (
+		tarballPath           = "/googleapis/googleapis/archive/5d5b1bf126485b0e2c972bac41b376438601e266.tar.gz"
+		latestShaContents     = "The quick brown fox jumps over the lazy dog"
+		latestShaContentsHash = "d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592"
+	)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != tarballPath {
+			t.Fatalf("unexpected request path %q", r.URL.Path)
+		}
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(latestShaContents))
+	}))
+	defer server.Close()
+
+	got, err := Sha256(server.URL + tarballPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != latestShaContentsHash {
+		t.Errorf("Sha256() = %q, want %q", got, latestShaContentsHash)
+	}
+}
+
+func TestSha256Error(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		url  string
+	}{
+		{
+			name: "http status error",
+			url: func() string {
+				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					w.WriteHeader(http.StatusBadRequest)
+					w.Write([]byte("ERROR - bad request"))
+				}))
+				t.Cleanup(server.Close)
+				return server.URL + "/test"
+			}(),
+		},
+		{
+			name: "invalid url",
+			url:  "http://invalid-url-that-does-not-exist-12345.local",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			if _, err := Sha256(test.url); err == nil {
+				t.Error("expected an error from Sha256()")
+			}
+		})
+	}
+}
+
+func TestLatestSha(t *testing.T) {
+	const (
+		getLatestShaPath = "/repos/googleapis/googleapis/commits/master"
+		latestSha        = "5d5b1bf126485b0e2c972bac41b376438601e266"
+	)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != getLatestShaPath {
+			t.Fatalf("unexpected request path %q", r.URL.Path)
+		}
+		got := r.Header.Get("Accept")
+		want := "application/vnd.github.VERSION.sha"
+		if got != want {
+			t.Fatalf("mismatched Accept header for %q, got=%q, want=%s", r.URL.Path, got, want)
+		}
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(latestSha))
+	}))
+	defer server.Close()
+
+	got, err := LatestSha(server.URL + getLatestShaPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != latestSha {
+		t.Errorf("LatestSha() = %q, want %q", got, latestSha)
+	}
+}
+
+func TestLatestShaError(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		url  string
+	}{
+		{
+			name: "http status error",
+			url: func() string {
+				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					w.WriteHeader(http.StatusBadRequest)
+					w.Write([]byte("ERROR - bad request"))
+				}))
+				t.Cleanup(server.Close)
+				return server.URL + "/test"
+			}(),
+		},
+		{
+			name: "invalid url",
+			url:  "http://invalid-url-that-does-not-exist-12345.local",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			if _, err := LatestSha(test.url); err == nil {
+				t.Error("expected an error from LatestSha()")
+			}
+		})
+	}
+}
+
+func TestTarballLink(t *testing.T) {
+	for _, test := range []struct {
+		githubDownload string
+		repo           *Repo
+		sha            string
+		want           string
+	}{
+		{
+			githubDownload: "https://github.com",
+			repo:           &Repo{Org: "googleapis", Repo: "googleapis"},
+			sha:            "abc123",
+			want:           "https://github.com/googleapis/googleapis/archive/abc123.tar.gz",
+		},
+		{
+			githubDownload: "https://test.example.com",
+			repo:           &Repo{Org: "my-org", Repo: "my-repo"},
+			sha:            "def456",
+			want:           "https://test.example.com/my-org/my-repo/archive/def456.tar.gz",
+		},
+	} {
+		got := TarballLink(test.githubDownload, test.repo, test.sha)
+		if got != test.want {
+			t.Errorf("TarballLink() = %q, want %q", got, test.want)
+		}
+	}
+}