diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..e255588 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,67 @@ +name: Docker Publish + +on: + release: + types: [published] + workflow_dispatch: + inputs: + version: + description: 'Version tag for the Docker image' + required: true + type: string + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract version tags + id: tags + run: | + if [ "${{ github.event_name }}" = "release" ]; then + VERSION="${{ github.event.release.tag_name }}" + else + VERSION="${{ inputs.version }}" + fi + + # Remove 'v' prefix if present + VERSION_CLEAN="${VERSION#v}" + + # Initialize tags with the version itself + TAGS="espressif/git-mirror:${VERSION}" + + # For releases, also add latest tag + if [ "${{ github.event_name }}" = "release" ]; then + TAGS="$TAGS,espressif/git-mirror:latest" + fi + + # If version follows semver format (x.y.z) without suffixes, add major.minor and major tags + if [[ "${VERSION_CLEAN}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + MAJOR_MINOR=$(echo "${VERSION_CLEAN}" | cut -d. -f1,2) + MAJOR=$(echo "${VERSION_CLEAN}" | cut -d. -f1) + TAGS="$TAGS,espressif/git-mirror:v${MAJOR_MINOR},espressif/git-mirror:v${MAJOR}" + fi + + echo "tags=${TAGS}" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.tags.outputs.tags }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c582aea..201288e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,6 +117,12 @@ jobs: echo "Git mirror server logs:" docker logs git-mirror-integration + # Check for any errors in logs + if docker logs git-mirror-integration | grep -i "error"; then + echo "Errors found in logs" + exit 1 + fi + # Test git clone echo "Testing git clone..." GIT_TRACE_PACKET=1 GIT_TRACE=1 GIT_CURL_VERBOSE=1 git clone --depth 1 http://localhost:8080/github.com/espressif/git-mirror-server.git test-clone diff --git a/Dockerfile b/Dockerfile index 1f407fa..8c0befa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23 AS builder +FROM golang:1.25 AS builder WORKDIR /app COPY go.mod ./ diff --git a/docker-compose.yml b/compose.yaml similarity index 100% rename from docker-compose.yml rename to compose.yaml diff --git a/config.go b/config.go index 62aef0f..a0e61a1 100644 --- a/config.go +++ b/config.go @@ -2,8 +2,8 @@ package main import ( "fmt" - "io/ioutil" "net/url" + "os" "path/filepath" "strings" "time" @@ -16,10 +16,11 @@ type duration struct { } type config struct { - ListenAddr string - Interval duration - BasePath string - Repo []repo + ListenAddr string + Interval duration + BitmapInterval duration + BasePath string + Repo []repo } type repo struct { @@ -35,7 +36,7 @@ func (d *duration) UnmarshalText(text []byte) (err error) { func parseConfig(filename string) (cfg config, repos map[string]repo, err error) { // Parse the raw TOML file. - raw, err := ioutil.ReadFile(filename) + raw, err := os.ReadFile(filename) if err != nil { err = fmt.Errorf("unable to read config file %s, %s", filename, err) return @@ -52,6 +53,9 @@ func parseConfig(filename string) (cfg config, repos map[string]repo, err error) if cfg.Interval.Duration == 0 { cfg.Interval.Duration = time.Minute } + if cfg.BitmapInterval.Duration == 0 { + cfg.BitmapInterval.Duration = 10 * time.Hour + } if cfg.BasePath == "" { cfg.BasePath = "." } @@ -60,14 +64,14 @@ func parseConfig(filename string) (cfg config, repos map[string]repo, err error) } // Fetch repos, injecting default values where needed. - if cfg.Repo == nil || len(cfg.Repo) == 0 { + if len(cfg.Repo) == 0 { err = fmt.Errorf("no repos found in config %s, please define repos under [[repo]] sections", filename) return } repos = map[string]repo{} for i, r := range cfg.Repo { if r.Origin == "" { - err = fmt.Errorf("Origin required for repo %d in config %s", i+1, filename) + err = fmt.Errorf("origin required for repo %d in config %s", i+1, filename) return } @@ -83,10 +87,10 @@ func parseConfig(filename string) (cfg config, repos map[string]repo, err error) } } if r.Name == "" { - err = fmt.Errorf("Could not generate name for Origin %s in config %s, please manually specify a Name", r.Origin, filename) + err = fmt.Errorf("could not generate name for Origin %s in config %s, please manually specify a Name", r.Origin, filename) } if _, ok := repos[r.Name]; ok { - err = fmt.Errorf("Multiple repos with name %s in config %s", r.Name, filename) + err = fmt.Errorf("multiple repos with name %s in config %s", r.Name, filename) return } diff --git a/example-config.toml b/example-config.toml index 3116032..5a2b365 100644 --- a/example-config.toml +++ b/example-config.toml @@ -4,6 +4,9 @@ ListenAddr = ":8080" # Interval is the default interval for updating mirrors, can be overridden per # repo. Defaults to 15 seconds. Interval = "15m" +# BitmapInterval is the default interval for rebuilding git bitmaps. +# Defaults to 10 hours. It is a global setting only. +BitmapInterval = "10h" # Base path for storing mirrors, absolute or relative. Defaults to "." BasePath = "/opt/git-mirror/data" diff --git a/main.go b/main.go index f65b1ca..cf5d159 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,21 @@ func main() { }(r) } + // Run full repack with bitmap generation once in a while + go func() { + for { + time.Sleep(cfg.BitmapInterval.Duration) + for _, r := range repos { + log.Printf("updating bitmap for %s", r.Name) + if err := refreshBitmapIndex(cfg, r); err != nil { + log.Printf("error updating bitmap for %s: %s", r.Name, err) + } else { + log.Printf("bitmap updated for %s", r.Name) + } + } + } + }() + // Set up git http-backend CGI handler gitBackend := &cgi.Handler{ Path: "/usr/bin/git", diff --git a/mirror.go b/mirror.go index f9688a9..db660e3 100644 --- a/mirror.go +++ b/mirror.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "log" "os" "os/exec" "path" @@ -17,7 +18,12 @@ func mirror(cfg config, r repo) (string, error) { out, err := cmd.CombinedOutput() outStr = string(out) if err != nil { - fmt.Fprintf(os.Stderr, "failed to update remote in %s, %s", repoPath, err) + return "", fmt.Errorf("failed to update remote in %s: %w", repoPath, err) + } + if err := refreshMultiPackIndex(cfg, r); err != nil { + log.Printf("error refreshing multi-pack index for %s: %s", r.Name, err) + } else { + log.Printf("successfully refreshed multi-pack index for %s", r.Name) } } else if os.IsNotExist(err) { // Clone @@ -29,7 +35,12 @@ func mirror(cfg config, r repo) (string, error) { cmd.Dir = parent out, err := cmd.CombinedOutput() if err != nil { - fmt.Fprintf(os.Stderr, "failed to clone %s, %s", r.Origin, err) + return "", fmt.Errorf("failed to clone %s: %w", r.Origin, err) + } + if err := refreshBitmapIndex(cfg, r); err != nil { + log.Printf("error refreshing bitmap index for %s: %s", r.Name, err) + } else { + log.Printf("successfully refreshed bitmap index for %s", r.Name) } return string(out), err } else { @@ -37,3 +48,31 @@ func mirror(cfg config, r repo) (string, error) { } return outStr, nil } + +// Rebuild git bitmap index for the repo to speed up fetches once in a while +func refreshBitmapIndex(cfg config, r repo) error { + repoPath := path.Join(cfg.BasePath, r.Name) + + // Run git repack with bitmap index + repackCmd := exec.Command("git", "repack", "-Ad", "--write-bitmap-index", "--pack-kept-objects") + repackCmd.Dir = repoPath + if out, err := repackCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to repack %s: %s, output: %s", repoPath, err, string(out)) + } + + return nil +} + +// Quickly write multi-pack-index with bitmap without full repack +func refreshMultiPackIndex(cfg config, r repo) error { + repoPath := path.Join(cfg.BasePath, r.Name) + + // Run git multi-pack-index write with bitmap + mpiCmd := exec.Command("git", "multi-pack-index", "write", "--bitmap") + mpiCmd.Dir = repoPath + if out, err := mpiCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to write multi-pack-index %s: %s, output: %s", repoPath, err, string(out)) + } + + return nil +}