Update bitmap index to improve mirror performance

.github/workflows/docker-publish.yml

@@ -0,0 +1,67 @@
+name: Docker Publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version tag for the Docker image'
+        required: true
+        type: string
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract version tags
+        id: tags
+        run: |
+          if [ "${{ github.event_name }}" = "release" ]; then
+            VERSION="${{ github.event.release.tag_name }}"
+          else
+            VERSION="${{ inputs.version }}"
+          fi
+
+          # Remove 'v' prefix if present
+          VERSION_CLEAN="${VERSION#v}"
+
+          # Initialize tags with the version itself
+          TAGS="espressif/git-mirror:${VERSION}"
+
+          # For releases, also add latest tag
+          if [ "${{ github.event_name }}" = "release" ]; then
+            TAGS="$TAGS,espressif/git-mirror:latest"
+          fi
+
+          # If version follows semver format (x.y.z) without suffixes, add major.minor and major tags
+          if [[ "${VERSION_CLEAN}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            MAJOR_MINOR=$(echo "${VERSION_CLEAN}" | cut -d. -f1,2)
+            MAJOR=$(echo "${VERSION_CLEAN}" | cut -d. -f1)
+            TAGS="$TAGS,espressif/git-mirror:v${MAJOR_MINOR},espressif/git-mirror:v${MAJOR}"
+          fi
+
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.tags.outputs.tags }}

.github/workflows/test.yml

@@ -117,6 +117,12 @@ jobs:
         echo "Git mirror server logs:"
         docker logs git-mirror-integration
 
+        # Check for any errors in logs
+        if docker logs git-mirror-integration | grep -i "error"; then
+          echo "Errors found in logs"
+          exit 1
+        fi
+
         # Test git clone
         echo "Testing git clone..."
         GIT_TRACE_PACKET=1 GIT_TRACE=1 GIT_CURL_VERBOSE=1 git clone --depth 1 http://localhost:8080/github.com/espressif/git-mirror-server.git test-clone

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.23 AS builder
+FROM golang:1.25 AS builder
 WORKDIR /app
 
 COPY go.mod ./

config.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net/url"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -16,10 +16,11 @@ type duration struct {
 }
 
 type config struct {
-	ListenAddr string
-	Interval   duration
-	BasePath   string
-	Repo       []repo
+	ListenAddr     string
+	Interval       duration
+	BitmapInterval duration
+	BasePath       string
+	Repo           []repo
 }
 
 type repo struct {
@@ -35,7 +36,7 @@ func (d *duration) UnmarshalText(text []byte) (err error) {
 
 func parseConfig(filename string) (cfg config, repos map[string]repo, err error) {
 	// Parse the raw TOML file.
-	raw, err := ioutil.ReadFile(filename)
+	raw, err := os.ReadFile(filename)
 	if err != nil {
 		err = fmt.Errorf("unable to read config file %s, %s", filename, err)
 		return
@@ -52,6 +53,9 @@ func parseConfig(filename string) (cfg config, repos map[string]repo, err error)
 	if cfg.Interval.Duration == 0 {
 		cfg.Interval.Duration = time.Minute
 	}
+	if cfg.BitmapInterval.Duration == 0 {
+		cfg.BitmapInterval.Duration = 10 * time.Hour
+	}
 	if cfg.BasePath == "" {
 		cfg.BasePath = "."
 	}
@@ -60,14 +64,14 @@ func parseConfig(filename string) (cfg config, repos map[string]repo, err error)
 	}
 
 	// Fetch repos, injecting default values where needed.
-	if cfg.Repo == nil || len(cfg.Repo) == 0 {
+	if len(cfg.Repo) == 0 {
 		err = fmt.Errorf("no repos found in config %s, please define repos under [[repo]] sections", filename)
 		return
 	}
 	repos = map[string]repo{}
 	for i, r := range cfg.Repo {
 		if r.Origin == "" {
-			err = fmt.Errorf("Origin required for repo %d in config %s", i+1, filename)
+			err = fmt.Errorf("origin required for repo %d in config %s", i+1, filename)
 			return
 		}
 
@@ -83,10 +87,10 @@ func parseConfig(filename string) (cfg config, repos map[string]repo, err error)
 			}
 		}
 		if r.Name == "" {
-			err = fmt.Errorf("Could not generate name for Origin %s in config %s, please manually specify a Name", r.Origin, filename)
+			err = fmt.Errorf("could not generate name for Origin %s in config %s, please manually specify a Name", r.Origin, filename)
 		}
 		if _, ok := repos[r.Name]; ok {
-			err = fmt.Errorf("Multiple repos with name %s in config %s", r.Name, filename)
+			err = fmt.Errorf("multiple repos with name %s in config %s", r.Name, filename)
 			return
 		}
 

example-config.toml

@@ -4,6 +4,9 @@ ListenAddr = ":8080"
 # Interval is the default interval for updating mirrors, can be overridden per
 # repo.  Defaults to 15 seconds.
 Interval = "15m"
+# BitmapInterval is the default interval for rebuilding git bitmaps.
+# Defaults to 10 hours. It is a global setting only.
+BitmapInterval = "10h"
 # Base path for storing mirrors, absolute or relative.  Defaults to "."
 BasePath = "/opt/git-mirror/data"
 

main.go

@@ -44,6 +44,21 @@ func main() {
 		}(r)
 	}
 
+	// Run full repack with bitmap generation once in a while
+	go func() {
+		for {
+			time.Sleep(cfg.BitmapInterval.Duration)
+			for _, r := range repos {
+				log.Printf("updating bitmap for %s", r.Name)
+				if err := refreshBitmapIndex(cfg, r); err != nil {
+					log.Printf("error updating bitmap for %s: %s", r.Name, err)
+				} else {
+					log.Printf("bitmap updated for %s", r.Name)
+				}
+			}
+		}
+	}()
+
 	// Set up git http-backend CGI handler
 	gitBackend := &cgi.Handler{
 		Path: "/usr/bin/git",

mirror.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"os/exec"
 	"path"
@@ -17,7 +18,12 @@ func mirror(cfg config, r repo) (string, error) {
 		out, err := cmd.CombinedOutput()
 		outStr = string(out)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to update remote in %s, %s", repoPath, err)
+			return "", fmt.Errorf("failed to update remote in %s: %w", repoPath, err)
+		}
+		if err := refreshMultiPackIndex(cfg, r); err != nil {
+			log.Printf("error refreshing multi-pack index for %s: %s", r.Name, err)
+		} else {
+			log.Printf("successfully refreshed multi-pack index for %s", r.Name)
 		}
 	} else if os.IsNotExist(err) {
 		// Clone
@@ -29,11 +35,44 @@ func mirror(cfg config, r repo) (string, error) {
 		cmd.Dir = parent
 		out, err := cmd.CombinedOutput()
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "failed to clone %s, %s", r.Origin, err)
+			return "", fmt.Errorf("failed to clone %s: %w", r.Origin, err)
+		}
+		if err := refreshBitmapIndex(cfg, r); err != nil {
+			log.Printf("error refreshing bitmap index for %s: %s", r.Name, err)
+		} else {
+			log.Printf("successfully refreshed bitmap index for %s", r.Name)
 		}
 		return string(out), err
 	} else {
 		return "", fmt.Errorf("failed to stat %s, %s", repoPath, err)
 	}
 	return outStr, nil
 }
+
+// Rebuild git bitmap index for the repo to speed up fetches once in a while
+func refreshBitmapIndex(cfg config, r repo) error {
+	repoPath := path.Join(cfg.BasePath, r.Name)
+
+	// Run git repack with bitmap index
+	repackCmd := exec.Command("git", "repack", "-Ad", "--write-bitmap-index", "--pack-kept-objects")
+	repackCmd.Dir = repoPath
+	if out, err := repackCmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to repack %s: %s, output: %s", repoPath, err, string(out))
+	}
+
+	return nil
+}
+
+// Quickly write multi-pack-index with bitmap without full repack
+func refreshMultiPackIndex(cfg config, r repo) error {
+	repoPath := path.Join(cfg.BasePath, r.Name)
+
+	// Run git multi-pack-index write with bitmap
+	mpiCmd := exec.Command("git", "multi-pack-index", "write", "--bitmap")
+	mpiCmd.Dir = repoPath
+	if out, err := mpiCmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("failed to write multi-pack-index %s: %s, output: %s", repoPath, err, string(out))
+	}
+
+	return nil
+}