diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..0efc747 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,45 @@ +name: build + +on: + push: + branches: + - "*" + tags: + - "v*" + pull_request: + branches: + - "main" + +env: + REGISTRY: quay.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + if: github.actor != 'dependabot[bot]' + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Login to Quay.io Container Registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: utilitywarehouse+drone_ci + password: ${{ secrets.SYSTEM_QUAY_TOKEN }} + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 1301b2b..248555b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ git-mirror -.bin \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fad17be --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# Build the manager binary +FROM golang:1.24-alpine AS builder +ARG TARGETOS +ARG TARGETARCH + +RUN apk --no-cache add git openssh-client + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum + +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY . . + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +RUN go test -v -cover ./... && \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o git-mirror + +FROM alpine:3.21 + +ENV USER_ID=65532 + +RUN adduser -S -H -u $USER_ID app-user \ + && apk --no-cache add ca-certificates git openssh-client + +WORKDIR / + +COPY --from=builder /workspace/git-mirror . + +ENV USER=app-user + +USER $USER_ID + +ENTRYPOINT ["/git-mirror"] diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..81aa2e1 --- /dev/null +++ b/config.yaml @@ -0,0 +1,12 @@ +defaults: + interval: 30s + mirror_timeout: 2m + git_gc: always + auth: + ssh_key_path: /etc/git-secret/ssh + ssh_known_hosts_path: /etc/git-secret/known_hosts +repositories: + - remote: https://github.com/utilitywarehouse/system-alerts + worktrees: + - link: target/alerts + ref: main diff --git a/go.mod b/go.mod index 3c9ab01..b82a7e9 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/prometheus/client_golang v1.21.0 github.com/sasha-s/go-deadlock v0.3.5 + github.com/urfave/cli/v3 v3.0.0-beta1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 37ffba0..ec60408 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/urfave/cli/v3 v3.0.0-beta1 h1:6DTaaUarcM0wX7qj5Hcvs+5Dm3dyUTBbEwIWAjcw9Zg= +github.com/urfave/cli/v3 v3.0.0-beta1/go.mod h1:FnIeEMYu+ko8zP1F9Ypr3xkZMIDqW3DR92yUtY39q1Y= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= diff --git a/main.go b/main.go new file mode 100644 index 0000000..f3eb790 --- /dev/null +++ b/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "path" + "strings" + "syscall" + "time" + + "github.com/urfave/cli/v3" + "github.com/utilitywarehouse/git-mirror/pkg/mirror" + "gopkg.in/yaml.v3" +) + +var ( + loggerLevel = new(slog.LevelVar) + logger *slog.Logger + + levelStrings = map[string]slog.Level{ + "trace": slog.Level(-8), + "debug": slog.LevelDebug, + "info": slog.LevelInfo, + "warn": slog.LevelWarn, + "error": slog.LevelError, + } + + reposRootPath = path.Join(os.TempDir(), "git-mirror", "src") + + flags = []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Sources: cli.EnvVars("GIT_MIRROR_CONFIG"), + Value: "/etc/git-mirror/config.yaml", + Usage: "Absolute path to the config file.", + }, + &cli.StringFlag{ + Name: "log-level", + Sources: cli.EnvVars("LOG_LEVEL"), + Value: "info", + Usage: "Log level", + }, + } +) + +func init() { + loggerLevel.Set(slog.LevelInfo) + logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: loggerLevel, + })) +} + +func parseConfigFile(path string) (*mirror.RepoPoolConfig, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + return nil, err + } + conf := &mirror.RepoPoolConfig{} + err = yaml.Unmarshal(yamlFile, conf) + if err != nil { + return nil, err + } + return conf, nil +} + +func applyGitDefaults(c *cli.Command, mirrorConf *mirror.RepoPoolConfig) *mirror.RepoPoolConfig { + if mirrorConf.Defaults.Root == "" { + mirrorConf.Defaults.Root = reposRootPath + } + + if mirrorConf.Defaults.GitGC == "" { + mirrorConf.Defaults.GitGC = "always" + } + + if mirrorConf.Defaults.Interval == 0 { + mirrorConf.Defaults.Interval = 30 * time.Second + } + + if mirrorConf.Defaults.MirrorTimeout == 0 { + mirrorConf.Defaults.MirrorTimeout = 2 * time.Minute + } + + return mirrorConf +} + +func main() { + cmd := &cli.Command{ + Name: "git-mirror", + Usage: "git-mirror is a tool to periodically mirror remote repositories locally.", + Flags: flags, + Action: func(ctx context.Context, c *cli.Command) error { + + // set log level according to argument + if v, ok := levelStrings[strings.ToLower(c.String("log-level"))]; ok { + loggerLevel.Set(v) + } + + conf, err := parseConfigFile(c.String("config")) + if err != nil { + logger.Error("unable to parse tf applier config file", "err", err) + os.Exit(1) + } + + // setup git-mirror + conf = applyGitDefaults(c, conf) + + // path to resolve strongbox + gitENV := []string{fmt.Sprintf("PATH=%s", os.Getenv("PATH"))} + + repos, err := mirror.NewRepoPool(*conf, logger.With("logger", "git-mirror"), gitENV) + if err != nil { + logger.Error("could not create git mirror pool", "err", err) + os.Exit(1) + } + + // start mirror Loop + repos.StartLoop() + + //listenForShutdown + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop + logger.Info("Shutting down") + + return nil + }, + } + + if err := cmd.Run(context.Background(), os.Args); err != nil { + logger.Error("failed to run app", "err", err) + os.Exit(1) + } +} diff --git a/pkg/mirror/repository.go b/pkg/mirror/repository.go index 0ed7905..85621b3 100644 --- a/pkg/mirror/repository.go +++ b/pkg/mirror/repository.go @@ -123,7 +123,7 @@ func NewRepository(repoConf RepositoryConfig, envs []string, log *slog.Logger) ( } for _, wtc := range repoConf.Worktrees { - if err := repo.AddWorktreeLink(wtc.Link, wtc.Ref, wtc.Ref); err != nil { + if err := repo.AddWorktreeLink(wtc.Link, wtc.Ref, wtc.Pathspec); err != nil { return nil, fmt.Errorf("unable to create worktree link err:%w", err) } } @@ -460,7 +460,7 @@ func (r *Repository) Mirror(ctx context.Context) error { return fmt.Errorf("unable to cleanup repo:%s err:%w", r.gitURL.Repo, err) } - r.log.Info("mirror cycle complete", "time", time.Since(start), "fetch-time", fetchTime, "updated-refs", len(refs)) + r.log.Debug("mirror cycle complete", "time", time.Since(start), "fetch-time", fetchTime, "updated-refs", len(refs)) return nil } @@ -706,7 +706,6 @@ func (r *Repository) ensureWorktreeLink(ctx context.Context, wl *WorkTreeLink) e if currentHash == remoteHash { if wl.sanityCheckWorktree(ctx) { - wl.log.Debug("current hash is same as remote and checks passed", "hash", currentHash) return nil } wl.log.Error("worktree failed checks, re-creating...", "path", currentPath)