From 2969341929f28a626e0620dfb7c0136513c8d4bf Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Wed, 12 Mar 2025 09:29:47 +0000 Subject: [PATCH 1/7] add build gh action --- .github/workflows/build.yaml | 45 ++++++++++++++++++++++++++++++++++++ .gitignore | 3 +-- Dockerfile | 41 ++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build.yaml create mode 100644 Dockerfile diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..0efc747 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,45 @@ +name: build + +on: + push: + branches: + - "*" + tags: + - "v*" + pull_request: + branches: + - "main" + +env: + REGISTRY: quay.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + if: github.actor != 'dependabot[bot]' + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Login to Quay.io Container Registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: utilitywarehouse+drone_ci + password: ${{ secrets.SYSTEM_QUAY_TOKEN }} + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 1301b2b..78b707d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1 @@ -git-mirror -.bin \ No newline at end of file +git-mirror \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e8bb117 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +# Build the manager binary +FROM golang:1.24-alpine AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum + +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY . . + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +RUN go test -v -cover ./... && \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o git-mirror + +FROM alpine:3.21 + +ENV USER_ID=65532 + +RUN adduser -S -H -u $USER_ID app-user \ + && apk --no-cache add ca-certificates git openssh-client + +WORKDIR / + +COPY --from=builder /workspace/git-mirror . + +ENV USER=app-user + +USER $USER_ID + +ENTRYPOINT ["/git-mirror"] \ No newline at end of file From 87fbb3e0a37378f65143d8f7267bb9fcf002596a Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Wed, 12 Mar 2025 09:30:18 +0000 Subject: [PATCH 2/7] fix bug --- pkg/mirror/repository.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/mirror/repository.go b/pkg/mirror/repository.go index 0ed7905..7a16ea4 100644 --- a/pkg/mirror/repository.go +++ b/pkg/mirror/repository.go @@ -123,7 +123,7 @@ func NewRepository(repoConf RepositoryConfig, envs []string, log *slog.Logger) ( } for _, wtc := range repoConf.Worktrees { - if err := repo.AddWorktreeLink(wtc.Link, wtc.Ref, wtc.Ref); err != nil { + if err := repo.AddWorktreeLink(wtc.Link, wtc.Ref, wtc.Pathspec); err != nil { return nil, fmt.Errorf("unable to create worktree link err:%w", err) } } @@ -706,7 +706,6 @@ func (r *Repository) ensureWorktreeLink(ctx context.Context, wl *WorkTreeLink) e if currentHash == remoteHash { if wl.sanityCheckWorktree(ctx) { - wl.log.Debug("current hash is same as remote and checks passed", "hash", currentHash) return nil } wl.log.Error("worktree failed checks, re-creating...", "path", currentPath) From e52e0c7b61f481e334307b4d3756247a691dae35 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Wed, 12 Mar 2025 09:31:21 +0000 Subject: [PATCH 3/7] add poc cli wrapper --- config.yaml | 12 ++++ go.mod | 1 + go.sum | 2 + main.go | 156 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 config.yaml create mode 100644 main.go diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..81aa2e1 --- /dev/null +++ b/config.yaml @@ -0,0 +1,12 @@ +defaults: + interval: 30s + mirror_timeout: 2m + git_gc: always + auth: + ssh_key_path: /etc/git-secret/ssh + ssh_known_hosts_path: /etc/git-secret/known_hosts +repositories: + - remote: https://github.com/utilitywarehouse/system-alerts + worktrees: + - link: target/alerts + ref: main diff --git a/go.mod b/go.mod index 3c9ab01..b82a7e9 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/prometheus/client_golang v1.21.0 github.com/sasha-s/go-deadlock v0.3.5 + github.com/urfave/cli/v3 v3.0.0-beta1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 37ffba0..ec60408 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/urfave/cli/v3 v3.0.0-beta1 h1:6DTaaUarcM0wX7qj5Hcvs+5Dm3dyUTBbEwIWAjcw9Zg= +github.com/urfave/cli/v3 v3.0.0-beta1/go.mod h1:FnIeEMYu+ko8zP1F9Ypr3xkZMIDqW3DR92yUtY39q1Y= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= diff --git a/main.go b/main.go new file mode 100644 index 0000000..848d84a --- /dev/null +++ b/main.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "path" + "strings" + "syscall" + "time" + + "github.com/urfave/cli/v3" + "github.com/utilitywarehouse/git-mirror/pkg/mirror" + "gopkg.in/yaml.v3" +) + +var ( + loggerLevel = new(slog.LevelVar) + logger *slog.Logger + + levelStrings = map[string]slog.Level{ + "trace": slog.Level(-8), + "debug": slog.LevelDebug, + "info": slog.LevelInfo, + "warn": slog.LevelWarn, + "error": slog.LevelError, + } + + reposRootPath = path.Join(os.TempDir(), "git-mirror", "src") + + flags = []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Sources: cli.EnvVars("GIT_MIRROR_CONFIG"), + Value: "/etc/git-mirror/config.yaml", + Usage: "Absolute path to the config file.", + }, + &cli.StringFlag{ + Name: "log-level", + Sources: cli.EnvVars("LOG_LEVEL"), + Value: "info", + Usage: "Log level", + }, + &cli.StringFlag{ + Name: "git-ssh-key-file", + Sources: cli.EnvVars("GIT_SSH_KEY_FILE"), + Value: "/etc/git-secret/ssh", + Usage: "The path to git ssh key which will be used to setup GIT_SSH_COMMAND env.", + }, + &cli.StringFlag{ + Name: "git-ssh-known-hosts-file", + Sources: cli.EnvVars("GIT_SSH_KNOWN_HOSTS_FILE"), + Value: "/etc/git-secret/known_hosts", + Usage: "The local path to the known hosts file used to setup GIT_SSH_COMMAND env.", + }, + } +) + +func init() { + loggerLevel.Set(slog.LevelInfo) + logger = slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: loggerLevel, + })) +} + +func parseConfigFile(path string) (*mirror.RepoPoolConfig, error) { + yamlFile, err := os.ReadFile(path) + if err != nil { + return nil, err + } + conf := &mirror.RepoPoolConfig{} + err = yaml.Unmarshal(yamlFile, conf) + if err != nil { + return nil, err + } + return conf, nil +} + +func applyGitDefaults(c *cli.Command, mirrorConf *mirror.RepoPoolConfig) *mirror.RepoPoolConfig { + if mirrorConf.Defaults.Root == "" { + mirrorConf.Defaults.Root = reposRootPath + } + + if mirrorConf.Defaults.GitGC == "" { + mirrorConf.Defaults.GitGC = "always" + } + + if mirrorConf.Defaults.Interval == 0 { + mirrorConf.Defaults.Interval = 30 * time.Second + } + + if mirrorConf.Defaults.MirrorTimeout == 0 { + mirrorConf.Defaults.MirrorTimeout = 2 * time.Minute + } + + if mirrorConf.Defaults.Auth.SSHKeyPath == "" { + mirrorConf.Defaults.Auth.SSHKeyPath = c.String("git-ssh-key-file") + } + + if mirrorConf.Defaults.Auth.SSHKnownHostsPath == "" { + mirrorConf.Defaults.Auth.SSHKnownHostsPath = c.String("git-ssh-known-hosts-file") + } + + return mirrorConf +} + +func main() { + cmd := &cli.Command{ + Name: "git-mirror", + Usage: "git-mirror is a tool to periodically mirror remote repositories locally.", + Flags: flags, + Action: func(ctx context.Context, c *cli.Command) error { + + // set log level according to argument + if v, ok := levelStrings[strings.ToLower(c.String("log-level"))]; ok { + loggerLevel.Set(v) + } + + conf, err := parseConfigFile(c.String("config")) + if err != nil { + logger.Error("unable to parse tf applier config file", "err", err) + os.Exit(1) + } + + // setup git-mirror + conf = applyGitDefaults(c, conf) + + // path to resolve strongbox + gitENV := []string{fmt.Sprintf("PATH=%s", os.Getenv("PATH"))} + + repos, err := mirror.NewRepoPool(*conf, logger.With("logger", "git-mirror"), gitENV) + if err != nil { + logger.Error("could not create git mirror pool", "err", err) + os.Exit(1) + } + + // start mirror Loop + repos.StartLoop() + + //listenForShutdown + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + <-stop + logger.Info("Shutting down") + + return nil + }, + } + + if err := cmd.Run(context.Background(), os.Args); err != nil { + logger.Error("failed to run app", "err", err) + os.Exit(1) + } +} From c3ca483edb584789d08ec0c7fffa327518750c06 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Wed, 12 Mar 2025 09:46:17 +0000 Subject: [PATCH 4/7] add git for tests --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e8bb117..cc3168f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ FROM golang:1.24-alpine AS builder ARG TARGETOS ARG TARGETARCH +RUN apk --no-cache add git openssh-client + WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod @@ -20,7 +22,7 @@ COPY . . # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN go test -v -cover ./... && \ +RUN go test -v -cover ./pkg/mirror && \ CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o git-mirror FROM alpine:3.21 From 8f1b0ab31a58a33cff54ca4059f7be6bd0801d4d Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Wed, 12 Mar 2025 16:11:58 +0000 Subject: [PATCH 5/7] run all tests --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cc3168f..9426da8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ COPY . . # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN go test -v -cover ./pkg/mirror && \ +RUN go test -v -cover ./... && \ CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o git-mirror FROM alpine:3.21 From 385eb05c833039b78d3bde0d5504cdd973ed5cfe Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Thu, 13 Mar 2025 11:18:49 +0000 Subject: [PATCH 6/7] use config for all defaults --- main.go | 20 -------------------- pkg/mirror/repository.go | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/main.go b/main.go index 848d84a..f3eb790 100644 --- a/main.go +++ b/main.go @@ -43,18 +43,6 @@ var ( Value: "info", Usage: "Log level", }, - &cli.StringFlag{ - Name: "git-ssh-key-file", - Sources: cli.EnvVars("GIT_SSH_KEY_FILE"), - Value: "/etc/git-secret/ssh", - Usage: "The path to git ssh key which will be used to setup GIT_SSH_COMMAND env.", - }, - &cli.StringFlag{ - Name: "git-ssh-known-hosts-file", - Sources: cli.EnvVars("GIT_SSH_KNOWN_HOSTS_FILE"), - Value: "/etc/git-secret/known_hosts", - Usage: "The local path to the known hosts file used to setup GIT_SSH_COMMAND env.", - }, } ) @@ -95,14 +83,6 @@ func applyGitDefaults(c *cli.Command, mirrorConf *mirror.RepoPoolConfig) *mirror mirrorConf.Defaults.MirrorTimeout = 2 * time.Minute } - if mirrorConf.Defaults.Auth.SSHKeyPath == "" { - mirrorConf.Defaults.Auth.SSHKeyPath = c.String("git-ssh-key-file") - } - - if mirrorConf.Defaults.Auth.SSHKnownHostsPath == "" { - mirrorConf.Defaults.Auth.SSHKnownHostsPath = c.String("git-ssh-known-hosts-file") - } - return mirrorConf } diff --git a/pkg/mirror/repository.go b/pkg/mirror/repository.go index 7a16ea4..85621b3 100644 --- a/pkg/mirror/repository.go +++ b/pkg/mirror/repository.go @@ -460,7 +460,7 @@ func (r *Repository) Mirror(ctx context.Context) error { return fmt.Errorf("unable to cleanup repo:%s err:%w", r.gitURL.Repo, err) } - r.log.Info("mirror cycle complete", "time", time.Since(start), "fetch-time", fetchTime, "updated-refs", len(refs)) + r.log.Debug("mirror cycle complete", "time", time.Since(start), "fetch-time", fetchTime, "updated-refs", len(refs)) return nil } From 3f6fdd4639778238eae1e3d4b21c8b10409408e8 Mon Sep 17 00:00:00 2001 From: Ashok Siyani Date: Thu, 13 Mar 2025 11:20:09 +0000 Subject: [PATCH 7/7] fmted --- .gitignore | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 78b707d..248555b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -git-mirror \ No newline at end of file +git-mirror diff --git a/Dockerfile b/Dockerfile index 9426da8..fad17be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,4 +40,4 @@ ENV USER=app-user USER $USER_ID -ENTRYPOINT ["/git-mirror"] \ No newline at end of file +ENTRYPOINT ["/git-mirror"]