Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove git2go dependency #1001

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 2 additions & 4 deletions .github/workflows/01-golang-lint.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,10 @@ jobs:
steps:
- uses: actions/setup-go@v3
with:
go-version: 1.18
go-version: 1.19
- uses: actions/checkout@v3
with:
submodules: recursive
- name: Install libgit2
run: make libgit2
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
Expand All @@ -37,7 +35,7 @@ jobs:

# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
args: --timeout 10m --build-tags=static
args: --timeout 10m
#--new-from-rev dev

# Optional: show only new issues if it's a pull request. The default value is `false`.
Expand Down
6 changes: 1 addition & 5 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,11 @@ jobs:
with:
go-version: 1.19

- name: Install MSYS2 & libgit2 (Windows)
- name: Install MSYS2 (Windows)
shell: cmd
run: .\build.bat all
if: matrix.os == 'windows-latest'

- name: Install libgit2 (Linux/macOS)
run: make libgit2
if: matrix.os != 'windows-latest'

- name: Build
env:
RELEASE: v2.0.${{ github.run_number }}
Expand Down
10 changes: 3 additions & 7 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,20 +63,16 @@ jobs:
with:
go-version: 1.19

- name: Install MSYS2 & libgit2 (Windows)
- name: Install MSYS2 (Windows)
shell: cmd
run: .\build.bat all
if: matrix.os == 'windows-latest'

- name: Install libgit2 (Linux/macOS)
run: make libgit2
if: matrix.os != 'windows-latest'

- name: Test core pkg
run: go test -tags=static -v ./...
run: go test -v ./...

- name: Test httphandler pkg
run: cd httphandler && go test -tags=static -v ./...
run: cd httphandler && go test -v ./...

- name: Build
env:
Expand Down
3 changes: 0 additions & 3 deletions .gitmodules

This file was deleted.

3 changes: 0 additions & 3 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,3 @@ issues:
- linters:
- stylecheck
text: "ST1003"
run:
skip-dirs:
- git2go
16 changes: 4 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,20 +1,12 @@
.PHONY: test all build libgit2
.PHONY: test all build

# default task invoked while running make
all: libgit2 build
all: build

export CGO_ENABLED=1

# build and install libgit2
libgit2:
-git submodule update --init --recursive
cd git2go; make install-static

# go build tags
TAGS = "static"

build:
go build -v -tags=$(TAGS) .
go build -v .

test:
go test -v -tags=$(TAGS) ./...
go test -v ./...
7 changes: 0 additions & 7 deletions build.bat
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
@ECHO OFF

IF "%1"=="install" goto Install
IF "%1"=="build" goto Build
IF "%1"=="all" goto All
IF "%1"=="" goto Error ELSE goto Error

Expand Down Expand Up @@ -33,12 +32,6 @@ pacman -S --needed --noconfirm mingw-w64-x86_64-gcc
pacman -S --needed --noconfirm mingw-w64-x86_64-pkg-config
pacman -S --needed --noconfirm msys2-w32api-runtime

IF "%1"=="all" GOTO Build
GOTO End

:Build
SET "PATH=C:\MSYS2\mingw64\bin;C:\MSYS2\usr\bin;%PATH%"
make libgit2
GOTO End

:All
Expand Down
2 changes: 1 addition & 1 deletion build.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def main():
if client_name:
ldflags += " -X {}={}".format(client_var, client_name)

build_command = ["go", "build", "-buildmode=pie", "-tags=static", "-o", ks_file, "-ldflags" ,ldflags]
build_command = ["go", "build", "-buildmode=pie", "-o", ks_file, "-ldflags" ,ldflags]

print("Building kubescape and saving here: {}".format(ks_file))
print("Build command: {}".format(" ".join(build_command)))
Expand Down
3 changes: 0 additions & 3 deletions build/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@ RUN pip3 install --no-cache --upgrade pip setuptools
WORKDIR /work
ADD . .

# install libgit2
RUN rm -rf git2go && make libgit2

# build kubescape server
WORKDIR /work/httphandler
RUN python build.py
Expand Down
86 changes: 29 additions & 57 deletions core/cautils/localgitrepository.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,15 @@ import (
gitv5 "github.com/go-git/go-git/v5"
configv5 "github.com/go-git/go-git/v5/config"
plumbingv5 "github.com/go-git/go-git/v5/plumbing"
objectv5 "github.com/go-git/go-git/v5/plumbing/object"
"github.com/kubescape/go-git-url/apis"
git2go "github.com/libgit2/git2go/v33"
)

type LocalGitRepository struct {
goGitRepo *gitv5.Repository
git2GoRepo *git2go.Repository
head *plumbingv5.Reference
config *configv5.Config
fileToLastCommit map[string]*git2go.Commit
fileToLastCommit map[string]*objectv5.Commit
}

func NewLocalGitRepository(path string) (*LocalGitRepository, error) {
Expand Down Expand Up @@ -51,14 +50,6 @@ func NewLocalGitRepository(path string) (*LocalGitRepository, error) {
config: config,
}

if repoRoot, err := l.GetRootDir(); err == nil {
git2GoRepo, err := git2go.OpenRepository(repoRoot)
if err != nil {
return l, err
}
l.git2GoRepo = git2GoRepo
}

return l, nil
}

Expand Down Expand Up @@ -98,13 +89,11 @@ func (g *LocalGitRepository) GetName() (string, error) {

// GetLastCommit get latest commit object
func (g *LocalGitRepository) GetLastCommit() (*apis.Commit, error) {
cIter, err := g.goGitRepo.Log(&gitv5.LogOptions{})
ref, err := g.goGitRepo.Head()
if err != nil {
return nil, err
}

commit, err := cIter.Next()
defer cIter.Close()
commit, err := g.goGitRepo.CommitObject(ref.Hash())
if err != nil {
return nil, err
}
Expand All @@ -122,82 +111,65 @@ func (g *LocalGitRepository) GetLastCommit() (*apis.Commit, error) {
}, nil
}

func (g *LocalGitRepository) getAllCommits() ([]*git2go.Commit, error) {
logItr, itrErr := g.git2GoRepo.Walk()
if itrErr != nil {

return nil, itrErr
func (g *LocalGitRepository) getAllCommits() ([]*objectv5.Commit, error) {
ref, err := g.goGitRepo.Head()
if err != nil {
return nil, err
}

pushErr := logItr.PushHead()
if pushErr != nil {
return nil, pushErr
logItr, err := g.goGitRepo.Log(&gitv5.LogOptions{From: ref.Hash()})
if err != nil {
return nil, err
}

var allCommits []*git2go.Commit
err := logItr.Iterate(func(commit *git2go.Commit) bool {
var allCommits []*objectv5.Commit
err = logItr.ForEach(func(commit *objectv5.Commit) error {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@matthyx so if I understand correctly, this is the critical part that used to be slow (and might still be) and justified the use of the native git bindings? Is that correct?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check the other comment

if commit != nil {
allCommits = append(allCommits, commit)
return true
}
return false
return nil
})

if err != nil {
return nil, err
}

if err != nil {
return nil, err
}

return allCommits, nil
}

func (g *LocalGitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) {
if len(g.fileToLastCommit) == 0 {
filePathToCommitTime := map[string]time.Time{}
filePathToCommit := map[string]*git2go.Commit{}
filePathToCommit := map[string]*objectv5.Commit{}
allCommits, _ := g.getAllCommits()

// builds a map of all files to their last commit
for _, commit := range allCommits {
// Ignore merge commits (2+ parents)
if commit.ParentCount() <= 1 {
if commit.NumParents() <= 1 {
tree, err := commit.Tree()
if err != nil {
continue
}

// ParentCount can be either 1 or 0 (initial commit)
// In case it's the initial commit, prevTree is nil
var prevTree *git2go.Tree
if commit.ParentCount() == 1 {
prevCommit := commit.Parent(0)
var prevTree *objectv5.Tree
if commit.NumParents() == 1 {
prevCommit, _ := commit.Parent(0)
prevTree, err = prevCommit.Tree()
if err != nil {
continue
}
}

diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil)
changes, err := prevTree.Diff(tree)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is the slow part... merkle tree diffs are painfully slow
I have tried to walk around with some commit.Files() to skip the tree diffs, but it's still slow to open blobs in order to get files changed.

if err != nil {
continue
}

numDeltas, err := diff.NumDeltas()
if err != nil {
continue
}

for i := 0; i < numDeltas; i++ {
delta, err := diff.Delta(i)
if err != nil {
continue
}

deltaFilePath := delta.NewFile.Path
commitTime := commit.Author().When
for _, change := range changes {
deltaFilePath := change.To.Name
commitTime := commit.Author.When

// In case we have the commit information for the file which is not the latest - we override it
if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists {
Expand All @@ -222,15 +194,15 @@ func (g *LocalGitRepository) GetFileLastCommit(filePath string) (*apis.Commit, e
return nil, fmt.Errorf("failed to get commit information for file: %s", filePath)
}

func (g *LocalGitRepository) getCommit(commit *git2go.Commit) *apis.Commit {
func (g *LocalGitRepository) getCommit(commit *objectv5.Commit) *apis.Commit {
return &apis.Commit{
SHA: commit.Id().String(),
SHA: commit.Hash.String(),
Author: apis.Committer{
Name: commit.Author().Name,
Email: commit.Author().Email,
Date: commit.Author().When,
Name: commit.Author.Name,
Email: commit.Author.Email,
Date: commit.Author.When,
},
Message: commit.Message(),
Message: commit.Message,
Committer: apis.Committer{},
Files: []apis.Files{},
}
Expand Down
1 change: 0 additions & 1 deletion git2go
Submodule git2go deleted from eae007
3 changes: 0 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ require (
github.com/kubescape/k8s-interface v0.0.94-0.20221228202834-4b64f2440950
github.com/kubescape/opa-utils v0.0.222
github.com/kubescape/rbac-utils v0.0.19
github.com/libgit2/git2go/v33 v33.0.9
github.com/mattn/go-isatty v0.0.14
github.com/mikefarah/yq/v4 v4.29.1
github.com/olekukonko/tablewriter v0.0.5
Expand Down Expand Up @@ -188,5 +187,3 @@ require (
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
)

replace github.com/libgit2/git2go/v33 => ./git2go
12 changes: 2 additions & 10 deletions httphandler/Makefile
Original file line number Diff line number Diff line change
@@ -1,18 +1,10 @@
.PHONY: test all build libgit2
.PHONY: test all build

# default task invoked while running make
all: libgit2 build
all: build

export CGO_ENABLED=1

# build and install libgit2
libgit2:
git submodule update --init --recursive
cd git2go; make install-static

# go build tags
TAGS = "static"

build:
go build -v -tags=$(TAGS) .

Expand Down
2 changes: 1 addition & 1 deletion httphandler/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def main():
if client_name:
ldflags += " -X {}={}".format(client_var, client_name)

build_command = ["go", "build", "-buildmode=pie", "-tags=static", "-o", ks_file, "-ldflags" ,ldflags]
build_command = ["go", "build", "-buildmode=pie", "-o", ks_file, "-ldflags" ,ldflags]

print("Building kubescape and saving here: {}".format(ks_file))
print("Build command: {}".format(" ".join(build_command)))
Expand Down
3 changes: 0 additions & 3 deletions httphandler/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ require (
github.com/kubescape/k8s-interface v0.0.94-0.20221228202834-4b64f2440950 // indirect
github.com/kubescape/rbac-utils v0.0.19 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/libgit2/git2go/v33 v33.0.9 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
Expand Down Expand Up @@ -195,5 +194,3 @@ require (
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace github.com/libgit2/git2go/v33 => ../git2go