Skip to content

Commit

Permalink
Merge branch 'main' into devchat/issue
Browse files Browse the repository at this point in the history
  • Loading branch information
d4x1 committed Mar 5, 2024
2 parents cc697ca + 0dd9dc7 commit f934abd
Show file tree
Hide file tree
Showing 30 changed files with 1,817 additions and 703 deletions.
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -159,4 +159,7 @@ backend/server/api/docs/docs.go
# python
*.pyc
__pycache__
venv
venv

/backend/plugins/gitextractor/parser/output*
/backend/plugins/gitextractor/parser/demo_repo/
7 changes: 7 additions & 0 deletions backend/core/models/domainlayer/domainlayer.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,10 @@ func NewDomainEntity(id string) DomainEntity {
NoPKModel: common.NewNoPKModel(),
}
}

func NewDomainEntityExtended(id string) DomainEntityExtended {
return DomainEntityExtended{
Id: id,
NoPKModel: common.NewNoPKModel(),
}
}
14 changes: 6 additions & 8 deletions backend/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/gin-contrib/cors v1.3.1
github.com/gin-gonic/gin v1.9.1
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-git/go-git/v5 v5.9.0
github.com/go-git/go-git/v5 v5.11.0
github.com/go-playground/validator/v10 v10.14.1
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e
github.com/google/uuid v1.3.0
Expand All @@ -24,13 +24,13 @@ require (
github.com/spf13/cast v1.5.1
github.com/spf13/cobra v1.5.0
github.com/spf13/viper v1.8.1
github.com/stretchr/testify v1.8.3
github.com/stretchr/testify v1.8.4
github.com/swaggo/files v1.0.1
github.com/swaggo/gin-swagger v1.6.0
github.com/swaggo/swag v1.16.1
github.com/tidwall/gjson v1.14.3
github.com/viant/afs v1.16.0
golang.org/x/crypto v0.14.0
golang.org/x/crypto v0.16.0
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f
golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
golang.org/x/sync v0.3.0
Expand Down Expand Up @@ -106,9 +106,9 @@ require (
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/net v0.16.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.13.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.30.0 // indirect
Expand All @@ -123,5 +123,3 @@ require (
github.com/golang-jwt/jwt/v5 v5.0.0-rc.1
golang.org/x/mod v0.13.0
)

//replace github.com/apache/incubator-devlake => ./
15 changes: 15 additions & 0 deletions backend/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,11 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY=
github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0=
github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4=
github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
Expand Down Expand Up @@ -272,6 +275,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
Expand Down Expand Up @@ -621,6 +625,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
Expand Down Expand Up @@ -714,6 +720,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand Down Expand Up @@ -806,6 +814,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -906,13 +916,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -927,6 +940,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
2 changes: 1 addition & 1 deletion backend/plugins/gitextractor/impl/impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st
func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
if taskData, ok := taskCtx.GetData().(*tasks.GitExtractorTaskData); ok {
if taskData.GitRepo != nil {
if err := taskData.GitRepo.Close(); err != nil {
if err := taskData.GitRepo.Close(taskCtx.GetContext()); err != nil {
return errors.Convert(err)
}
}
Expand Down
2 changes: 1 addition & 1 deletion backend/plugins/gitextractor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func main() {
if err != nil {
panic(err)
}
defer repo.Close()
defer repo.Close(ctx)
if err = repo.CollectAll(subTaskCtx); err != nil {
panic(err)
}
Expand Down
59 changes: 33 additions & 26 deletions backend/plugins/gitextractor/parser/clone.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
"github.com/go-git/go-git/v5/plumbing/transport"

"net"
"net/http"
neturl "net/url"
Expand Down Expand Up @@ -75,8 +74,8 @@ func cloneOverSSH(ctx plugin.SubTaskContext, url, dir, passphrase string, pk []b
return nil
}

func (l *GitRepoCreator) CloneOverHTTP(ctx plugin.SubTaskContext, repoId, url, user, password, proxy string) (*GitRepo, errors.Error) {
return withTempDirectory(func(dir string) (*GitRepo, error) {
func (l *GitRepoCreator) cloneOverHTTP(ctx plugin.SubTaskContext, withGoGit bool, repoId, url, user, password, proxy string) (RepoCollector, errors.Error) {
return withTempDirectory(func(dir string) (RepoCollector, error) {
var data []byte
buf := bytes.NewBuffer(data)
done := make(chan struct{}, 1)
Expand Down Expand Up @@ -127,12 +126,15 @@ func (l *GitRepoCreator) CloneOverHTTP(ctx plugin.SubTaskContext, repoId, url, u
l.logger.Error(err, "PlainCloneContext")
return nil, err
}
if withGoGit {
return l.LocalGoGitRepo(dir, repoId)
}
return l.LocalRepo(dir, repoId)
})
}

func (l *GitRepoCreator) CloneOverSSH(ctx plugin.SubTaskContext, repoId, url, privateKey, passphrase string) (*GitRepo, errors.Error) {
return withTempDirectory(func(dir string) (*GitRepo, error) {
func (l *GitRepoCreator) cloneOverSSH(ctx plugin.SubTaskContext, withGoGit bool, repoId, url, privateKey, passphrase string) (RepoCollector, errors.Error) {
return withTempDirectory(func(dir string) (RepoCollector, error) {
pk, err := base64.StdEncoding.DecodeString(privateKey)
if err != nil {
return nil, err
Expand All @@ -141,31 +143,13 @@ func (l *GitRepoCreator) CloneOverSSH(ctx plugin.SubTaskContext, repoId, url, pr
if err != nil {
return nil, err
}
if withGoGit {
return l.LocalGoGitRepo(dir, repoId)
}
return l.LocalRepo(dir, repoId)
})
}

func withTempDirectory(f func(tempDir string) (*GitRepo, error)) (*GitRepo, errors.Error) {
dir, err := os.MkdirTemp("", "gitextractor")
if err != nil {
return nil, errors.Convert(err)
}
cleanup := func() {
_ = os.RemoveAll(dir)
}
defer func() {
if err != nil {
cleanup()
}
}()
repo, err := f(dir)
if err != nil {
return nil, errors.Convert(err)
}
repo.cleanup = cleanup
return repo, errors.Convert(err)
}

func setCloneProgress(subTaskCtx plugin.SubTaskContext, cloneProgressInfo string) {
if cloneProgressInfo == "" {
return
Expand Down Expand Up @@ -208,3 +192,26 @@ func refreshCloneProgress(subTaskCtx plugin.SubTaskContext, done chan struct{},
func isAzureRepo(ctx context.Context, repoUrl string) bool {
return strings.Contains(repoUrl, "dev.azure.com")
}

func withTempDirectory(f func(tempDir string) (RepoCollector, error)) (RepoCollector, errors.Error) {
dir, err := os.MkdirTemp("", "gitextractor")
if err != nil {
return nil, errors.Convert(err)
}
cleanup := func() {
_ = os.RemoveAll(dir)
}
defer func() {
if err != nil {
cleanup()
}
}()
repo, err := f(dir)
if err != nil {
return nil, errors.Convert(err)
}
if err := repo.SetCleanUp(cleanup); err != nil {
return nil, errors.Convert(err)
}
return repo, nil
}
31 changes: 31 additions & 0 deletions backend/plugins/gitextractor/parser/clone_gogit.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package parser

import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
)

func (l *GitRepoCreator) CloneGoGitRepoOverHTTP(ctx plugin.SubTaskContext, repoId, url, user, password, proxy string) (RepoCollector, errors.Error) {
return l.cloneOverHTTP(ctx, true, repoId, url, user, password, proxy)
}

func (l *GitRepoCreator) CloneGoGitRepoOverSSH(ctx plugin.SubTaskContext, repoId, url, privateKey, passphrase string) (RepoCollector, errors.Error) {
return l.cloneOverSSH(ctx, true, repoId, url, privateKey, passphrase)
}
31 changes: 31 additions & 0 deletions backend/plugins/gitextractor/parser/clone_libgit2.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package parser

import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
)

func (l *GitRepoCreator) CloneOverHTTP(ctx plugin.SubTaskContext, repoId, url, user, password, proxy string) (RepoCollector, errors.Error) {
return l.cloneOverHTTP(ctx, false, repoId, url, user, password, proxy)
}

func (l *GitRepoCreator) CloneOverSSH(ctx plugin.SubTaskContext, repoId, url, privateKey, passphrase string) (RepoCollector, errors.Error) {
return l.cloneOverSSH(ctx, false, repoId, url, privateKey, passphrase)
}
Loading

0 comments on commit f934abd

Please sign in to comment.