Skip to content

refactor: code scanner signature [IDE-195] #23

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 8, 2024
Merged

refactor: code scanner signature [IDE-195] #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ac19d0e
refactor: change signature of code scanner
teodora-sandu Apr 5, 2024
2e5a867
refactor: move the bundle package
teodora-sandu Apr 5, 2024
d782ad9
refactor: move the deepcode package
teodora-sandu Apr 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 7 additions & 22 deletions bundle/bundle.go → internal/bundle/bundle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,17 @@ package bundle

import (
"context"

"github.com/rs/zerolog"

"github.com/snyk/code-client-go/deepcode"
"github.com/snyk/code-client-go/internal/deepcode"
Copy link
Contributor

@acke acke Apr 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Very nice!

"github.com/snyk/code-client-go/observability"
)

//go:generate mockgen -destination=mocks/bundle.go -source=bundle.go -package mocks
type Bundle interface {
UploadBatch(ctx context.Context, batch *Batch) error
UploadBatch(ctx context.Context, requestId string, batch *Batch) error
GetBundleHash() string
GetRootPath() string
GetRequestId() string
GetFiles() map[string]deepcode.BundleFile
GetMissingFiles() []string
}
Expand All @@ -39,8 +38,6 @@ type deepCodeBundle struct {
instrumentor observability.Instrumentor
errorReporter observability.ErrorReporter
logger *zerolog.Logger
requestId string
rootPath string
files map[string]deepcode.BundleFile
bundleHash string
batches []*Batch
Expand All @@ -54,8 +51,6 @@ func NewBundle(
errorReporter observability.ErrorReporter,
logger *zerolog.Logger,
bundleHash string,
requestId string,
rootPath string,
files map[string]deepcode.BundleFile,
limitToFiles []string,
missingFiles []string,
Expand All @@ -66,8 +61,6 @@ func NewBundle(
errorReporter: errorReporter,
logger: logger,
bundleHash: bundleHash,
requestId: requestId,
rootPath: rootPath,
batches: []*Batch{},
files: files,
limitToFiles: limitToFiles,
Expand All @@ -79,14 +72,6 @@ func (b *deepCodeBundle) GetBundleHash() string {
return b.bundleHash
}

func (b *deepCodeBundle) GetRootPath() string {
return b.rootPath
}

func (b *deepCodeBundle) GetRequestId() string {
return b.requestId
}

func (b *deepCodeBundle) GetFiles() map[string]deepcode.BundleFile {
return b.files
}
Expand All @@ -95,20 +80,20 @@ func (b *deepCodeBundle) GetMissingFiles() []string {
return b.missingFiles
}

func (b *deepCodeBundle) UploadBatch(ctx context.Context, batch *Batch) error {
err := b.extendBundle(ctx, batch)
func (b *deepCodeBundle) UploadBatch(ctx context.Context, requestId string, batch *Batch) error {
err := b.extendBundle(ctx, requestId, batch)
if err != nil {
return err
}
b.batches = append(b.batches, batch)
return nil
}

func (b *deepCodeBundle) extendBundle(ctx context.Context, uploadBatch *Batch) error {
func (b *deepCodeBundle) extendBundle(ctx context.Context, requestId string, uploadBatch *Batch) error {
var err error
if uploadBatch.hasContent() {
b.bundleHash, b.missingFiles, err = b.SnykCode.ExtendBundle(ctx, b.bundleHash, uploadBatch.documents, []string{})
b.logger.Debug().Str("requestId", b.requestId).Interface("MissingFiles", b.missingFiles).Msg("extended deepCodeBundle on backend")
b.logger.Debug().Str("requestId", requestId).Interface("MissingFiles", b.missingFiles).Msg("extended deepCodeBundle on backend")
}

return err
Expand Down
26 changes: 13 additions & 13 deletions bundle/bundle_manager.go → internal/bundle/bundle_manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,20 @@ package bundle

import (
"context"
deepcode2 "github.com/snyk/code-client-go/internal/deepcode"
"os"
"path/filepath"

"github.com/puzpuzpuz/xsync"
"github.com/rs/zerolog"

"github.com/snyk/code-client-go/deepcode"
"github.com/snyk/code-client-go/internal/util"
"github.com/snyk/code-client-go/observability"
)

// TODO: add progress tracker for percentage progress
type bundleManager struct {
SnykCode deepcode.SnykCodeClient
SnykCode deepcode2.SnykCodeClient
instrumentor observability.Instrumentor
errorReporter observability.ErrorReporter
logger *zerolog.Logger
Expand All @@ -50,14 +50,15 @@ type BundleManager interface {

Upload(
ctx context.Context,
requestId string,
originalBundle Bundle,
files map[string]deepcode.BundleFile,
files map[string]deepcode2.BundleFile,
) (Bundle, error)
}

func NewBundleManager(
logger *zerolog.Logger,
SnykCode deepcode.SnykCodeClient,
SnykCode deepcode2.SnykCodeClient,
instrumentor observability.Instrumentor,
errorReporter observability.ErrorReporter,
) *bundleManager {
Expand All @@ -82,7 +83,7 @@ func (b *bundleManager) Create(ctx context.Context,

var limitToFiles []string
fileHashes := make(map[string]string)
bundleFiles := make(map[string]deepcode.BundleFile)
bundleFiles := make(map[string]deepcode2.BundleFile)
noFiles := true
for absoluteFilePath := range filePaths {
noFiles = false
Expand Down Expand Up @@ -115,7 +116,7 @@ func (b *bundleManager) Create(ctx context.Context,
}
relativePath = util.EncodePath(relativePath)

bundleFile := deepcode.BundleFileFrom(absoluteFilePath, fileContent)
bundleFile := deepcode2.BundleFileFrom(absoluteFilePath, fileContent)
bundleFiles[relativePath] = bundleFile
fileHashes[relativePath] = bundleFile.Hash

Expand All @@ -139,8 +140,6 @@ func (b *bundleManager) Create(ctx context.Context,
b.errorReporter,
b.logger,
bundleHash,
requestId,
rootPath,
bundleFiles,
limitToFiles,
missingFiles,
Expand All @@ -150,8 +149,9 @@ func (b *bundleManager) Create(ctx context.Context,

func (b *bundleManager) Upload(
ctx context.Context,
requestId string,
bundle Bundle,
files map[string]deepcode.BundleFile,
files map[string]deepcode2.BundleFile,
) (Bundle, error) {
method := "code.Batch"
s := b.instrumentor.StartSpan(ctx, method)
Expand All @@ -168,7 +168,7 @@ func (b *bundleManager) Upload(
if err := ctx.Err(); err != nil {
return bundle, err
}
err := bundle.UploadBatch(s.Context(), batch)
err := bundle.UploadBatch(s.Context(), requestId, batch)
if err != nil {
return bundle, err
}
Expand All @@ -181,14 +181,14 @@ func (b *bundleManager) Upload(
func (b *bundleManager) groupInBatches(
ctx context.Context,
bundle Bundle,
files map[string]deepcode.BundleFile,
files map[string]deepcode2.BundleFile,
) []*Batch {
method := "code.groupInBatches"
s := b.instrumentor.StartSpan(ctx, method)
defer b.instrumentor.Finish(s)

var batches []*Batch
batch := NewBatch(map[string]deepcode.BundleFile{})
batch := NewBatch(map[string]deepcode2.BundleFile{})
for _, filePath := range bundle.GetMissingFiles() {
if len(batches) == 0 { // first batch added after first file found
batches = append(batches, batch)
Expand All @@ -201,7 +201,7 @@ func (b *bundleManager) groupInBatches(
batch.documents[filePath] = file
} else {
b.logger.Trace().Str("path", filePath).Int("size", len(fileContent)).Msgf("created new deepCodeBundle - %v bundles in this upload so far", len(batches))
newUploadBatch := NewBatch(map[string]deepcode.BundleFile{})
newUploadBatch := NewBatch(map[string]deepcode2.BundleFile{})
newUploadBatch.documents[filePath] = file
batches = append(batches, newUploadBatch)
batch = newUploadBatch
Expand Down
62 changes: 33 additions & 29 deletions bundle/bundle_manager_test.go → internal/bundle/bundle_manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/snyk/code-client-go/bundle"
deepcode2 "github.com/snyk/code-client-go/deepcode"
mocks2 "github.com/snyk/code-client-go/deepcode/mocks"
"github.com/snyk/code-client-go/internal/bundle"
"github.com/snyk/code-client-go/internal/deepcode"
deepcodeMocks "github.com/snyk/code-client-go/internal/deepcode/mocks"
"github.com/snyk/code-client-go/internal/util"
"github.com/snyk/code-client-go/observability/mocks"
)
Expand All @@ -42,8 +42,8 @@ func Test_Create(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: []string{},
Extensions: []string{".java"},
}, nil)
Expand Down Expand Up @@ -77,8 +77,8 @@ func Test_Create(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: []string{},
Extensions: []string{".java"},
}, nil)
Expand Down Expand Up @@ -110,8 +110,8 @@ func Test_Create(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: []string{},
Extensions: []string{".java"},
}, nil)
Expand Down Expand Up @@ -147,8 +147,8 @@ func Test_Create(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: []string{},
Extensions: []string{".java"},
}, nil)
Expand Down Expand Up @@ -181,8 +181,8 @@ func Test_Create(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: []string{".test"},
Extensions: []string{},
}, nil)
Expand Down Expand Up @@ -214,8 +214,8 @@ func Test_Create(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: []string{},
Extensions: []string{".java"},
}, nil)
Expand Down Expand Up @@ -272,7 +272,7 @@ func Test_Upload(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().ExtendBundle(gomock.Any(), "bundleHash", gomock.Len(1), []string{}).Times(1)
mockInstrumentor := mocks.NewMockInstrumentor(ctrl)
mockInstrumentor.EXPECT().StartSpan(gomock.Any(), gomock.Any()).Return(mockSpan).Times(2)
Expand All @@ -281,11 +281,13 @@ func Test_Upload(t *testing.T) {

var bundleManager = bundle.NewBundleManager(newLogger(t), mockSnykCodeClient, mockInstrumentor, mockErrorReporter)
documentURI, bundleFile := createTempFileInDir(t, "bundleDoc.java", 10, temporaryDir)
bundleFileMap := map[string]deepcode2.BundleFile{}
bundleFileMap := map[string]deepcode.BundleFile{}
bundleFileMap[documentURI] = bundleFile

_, err := bundleManager.Upload(context.Background(),
bundle.NewBundle(mockSnykCodeClient, mockInstrumentor, mockErrorReporter, &logger, "bundleHash", "testRequestId", "", bundleFileMap, []string{}, []string{documentURI}),
_, err := bundleManager.Upload(
context.Background(),
"testRequestId",
bundle.NewBundle(mockSnykCodeClient, mockInstrumentor, mockErrorReporter, &logger, "bundleHash", bundleFileMap, []string{}, []string{documentURI}),
bundleFileMap)
assert.NoError(t, err)
})
Expand All @@ -294,7 +296,7 @@ func Test_Upload(t *testing.T) {
ctrl := gomock.NewController(t)
mockSpan := mocks.NewMockSpan(ctrl)
mockSpan.EXPECT().Context().AnyTimes()
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().ExtendBundle(gomock.Any(), "bundleHash", gomock.Len(3), []string{}).Return("newBundleHash", []string{}, nil).Times(1)
mockSnykCodeClient.EXPECT().ExtendBundle(gomock.Any(), "newBundleHash", gomock.Len(2), []string{}).Return("newerBundleHash", []string{}, nil).Times(1)
mockInstrumentor := mocks.NewMockInstrumentor(ctrl)
Expand All @@ -303,7 +305,7 @@ func Test_Upload(t *testing.T) {
mockErrorReporter := mocks.NewMockErrorReporter(ctrl)
var bundleManager = bundle.NewBundleManager(newLogger(t), mockSnykCodeClient, mockInstrumentor, mockErrorReporter)

bundleFileMap := map[string]deepcode2.BundleFile{}
bundleFileMap := map[string]deepcode.BundleFile{}
var missingFiles []string
path, bundleFile := createTempFileInDir(t, "bundleDoc1.java", (1024*1024)-1, temporaryDir)
bundleFileMap[path] = bundleFile
Expand All @@ -321,24 +323,26 @@ func Test_Upload(t *testing.T) {
bundleFileMap[path] = bundleFile
missingFiles = append(missingFiles, path)

_, err := bundleManager.Upload(context.Background(),
bundle.NewBundle(mockSnykCodeClient, mockInstrumentor, mockErrorReporter, &logger, "bundleHash", "testRequestId", "", bundleFileMap, []string{}, missingFiles),
_, err := bundleManager.Upload(
context.Background(),
"testRequestId",
bundle.NewBundle(mockSnykCodeClient, mockInstrumentor, mockErrorReporter, &logger, "bundleHash", bundleFileMap, []string{}, missingFiles),
bundleFileMap)
assert.Nil(t, err)
})
}

func createTempFileInDir(t *testing.T, name string, size int, temporaryDir string) (string, deepcode2.BundleFile) {
func createTempFileInDir(t *testing.T, name string, size int, temporaryDir string) (string, deepcode.BundleFile) {
t.Helper()

documentURI, fileContent := createFileOfSize(t, name, size, temporaryDir)
return documentURI, deepcode2.BundleFile{Hash: util.Hash(fileContent), Content: string(fileContent)}
return documentURI, deepcode.BundleFile{Hash: util.Hash(fileContent), Content: string(fileContent)}
}

func Test_IsSupported_Extensions(t *testing.T) {
ctrl := gomock.NewController(t)
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: []string{},
Extensions: []string{".java"},
}, nil)
Expand Down Expand Up @@ -376,8 +380,8 @@ func Test_IsSupported_ConfigFiles(t *testing.T) {
}

ctrl := gomock.NewController(t)
mockSnykCodeClient := mocks2.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode2.FiltersResponse{
mockSnykCodeClient := deepcodeMocks.NewMockSnykCodeClient(ctrl)
mockSnykCodeClient.EXPECT().GetFilters(gomock.Any()).Return(deepcode.FiltersResponse{
ConfigFiles: configFilesFromFiltersEndpoint,
Extensions: []string{},
}, nil)
Expand Down
Loading