Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Include Pagination for KeyValue #1904

Merged
merged 8 commits into from
May 20, 2024
Merged

Include Pagination for KeyValue #1904

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
cfca9d5
Include pagination for some more commands
nathannaveen Apr 26, 2024
b612ced
Included pagination for certifyScorecard
nathannaveen Apr 24, 2024
82a48ef
Included Pagination for certifyBad
nathannaveen Apr 26, 2024
e1abf5a
Included pagination for more commands
nathannaveen May 1, 2024
94e1980
Added pagination for more queries
nathannaveen May 2, 2024
00954b4
Fixed tests
nathannaveen May 6, 2024
4390123
Updated based on code review
nathannaveen May 16, 2024
8695bdc
Updated based on code review
nathannaveen May 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions internal/testing/backend/helpers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ var commonOpts = cmp.Options{
cmpopts.SortSlices(lessHM),
cmpopts.SortSlices(lessPackageOrArtifact),
cmpopts.SortSlices(lessSLSAPred),
cmpopts.SortSlices(hasSlsaLess),
cmpopts.SortSlices(lessHSA),
cmpopts.SortSlices(lessIsDep),
cmpopts.SortSlices(lessIsOcc),
Expand Down Expand Up @@ -586,6 +587,10 @@ func lessSLSAPred(a, b *model.SLSAPredicate) bool {
return false
}

func hasSlsaLess(a, b *model.HasSlsa) bool {
return cmpArt(a.Subject, b.Subject) < 0
}

func lessPackageOrArtifact(a, b model.PackageOrArtifact) bool {
return cmpPackageOrArtifact(a, b) < 0
}
Expand Down
84 changes: 42 additions & 42 deletions internal/testing/backend/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,40 +38,40 @@ const (

var skipMatrix = map[string]map[string]bool{
// pagination not implemented
"TestArtifacts": {arango: true, memmap: true, redis: true, tikv: true},
"TestBuilder": {arango: true, memmap: true, redis: true, tikv: true},
"TestBuilders": {arango: true, memmap: true, redis: true, tikv: true},
"TestCertifyBad": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyBads": {arango: true, memmap: true, redis: true, tikv: true},
"TestCertifyGood": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyGoods": {arango: true, memmap: true, redis: true, tikv: true},
"TestLegal": {arango: true, memmap: true, redis: true, tikv: true},
"TestLegals": {arango: true, memmap: true, redis: true, tikv: true},
"TestCertifyScorecard": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestScorecards": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyVulnerability": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyVulns": {arango: true, memmap: true, redis: true, tikv: true},
"TestHasMetadata": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestBulkHasMetadata": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHasSBOMs": {arango: true, memmap: true, redis: true, tikv: true},
"TestHasSLSA": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHasSLSAs": {arango: true, memmap: true, redis: true, tikv: true},
"TestHasSourceAt": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHasSourceAts": {arango: true, memmap: true, redis: true, tikv: true},
"TestHashEqual": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHashEquals": {arango: true, memmap: true, redis: true, tikv: true},
"TestIsDependencies": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestOccurrences": {arango: true, memmap: true, redis: true, tikv: true},
"TestLicenses": {arango: true, memmap: true, redis: true, tikv: true},
"TestLicensesBulk": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestPkgEquals": {arango: true, memmap: true, redis: true, tikv: true},
"TestPackages": {arango: true, memmap: true, redis: true, tikv: true},
"TestPointOfContact": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestPointOfContacts": {arango: true, memmap: true, redis: true, tikv: true},
"TestSources": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestVulnEquals": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestVulnMetadata": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestVulnMetadatas": {arango: true, memmap: true, redis: true, tikv: true},
"TestArtifacts": {arango: true, redis: true, tikv: true},
"TestBuilder": {arango: true, redis: true, tikv: true},
"TestBuilders": {arango: true, redis: true, tikv: true},
"TestCertifyBad": {arango: true, redis: true, tikv: true},
"TestIngestCertifyBads": {arango: true, redis: true, tikv: true},
"TestCertifyGood": {arango: true, redis: true, tikv: true},
"TestIngestCertifyGoods": {arango: true, redis: true, tikv: true},
"TestLegal": {arango: true, redis: true, tikv: true},
"TestLegals": {arango: true, redis: true, tikv: true},
"TestCertifyScorecard": {arango: true, redis: true, tikv: true},
"TestIngestScorecards": {arango: true, redis: true, tikv: true},
"TestIngestCertifyVulnerability": {arango: true, redis: true, tikv: true},
"TestIngestCertifyVulns": {arango: true, redis: true, tikv: true},
"TestHasMetadata": {arango: true, redis: true, tikv: true},
"TestIngestBulkHasMetadata": {arango: true, redis: true, tikv: true},
"TestIngestHasSBOMs": {arango: true, redis: true, tikv: true},
"TestHasSLSA": {arango: true, redis: true, tikv: true},
"TestIngestHasSLSAs": {arango: true, redis: true, tikv: true},
"TestHasSourceAt": {arango: true, redis: true, tikv: true},
"TestIngestHasSourceAts": {arango: true, redis: true, tikv: true},
"TestHashEqual": {arango: true, redis: true, tikv: true},
"TestIngestHashEquals": {arango: true, redis: true, tikv: true},
"TestIsDependencies": {arango: true, redis: true, tikv: true},
"TestIngestOccurrences": {arango: true, redis: true, tikv: true},
"TestLicenses": {arango: true, redis: true, tikv: true},
"TestLicensesBulk": {arango: true, redis: true, tikv: true},
"TestIngestPkgEquals": {arango: true, redis: true, tikv: true},
"TestPackages": {arango: true, redis: true, tikv: true},
"TestPointOfContact": {arango: true, redis: true, tikv: true},
"TestIngestPointOfContacts": {arango: true, redis: true, tikv: true},
"TestSources": {arango: true, redis: true, tikv: true},
"TestIngestVulnEquals": {arango: true, redis: true, tikv: true},
"TestIngestVulnMetadata": {arango: true, redis: true, tikv: true},
"TestIngestVulnMetadatas": {arango: true, redis: true, tikv: true},

// arango fails IncludedOccurrences_-_Valid_Included_ID and IncludedDependencies_-_Valid_Included_ID
"TestHasSBOM": {arango: true},
Expand All @@ -87,11 +87,11 @@ var skipMatrix = map[string]map[string]bool{
"TestPkgEqual": {arango: true, memmap: true, redis: true, tikv: true},
// keyvalue: Query_on_OSV_and_novuln_(return_nothing_as_not_valid) fails
// arango: errors when ID is not found
"TestVulnEqual": {memmap: true, redis: true, tikv: true, arango: true},
"TestVulnEqual": {redis: true, memmap: true, tikv: true, arango: true},
// arango: errors when ID is not found
"TestVulnerability": {arango: true},
"TestVulnerability": {arango: true, redis: true, tikv: true},
// redis order issues
"TestVEX": {arango: true, redis: true},
"TestVEX": {arango: true, redis: true, tikv: true},
// redis order issues
"TestVEXBulkIngest": {arango: true, redis: true},
"TestFindSoftware": {redis: true, arango: true},
Expand All @@ -105,11 +105,11 @@ type backend interface {
}

var testBackends = map[string]backend{
// memmap: newMemMap(),
arango: newArango(),
// redis: newRedis(),
ent: newEnt(),
// tikv: newTikv(),
memmap: newMemMap(),
//arango: newArango(),
//redis: newRedis(),
//ent: newEnt(),
//tikv: newTikv(),
}

var currentBackend string
Expand Down
43 changes: 28 additions & 15 deletions pkg/assembler/backends/keyvalue/artifact.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,60 +257,73 @@ func (c *demoClient) ArtifactsList(ctx context.Context, artifactSpec model.Artif
currentPage = true
}
hasNextPage := false
totalCount := 0

algorithm := strings.ToLower(nilToEmpty(artifactSpec.Algorithm))
digest := strings.ToLower(nilToEmpty(artifactSpec.Digest))

var done bool
scn := c.kv.Keys(artCol)

var artKeys []string

for !done {
artKeys, done, err = scn.Scan(ctx)
if err != nil {
return nil, err
}

sort.Strings(artKeys)

totalCount = len(artKeys)

for i, ak := range artKeys {
a, err := byKeykv[*artStruct](ctx, artCol, ak, c)
if err != nil {
return nil, err
}

convArt := c.convArtifact(a)
if after != nil && !currentPage {
if convArt.ID == *after {
currentPage = true
continue
} else {
continue
totalCount = len(artKeys) - (i + 1)
}
continue
}

if convArt == nil {
continue
}

artEdge := createArtifactEdges(algorithm, a, digest, convArt)

if artEdge == nil {
continue
}

if first != nil {
if currentPage && count < *first {
artEdge := createArtifactEdges(algorithm, a, digest, convArt)
if artEdge != nil {
edges = append(edges, artEdge)
count++
}
edges = append(edges, artEdge)
count++
}
// If there are any elements left after the current page we indicate that in the response
if count == *first && i < len(artKeys) {
hasNextPage = true
}
} else {
artEdge := createArtifactEdges(algorithm, a, digest, convArt)
if artEdge != nil {
edges = append(edges, artEdge)
count++
}
edges = append(edges, artEdge)
count++
}
}
}
if len(edges) > 0 {
return &model.ArtifactConnection{
TotalCount: len(artKeys),
TotalCount: totalCount,
PageInfo: &model.PageInfo{
HasNextPage: hasNextPage,
StartCursor: ptrfrom.String(edges[0].Node.ID),
EndCursor: ptrfrom.String(edges[count-1].Node.ID),
EndCursor: ptrfrom.String(edges[max(count-1, 0)].Node.ID),
},
Edges: edges}, nil
} else {
Expand Down
22 changes: 15 additions & 7 deletions pkg/assembler/backends/keyvalue/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ func (c *demoClient) BuildersList(ctx context.Context, builderSpec model.Builder
currentPage = true
}
hasNextPage := false
totalCount := 0

var done bool

scn := c.kv.Keys(builderCol)
Expand All @@ -156,19 +158,26 @@ func (c *demoClient) BuildersList(ctx context.Context, builderSpec model.Builder
return nil, err
}
sort.Strings(bKeys)

totalCount = len(bKeys)

for i, bk := range bKeys {
b, err := byKeykv[*builderStruct](ctx, builderCol, bk, c)
if err != nil {
return nil, err
}
convBuild := c.convBuilder(b)

if convBuild == nil {
continue
}

if after != nil && !currentPage {
if convBuild.ID == *after {
currentPage = true
continue
} else {
continue
totalCount = len(bKeys) - (i + 1)
}
continue
}
if first != nil {
if currentPage && count < *first {
Expand All @@ -193,16 +202,15 @@ func (c *demoClient) BuildersList(ctx context.Context, builderSpec model.Builder
}
if len(edges) > 0 {
return &model.BuilderConnection{
TotalCount: len(bKeys),
TotalCount: totalCount,
PageInfo: &model.PageInfo{
HasNextPage: hasNextPage,
StartCursor: ptrfrom.String(edges[0].Node.ID),
EndCursor: ptrfrom.String(edges[count-1].Node.ID),
EndCursor: ptrfrom.String(edges[max(0, count-1)].Node.ID),
},
Edges: edges}, nil
} else {
return nil, nil
}
return nil, nil
}

func (c *demoClient) Builders(ctx context.Context, builderSpec *model.BuilderSpec) ([]*model.Builder, error) {
Expand Down