Skip to content

Commit

Permalink
Include Pagination for KeyValue (#1904)
Browse files Browse the repository at this point in the history 
* Include pagination for some more commands

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

* Included pagination for certifyScorecard

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

* Included Pagination for certifyBad

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

* Included pagination for more commands

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

* Added pagination for more queries

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

* Fixed tests

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

* Updated based on code review

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

* Updated based on code review

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>

---------

Signed-off-by: nathannaveen <42319948+nathannaveen@users.noreply.github.com>
  • Loading branch information
@nathannaveen
nathannaveen committed May 20, 2024
1 parent b74c853 commit 0970c35
Show file tree
Hide file tree
Showing 47 changed files with 4,461 additions and 294 deletions.
5 changes: 5 additions & 0 deletions internal/testing/backend/helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ var commonOpts = cmp.Options{
cmpopts.SortSlices(lessHM),
cmpopts.SortSlices(lessPackageOrArtifact),
cmpopts.SortSlices(lessSLSAPred),
cmpopts.SortSlices(hasSlsaLess),
cmpopts.SortSlices(lessHSA),
cmpopts.SortSlices(lessIsDep),
cmpopts.SortSlices(lessIsOcc),
Expand Down Expand Up @@ -586,6 +587,10 @@ func lessSLSAPred(a, b *model.SLSAPredicate) bool {
return false
}

func hasSlsaLess(a, b *model.HasSlsa) bool {
return cmpArt(a.Subject, b.Subject) < 0
}

func lessPackageOrArtifact(a, b model.PackageOrArtifact) bool {
return cmpPackageOrArtifact(a, b) < 0
}
Expand Down
82 changes: 41 additions & 41 deletions internal/testing/backend/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,40 +38,40 @@ const (

var skipMatrix = map[string]map[string]bool{
// pagination not implemented
"TestArtifacts": {arango: true, memmap: true, redis: true, tikv: true},
"TestBuilder": {arango: true, memmap: true, redis: true, tikv: true},
"TestBuilders": {arango: true, memmap: true, redis: true, tikv: true},
"TestCertifyBad": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyBads": {arango: true, memmap: true, redis: true, tikv: true},
"TestCertifyGood": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyGoods": {arango: true, memmap: true, redis: true, tikv: true},
"TestLegal": {arango: true, memmap: true, redis: true, tikv: true},
"TestLegals": {arango: true, memmap: true, redis: true, tikv: true},
"TestCertifyScorecard": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestScorecards": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyVulnerability": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestCertifyVulns": {arango: true, memmap: true, redis: true, tikv: true},
"TestHasMetadata": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestBulkHasMetadata": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHasSBOMs": {arango: true, memmap: true, redis: true, tikv: true},
"TestHasSLSA": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHasSLSAs": {arango: true, memmap: true, redis: true, tikv: true},
"TestHasSourceAt": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHasSourceAts": {arango: true, memmap: true, redis: true, tikv: true},
"TestHashEqual": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestHashEquals": {arango: true, memmap: true, redis: true, tikv: true},
"TestIsDependencies": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestOccurrences": {arango: true, memmap: true, redis: true, tikv: true},
"TestLicenses": {arango: true, memmap: true, redis: true, tikv: true},
"TestLicensesBulk": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestPkgEquals": {arango: true, memmap: true, redis: true, tikv: true},
"TestPackages": {arango: true, memmap: true, redis: true, tikv: true},
"TestPointOfContact": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestPointOfContacts": {arango: true, memmap: true, redis: true, tikv: true},
"TestSources": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestVulnEquals": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestVulnMetadata": {arango: true, memmap: true, redis: true, tikv: true},
"TestIngestVulnMetadatas": {arango: true, memmap: true, redis: true, tikv: true},
"TestArtifacts": {arango: true, redis: true, tikv: true},
"TestBuilder": {arango: true, redis: true, tikv: true},
"TestBuilders": {arango: true, redis: true, tikv: true},
"TestCertifyBad": {arango: true, redis: true, tikv: true},
"TestIngestCertifyBads": {arango: true, redis: true, tikv: true},
"TestCertifyGood": {arango: true, redis: true, tikv: true},
"TestIngestCertifyGoods": {arango: true, redis: true, tikv: true},
"TestLegal": {arango: true, redis: true, tikv: true},
"TestLegals": {arango: true, redis: true, tikv: true},
"TestCertifyScorecard": {arango: true, redis: true, tikv: true},
"TestIngestScorecards": {arango: true, redis: true, tikv: true},
"TestIngestCertifyVulnerability": {arango: true, redis: true, tikv: true},
"TestIngestCertifyVulns": {arango: true, redis: true, tikv: true},
"TestHasMetadata": {arango: true, redis: true, tikv: true},
"TestIngestBulkHasMetadata": {arango: true, redis: true, tikv: true},
"TestIngestHasSBOMs": {arango: true, redis: true, tikv: true},
"TestHasSLSA": {arango: true, redis: true, tikv: true},
"TestIngestHasSLSAs": {arango: true, redis: true, tikv: true},
"TestHasSourceAt": {arango: true, redis: true, tikv: true},
"TestIngestHasSourceAts": {arango: true, redis: true, tikv: true},
"TestHashEqual": {arango: true, redis: true, tikv: true},
"TestIngestHashEquals": {arango: true, redis: true, tikv: true},
"TestIsDependencies": {arango: true, redis: true, tikv: true},
"TestIngestOccurrences": {arango: true, redis: true, tikv: true},
"TestLicenses": {arango: true, redis: true, tikv: true},
"TestLicensesBulk": {arango: true, redis: true, tikv: true},
"TestIngestPkgEquals": {arango: true, redis: true, tikv: true},
"TestPackages": {arango: true, redis: true, tikv: true},
"TestPointOfContact": {arango: true, redis: true, tikv: true},
"TestIngestPointOfContacts": {arango: true, redis: true, tikv: true},
"TestSources": {arango: true, redis: true, tikv: true},
"TestIngestVulnEquals": {arango: true, redis: true, tikv: true},
"TestIngestVulnMetadata": {arango: true, redis: true, tikv: true},
"TestIngestVulnMetadatas": {arango: true, redis: true, tikv: true},

// arango fails IncludedOccurrences_-_Valid_Included_ID and IncludedDependencies_-_Valid_Included_ID
"TestHasSBOM": {arango: true},
Expand All @@ -87,11 +87,11 @@ var skipMatrix = map[string]map[string]bool{
"TestPkgEqual": {arango: true, memmap: true, redis: true, tikv: true},
// keyvalue: Query_on_OSV_and_novuln_(return_nothing_as_not_valid) fails
// arango: errors when ID is not found
"TestVulnEqual": {memmap: true, redis: true, tikv: true, arango: true},
"TestVulnEqual": {redis: true, memmap: true, tikv: true, arango: true},
// arango: errors when ID is not found
"TestVulnerability": {arango: true},
"TestVulnerability": {arango: true, redis: true, tikv: true},
// redis order issues
"TestVEX": {arango: true, redis: true},
"TestVEX": {arango: true, redis: true, tikv: true},
// redis order issues
"TestVEXBulkIngest": {arango: true, redis: true},
"TestFindSoftware": {redis: true, arango: true},
Expand All @@ -105,11 +105,11 @@ type backend interface {
}

var testBackends = map[string]backend{
// memmap: newMemMap(),
memmap: newMemMap(),
arango: newArango(),
// redis: newRedis(),
ent: newEnt(),
// tikv: newTikv(),
redis: newRedis(),
ent: newEnt(),
tikv: newTikv(),
}

var currentBackend string
Expand Down
43 changes: 28 additions & 15 deletions pkg/assembler/backends/keyvalue/artifact.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,60 +257,73 @@ func (c *demoClient) ArtifactsList(ctx context.Context, artifactSpec model.Artif
currentPage = true
}
hasNextPage := false
totalCount := 0

algorithm := strings.ToLower(nilToEmpty(artifactSpec.Algorithm))
digest := strings.ToLower(nilToEmpty(artifactSpec.Digest))

var done bool
scn := c.kv.Keys(artCol)

var artKeys []string

for !done {
artKeys, done, err = scn.Scan(ctx)
if err != nil {
return nil, err
}

sort.Strings(artKeys)

totalCount = len(artKeys)

for i, ak := range artKeys {
a, err := byKeykv[*artStruct](ctx, artCol, ak, c)
if err != nil {
return nil, err
}

convArt := c.convArtifact(a)
if after != nil && !currentPage {
if convArt.ID == *after {
currentPage = true
continue
} else {
continue
totalCount = len(artKeys) - (i + 1)
}
continue
}

if convArt == nil {
continue
}

artEdge := createArtifactEdges(algorithm, a, digest, convArt)

if artEdge == nil {
continue
}

if first != nil {
if currentPage && count < *first {
artEdge := createArtifactEdges(algorithm, a, digest, convArt)
if artEdge != nil {
edges = append(edges, artEdge)
count++
}
edges = append(edges, artEdge)
count++
}
// If there are any elements left after the current page we indicate that in the response
if count == *first && i < len(artKeys) {
hasNextPage = true
}
} else {
artEdge := createArtifactEdges(algorithm, a, digest, convArt)
if artEdge != nil {
edges = append(edges, artEdge)
count++
}
edges = append(edges, artEdge)
count++
}
}
}
if len(edges) > 0 {
return &model.ArtifactConnection{
TotalCount: len(artKeys),
TotalCount: totalCount,
PageInfo: &model.PageInfo{
HasNextPage: hasNextPage,
StartCursor: ptrfrom.String(edges[0].Node.ID),
EndCursor: ptrfrom.String(edges[count-1].Node.ID),
EndCursor: ptrfrom.String(edges[max(count-1, 0)].Node.ID),
},
Edges: edges}, nil
} else {
Expand Down
22 changes: 15 additions & 7 deletions pkg/assembler/backends/keyvalue/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ func (c *demoClient) BuildersList(ctx context.Context, builderSpec model.Builder
currentPage = true
}
hasNextPage := false
totalCount := 0

var done bool

scn := c.kv.Keys(builderCol)
Expand All @@ -156,19 +158,26 @@ func (c *demoClient) BuildersList(ctx context.Context, builderSpec model.Builder
return nil, err
}
sort.Strings(bKeys)

totalCount = len(bKeys)

for i, bk := range bKeys {
b, err := byKeykv[*builderStruct](ctx, builderCol, bk, c)
if err != nil {
return nil, err
}
convBuild := c.convBuilder(b)

if convBuild == nil {
continue
}

if after != nil && !currentPage {
if convBuild.ID == *after {
currentPage = true
continue
} else {
continue
totalCount = len(bKeys) - (i + 1)
}
continue
}
if first != nil {
if currentPage && count < *first {
Expand All @@ -193,16 +202,15 @@ func (c *demoClient) BuildersList(ctx context.Context, builderSpec model.Builder
}
if len(edges) > 0 {
return &model.BuilderConnection{
TotalCount: len(bKeys),
TotalCount: totalCount,
PageInfo: &model.PageInfo{
HasNextPage: hasNextPage,
StartCursor: ptrfrom.String(edges[0].Node.ID),
EndCursor: ptrfrom.String(edges[count-1].Node.ID),
EndCursor: ptrfrom.String(edges[max(0, count-1)].Node.ID),
},
Edges: edges}, nil
} else {
return nil, nil
}
return nil, nil
}

func (c *demoClient) Builders(ctx context.Context, builderSpec *model.BuilderSpec) ([]*model.Builder, error) {
Expand Down
Loading

0 comments on commit 0970c35

Please sign in to comment.