Skip to content

Commit

Permalink
add query for IsOccurrence, isDependency and HasSBOM for Arango backe…
Browse files Browse the repository at this point in the history 
…nd (#1096)

* remove lables and fix insert for dependency

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add query for isDependency

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add query for hasSBOM

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add query for hasSBOM and isOccurrence

Signed-off-by: pxp928 <parth.psu@gmail.com>

* remove unused function

Signed-off-by: pxp928 <parth.psu@gmail.com>

---------

Signed-off-by: pxp928 <parth.psu@gmail.com>
  • Loading branch information
@pxp928
pxp928 committed Jul 25, 2023
1 parent 093d702 commit 3f93cd4
Show file tree
Hide file tree
Showing 10 changed files with 485 additions and 71 deletions.
9 changes: 6 additions & 3 deletions pkg/assembler/backends/arangodb/artifact.go
View file
Expand Up @@ -27,7 +27,8 @@ import (

func (c *arangoClient) Artifacts(ctx context.Context, artifactSpec *model.ArtifactSpec) ([]*model.Artifact, error) {
values := map[string]any{}
arangoQueryBuilder := setArtifactMatchValues(artifactSpec, values)

arangoQueryBuilder := setArtifactMatchValues(nil, artifactSpec, values)
arangoQueryBuilder.query.WriteString("\n")
arangoQueryBuilder.query.WriteString(`RETURN {
"id": art._id,
Expand All @@ -45,8 +46,10 @@ func (c *arangoClient) Artifacts(ctx context.Context, artifactSpec *model.Artifa
return getArtifacts(ctx, cursor)
}

func setArtifactMatchValues(artifactSpec *model.ArtifactSpec, queryValues map[string]any) *arangoQueryBuilder {
arangoQueryBuilder := newForQuery(artifactsStr, "art")
func setArtifactMatchValues(arangoQueryBuilder *arangoQueryBuilder, artifactSpec *model.ArtifactSpec, queryValues map[string]any) *arangoQueryBuilder {
if arangoQueryBuilder == nil {
arangoQueryBuilder = newForQuery(artifactsStr, "art")
}
if artifactSpec != nil {
if artifactSpec.ID != nil {
arangoQueryBuilder.filter("art", "_id", "==", "@id")
Expand Down
33 changes: 29 additions & 4 deletions pkg/assembler/backends/arangodb/backend.go
View file
Expand Up @@ -700,12 +700,37 @@ func newForQuery(repositoryName string, counterName string) *arangoQueryBuilder
return aqb
}

func (aqb *arangoQueryBuilder) ForOutBound(edgeCollectionName string, counterName string, outBoundValueName string) *arangoQueryBuilder {
func (aqb *arangoQueryBuilder) forOutBound(edgeCollectionName string, counterVertexName string, outBoundStartVertexName string) *arangoQueryBuilder {
aqb.query.WriteString("\n")
aqb.query.WriteString(fmt.Sprintf("FOR %s IN OUTBOUND %s %s", counterName, outBoundValueName, edgeCollectionName))

aqb.query.WriteString(fmt.Sprintf("FOR %s IN OUTBOUND %s %s", counterVertexName, outBoundStartVertexName, edgeCollectionName))

return aqb
}

func (aqb *arangoQueryBuilder) forInBound(edgeCollectionName string, counterVertexName string, inBoundStartVertexName string) *arangoQueryBuilder {
aqb.query.WriteString("\n")

aqb.query.WriteString(fmt.Sprintf("FOR %s IN INBOUND %s %s", counterVertexName, inBoundStartVertexName, edgeCollectionName))

return aqb
}

func (aqb *arangoQueryBuilder) forInBoundWithEdgeCounter(edgeCollectionName string, counterVertexName string, counterEdgeName string, inBoundStartVertexName string) *arangoQueryBuilder {
aqb.query.WriteString("\n")

aqb.query.WriteString(fmt.Sprintf("FOR %s, %s IN INBOUND %s %s", counterVertexName, counterEdgeName, inBoundStartVertexName, edgeCollectionName))

return aqb
}

func (aqb *arangoQueryBuilder) prune(counterName string, fieldName string, condition string, value string) *arangoQueryFilter {
aqb.query.WriteString(" ")
aqb.query.WriteString(fmt.Sprintf("PRUNE %s.%s %s %s", counterName, fieldName, condition, value))

return newArangoQueryFilter(aqb)
}

func (aqb *arangoQueryBuilder) filter(counterName string, fieldName string, condition string, value string) *arangoQueryFilter {
aqb.query.WriteString(" ")

Expand Down Expand Up @@ -904,7 +929,7 @@ func preIngestPkgTypes(ctx context.Context, db driver.Database, pkgRoot *pkgRoot
)
LET pkgHasTypeCollection = (
INSERT { _key: CONCAT("pkgHasType", @rootKey, type._key), _from: @rootID, _to: type._id, label : "pkgHasType" } INTO pkgHasType OPTIONS { overwriteMode: "ignore" }
INSERT { _key: CONCAT("pkgHasType", @rootKey, type._key), _from: @rootID, _to: type._id} INTO pkgHasType OPTIONS { overwriteMode: "ignore" }
)
RETURN {
Expand Down Expand Up @@ -995,7 +1020,7 @@ func preIngestSrcTypes(ctx context.Context, db driver.Database, srcRoot *srcRoot
)
LET pkgHasTypeCollection = (
INSERT { _key: CONCAT("srcHasType", @rootKey, type._key), _from: @rootID, _to: type._id, label : "srcHasType" } INTO srcHasType OPTIONS { overwriteMode: "ignore" }
INSERT { _key: CONCAT("srcHasType", @rootKey, type._key), _from: @rootID, _to: type._id } INTO srcHasType OPTIONS { overwriteMode: "ignore" }
)
RETURN {
Expand Down
20 changes: 15 additions & 5 deletions pkg/assembler/backends/arangodb/certifyScorecard.go
View file
Expand Up @@ -39,10 +39,15 @@ const (
// Query Scorecards

func (c *arangoClient) Scorecards(ctx context.Context, certifyScorecardSpec *model.CertifyScorecardSpec) ([]*model.CertifyScorecard, error) {

values := map[string]any{}
var arangoQueryBuilder *arangoQueryBuilder

arangoQueryBuilder := setSrcMatchValues(certifyScorecardSpec.Source, values)
if certifyScorecardSpec.Source != nil {
arangoQueryBuilder = setSrcMatchValues(certifyScorecardSpec.Source, values)
arangoQueryBuilder.forOutBound(scorecardEdgesStr, "scorecard", "sName")
} else {
arangoQueryBuilder = newForQuery(scorecardStr, "scorecard")
}

setCertifyScorecardMatchValues(arangoQueryBuilder, certifyScorecardSpec, values)

Expand Down Expand Up @@ -80,7 +85,6 @@ func (c *arangoClient) Scorecards(ctx context.Context, certifyScorecardSpec *mod
}

func setCertifyScorecardMatchValues(arangoQueryBuilder *arangoQueryBuilder, certifyScorecardSpec *model.CertifyScorecardSpec, queryValues map[string]any) {
arangoQueryBuilder.ForOutBound(scorecardEdgesStr, "scorecard", "sName")
if certifyScorecardSpec.TimeScanned != nil {
arangoQueryBuilder.filter("scorecard", timeScannedStr, "==", "@"+timeScannedStr)
queryValues[timeScannedStr] = certifyScorecardSpec.TimeScanned.UTC()
Expand Down Expand Up @@ -110,6 +114,12 @@ func setCertifyScorecardMatchValues(arangoQueryBuilder *arangoQueryBuilder, cert
arangoQueryBuilder.filter("scorecard", collector, "==", "@"+collector)
queryValues["collector"] = certifyScorecardSpec.Collector
}
if certifyScorecardSpec.Source == nil {
// get sources
arangoQueryBuilder.forInBound(scorecardEdgesStr, "sName", "scorecard")
arangoQueryBuilder.forInBound(srcHasNameStr, "sNs", "sName")
arangoQueryBuilder.forInBound(srcHasNamespaceStr, "sType", "sNs")
}
}

func getChecks(qualifiersSpec []*model.ScorecardCheckSpec) []string {
Expand Down Expand Up @@ -221,7 +231,7 @@ func (c *arangoClient) IngestScorecards(ctx context.Context, sources []*model.So
)
LET edgeCollection = (
INSERT { _key: CONCAT("scorecardEdges", firstSrc.nameDoc._key, scorecard._key), _from: firstSrc.name_id, _to: scorecard._id, label : "certifyScorecard" } INTO scorecardEdges OPTIONS { overwriteMode: "ignore" }
INSERT { _key: CONCAT("scorecardEdges", firstSrc.nameDoc._key, scorecard._key), _from: firstSrc.name_id, _to: scorecard._id } INTO scorecardEdges OPTIONS { overwriteMode: "ignore" }
)
RETURN {
Expand Down Expand Up @@ -294,7 +304,7 @@ func (c *arangoClient) IngestScorecard(ctx context.Context, source model.SourceI
)
LET edgeCollection = (
INSERT { _key: CONCAT("scorecardEdges", firstSrc.nameDoc._key, scorecard._key), _from: firstSrc.name_id, _to: scorecard._id, label : "certifyScorecard" } INTO scorecardEdges OPTIONS { overwriteMode: "ignore" }
INSERT { _key: CONCAT("scorecardEdges", firstSrc.nameDoc._key, scorecard._key), _from: firstSrc.name_id, _to: scorecard._id } INTO scorecardEdges OPTIONS { overwriteMode: "ignore" }
)
RETURN {
Expand Down
152 changes: 147 additions & 5 deletions pkg/assembler/backends/arangodb/hasSBOM.go
View file
Expand Up @@ -25,7 +25,149 @@ import (
)

func (c *arangoClient) HasSBOM(ctx context.Context, hasSBOMSpec *model.HasSBOMSpec) ([]*model.HasSbom, error) {
return []*model.HasSbom{}, fmt.Errorf("not implemented: HasSBOM")

// TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval
values := map[string]any{}
var arangoQueryBuilder *arangoQueryBuilder
if hasSBOMSpec.Subject != nil {
if hasSBOMSpec.Subject.Package != nil {
arangoQueryBuilder = setPkgMatchValues(hasSBOMSpec.Subject.Package, values)
arangoQueryBuilder.forOutBound(hasSBOMEdgesStr, "hasSBOM", "pVersion")
setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values)

return getPkgHasSBOMForQuery(ctx, c, arangoQueryBuilder, values)
} else {
arangoQueryBuilder = setArtifactMatchValues(nil, hasSBOMSpec.Subject.Artifact, values)
arangoQueryBuilder.forOutBound(hasSBOMEdgesStr, "hasSBOM", "art")
setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values)

return getArtifactHasSBOMForQuery(ctx, c, arangoQueryBuilder, values)
}
} else {
var combinedHasSBOM []*model.HasSbom

// get packages
arangoQueryBuilder = newForQuery(hasSBOMsStr, "hasSBOM")
setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values)
arangoQueryBuilder.forInBoundWithEdgeCounter(hasSBOMEdgesStr, "pVersion", "hasSBOMEdge", "hasSBOM")
arangoQueryBuilder.prune("hasSBOMEdge", "label", "==", "@label")
arangoQueryBuilder.forInBound(pkgHasVersionStr, "pName", "pVersion")
arangoQueryBuilder.forInBound(pkgHasNameStr, "pNs", "pName")
arangoQueryBuilder.forInBound(pkgHasNamespaceStr, "pType", "pNs")

values["label"] = "package"

pkgHasSBOMs, err := getPkgHasSBOMForQuery(ctx, c, arangoQueryBuilder, values)
if err != nil {
return nil, fmt.Errorf("failed to retrieve package SBOMs with error: %w", err)
}
combinedHasSBOM = append(combinedHasSBOM, pkgHasSBOMs...)

// get artifacts
arangoQueryBuilder = newForQuery(hasSBOMsStr, "hasSBOM")
setHasSBOMMatchValues(arangoQueryBuilder, hasSBOMSpec, values)
arangoQueryBuilder.forInBoundWithEdgeCounter(hasSBOMEdgesStr, "art", "hasSBOMEdge", "hasSBOM")
arangoQueryBuilder.prune("hasSBOMEdge", "label", "==", "@label")
values["label"] = "artifact"

artifactHasSBOMs, err := getArtifactHasSBOMForQuery(ctx, c, arangoQueryBuilder, values)
if err != nil {
return nil, fmt.Errorf("failed to retrieve artifact SBOMs with error: %w", err)
}
combinedHasSBOM = append(combinedHasSBOM, artifactHasSBOMs...)

return combinedHasSBOM, nil
}
}

func getPkgHasSBOMForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.HasSbom, error) {
arangoQueryBuilder.query.WriteString("\n")
arangoQueryBuilder.query.WriteString(`RETURN {
'pkgVersion': {
"type_id": pType._id,
"type": pType.type,
"namespace_id": pNs._id,
"namespace": pNs.namespace,
"name_id": pName._id,
"name": pName.name,
"version_id": pVersion._id,
"version": pVersion.version,
"subpath": pVersion.subpath,
"qualifier_list": pVersion.qualifier_list
},
'hasSBOM_id': hasSBOM._id,
'uri': hasSBOM.uri,
'algorithm': hasSBOM.algorithm,
'digest': hasSBOM.digest,
'downloadLocation': hasSBOM.downloadLocation,
'collector': hasSBOM.collector,
'origin': hasSBOM.origin
}`)

fmt.Println(arangoQueryBuilder.string())

cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "HasSBOM")
if err != nil {
return nil, fmt.Errorf("failed to query for HasSBOM: %w", err)
}
defer cursor.Close()

return getPkgHasSBOM(ctx, cursor)
}

func getArtifactHasSBOMForQuery(ctx context.Context, c *arangoClient, arangoQueryBuilder *arangoQueryBuilder, values map[string]any) ([]*model.HasSbom, error) {
arangoQueryBuilder.query.WriteString("\n")
arangoQueryBuilder.query.WriteString(`RETURN {
'artifact': {
'id': art._id,
'algorithm': art.algorithm,
'digest': art.digest
},
'hasSBOM_id': hasSBOM._id,
'uri': hasSBOM.uri,
'algorithm': hasSBOM.algorithm,
'digest': hasSBOM.digest,
'downloadLocation': hasSBOM.downloadLocation,
'collector': hasSBOM.collector,
'origin': hasSBOM.origin
}`)

fmt.Println(arangoQueryBuilder.string())

cursor, err := executeQueryWithRetry(ctx, c.db, arangoQueryBuilder.string(), values, "HasSBOM")
if err != nil {
return nil, fmt.Errorf("failed to query for HasSBOM: %w", err)
}
defer cursor.Close()

return getArtifactHasSBOM(ctx, cursor)
}

func setHasSBOMMatchValues(arangoQueryBuilder *arangoQueryBuilder, hasSBOMSpec *model.HasSBOMSpec, queryValues map[string]any) {
if hasSBOMSpec.URI != nil {
arangoQueryBuilder.filter("hasSBOM", "uri", "==", "@uri")
queryValues["uri"] = hasSBOMSpec.URI
}
if hasSBOMSpec.Algorithm != nil {
arangoQueryBuilder.filter("hasSBOM", "algorithm", "==", "@algorithm")
queryValues["algorithm"] = hasSBOMSpec.Algorithm
}
if hasSBOMSpec.Digest != nil {
arangoQueryBuilder.filter("hasSBOM", "digest", "==", "@digest")
queryValues["digest"] = hasSBOMSpec.Digest
}
if hasSBOMSpec.DownloadLocation != nil {
arangoQueryBuilder.filter("hasSBOM", "downloadLocation", "==", "@downloadLocation")
queryValues["downloadLocation"] = hasSBOMSpec.DownloadLocation
}
if hasSBOMSpec.Origin != nil {
arangoQueryBuilder.filter("hasSBOM", buildTypeStr, "==", "@"+buildTypeStr)
queryValues[origin] = hasSBOMSpec.Origin
}
if hasSBOMSpec.Collector != nil {
arangoQueryBuilder.filter("hasSBOM", buildTypeStr, "==", "@"+buildTypeStr)
queryValues[collector] = hasSBOMSpec.Collector
}
}

func getHasSBOMQueryValues(pkg *model.PkgInputSpec, artifact *model.ArtifactInputSpec, hasSbom *model.HasSBOMInputSpec) map[string]any {
Expand Down Expand Up @@ -61,7 +203,7 @@ func (c *arangoClient) IngestHasSbom(ctx context.Context, subject model.PackageO
)
LET edgeCollection = (
INSERT { _key: CONCAT("hasSBOMEdges", artifact._key, hasSBOM._key), _from: artifact._id, _to: hasSBOM._id, label : "hasSBOM" } INTO hasSBOMEdges OPTIONS { overwriteMode: "ignore" }
INSERT { _key: CONCAT("hasSBOMEdges", artifact._key, hasSBOM._key), _from: artifact._id, _to: hasSBOM._id, label: "artifact" } INTO hasSBOMEdges OPTIONS { overwriteMode: "ignore" }
)
RETURN {
Expand All @@ -84,7 +226,7 @@ func (c *arangoClient) IngestHasSbom(ctx context.Context, subject model.PackageO
return nil, fmt.Errorf("failed to ingest hasSBOM: %w", err)
}
defer cursor.Close()
hasSBOMList, err := getPkgHasSBOM(ctx, cursor)
hasSBOMList, err := getArtifactHasSBOM(ctx, cursor)
if err != nil {
return nil, fmt.Errorf("failed to get hasSBOM from arango cursor: %w", err)
}
Expand Down Expand Up @@ -129,7 +271,7 @@ func (c *arangoClient) IngestHasSbom(ctx context.Context, subject model.PackageO
)
LET edgeCollection = (
INSERT { _key: CONCAT("hasSBOMEdges", firstPkg.versionDoc._key, hasSBOM._key), _from: firstPkg.version_id, _to: hasSBOM._id, label : "hasSBOM" } INTO hasSBOMEdges OPTIONS { overwriteMode: "ignore" }
INSERT { _key: CONCAT("hasSBOMEdges", firstPkg.versionDoc._key, hasSBOM._key), _from: firstPkg.version_id, _to: hasSBOM._id, label: "package" } INTO hasSBOMEdges OPTIONS { overwriteMode: "ignore" }
)
RETURN {
Expand Down Expand Up @@ -160,7 +302,7 @@ func (c *arangoClient) IngestHasSbom(ctx context.Context, subject model.PackageO
}
defer cursor.Close()

hasSBOMList, err := getArtifactHasSBOM(ctx, cursor)
hasSBOMList, err := getPkgHasSBOM(ctx, cursor)
if err != nil {
return nil, fmt.Errorf("failed to get hasSBOM from arango cursor: %w", err)
}
Expand Down
9 changes: 6 additions & 3 deletions pkg/assembler/backends/arangodb/hasSLSA.go
View file
Expand Up @@ -37,8 +37,10 @@ const (
)

func (c *arangoClient) HasSlsa(ctx context.Context, hasSLSASpec *model.HasSLSASpec) ([]*model.HasSlsa, error) {

// TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval
values := map[string]any{}
arangoQueryBuilder := setArtifactMatchValues(hasSLSASpec.Subject, values)
arangoQueryBuilder := setArtifactMatchValues(nil, hasSLSASpec.Subject, values)
setHasSLSAMatchValues(arangoQueryBuilder, hasSLSASpec, values)

arangoQueryBuilder.query.WriteString("\n")
Expand Down Expand Up @@ -77,7 +79,7 @@ func (c *arangoClient) HasSlsa(ctx context.Context, hasSLSASpec *model.HasSLSASp
func setHasSLSAMatchValues(arangoQueryBuilder *arangoQueryBuilder, hasSLSASpec *model.HasSLSASpec, queryValues map[string]any) {

// currently not filtering on builtFrom (artifacts). Is that a real usecase?
arangoQueryBuilder.ForOutBound(hasSLSASubjectEdgesStr, "hasSLSA", "art")
arangoQueryBuilder.forOutBound(hasSLSASubjectEdgesStr, "hasSLSA", "art")
if hasSLSASpec.BuildType != nil {
arangoQueryBuilder.filter("hasSLSA", buildTypeStr, "==", "@"+buildTypeStr)
queryValues[buildTypeStr] = hasSLSASpec.BuildType
Expand Down Expand Up @@ -107,7 +109,7 @@ func setHasSLSAMatchValues(arangoQueryBuilder *arangoQueryBuilder, hasSLSASpec *
arangoQueryBuilder.filter("hasSLSA", collector, "==", "@"+collector)
queryValues[collector] = hasSLSASpec.Collector
}
arangoQueryBuilder.ForOutBound(hasSLSABuiltByEdgesStr, "build", "hasSLSA")
arangoQueryBuilder.forOutBound(hasSLSABuiltByEdgesStr, "build", "hasSLSA")
if hasSLSASpec.BuiltBy != nil {
arangoQueryBuilder.filter("build", "uri", "==", "@uri")
queryValues["uri"] = hasSLSASpec.BuiltBy.URI
Expand Down Expand Up @@ -372,6 +374,7 @@ func getHasSLSA(c *arangoClient, ctx context.Context, cursor driver.Cursor, buil

var hasSLSAList []*model.HasSlsa
for _, createdValue := range createdValues {

var builtFromArtifacts []*model.Artifact
if val, ok := builtFromMap[artifactKey(createdValue.Subject.Algorithm, createdValue.Subject.Digest)]; ok {
builtFromArtifacts = val
Expand Down
3 changes: 3 additions & 0 deletions pkg/assembler/backends/arangodb/hashEqual.go
View file
Expand Up @@ -25,6 +25,9 @@ import (
)

func (c *arangoClient) HashEqual(ctx context.Context, hashEqualSpec *model.HashEqualSpec) ([]*model.HashEqual, error) {

// TODO (pxp928): Optimize/add other queries based on input and starting node/edge for most efficient retrieval

if hashEqualSpec.Artifacts != nil && len(hashEqualSpec.Artifacts) > 2 {
return nil, fmt.Errorf("cannot specify more than 2 artifacts in HashEquals")
}
Expand Down

0 comments on commit 3f93cd4

Please sign in to comment.