ENT - bulk ingestion and update to use IDorInputSpec (#1732)

* test updates on ent Signed-off-by: pxp928 <parth.psu@gmail.com> * update default postgres local address Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent IDs to be UUID Signed-off-by: pxp928 <parth.psu@gmail.com> * rebase on main Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent artifact to generate uuid and bulk ingest Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent builder to generate uuid and bulk ingest Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent schema to remove type and namespace from pkg and src. Update ent package Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent backend license, source and vulnerability Signed-off-by: pxp928 <parth.psu@gmail.com> * update part of certify for ent Signed-off-by: pxp928 <parth.psu@gmail.com> * update certifygood and bad for ent Signed-off-by: pxp928 <parth.psu@gmail.com> * consolidate guackeys with generics Signed-off-by: pxp928 <parth.psu@gmail.com> * update guackeys for arango Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent certifyLegal bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent certifvex bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent certifvuln bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent dependencies bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent hashEqual bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent hasMetadata bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * comment out search by nodeID for ent Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent isOccurrence bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent hasSourceAt bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent pkgEqual bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent pointOfContact bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent hasSBOM bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent hasSBOM bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent certifyscorecard bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent slsa bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent vulnEqual bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update ent vulnMetadata bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * comment out unused code for now Signed-off-by: pxp928 <parth.psu@gmail.com> * go mod tidy Signed-off-by: pxp928 <parth.psu@gmail.com> * fix reference issues related to go1.21 Signed-off-by: pxp928 <parth.psu@gmail.com> * fix dependencies ingest pkgName and pkgVersion ID Signed-off-by: pxp928 <parth.psu@gmail.com> * fix package query Signed-off-by: pxp928 <parth.psu@gmail.com> * change to uuidv7, hashes for included in hasSBOM, and create hasSBOM ID based on inputs to fix bulk ingestion bug Signed-off-by: pxp928 <parth.psu@gmail.com> * remove unused isvuln, add proper comments to schema Signed-off-by: pxp928 <parth.psu@gmail.com> * update certifylegal to generate ID to not have violates foreign key constraint issue when ingesting the same document again Signed-off-by: pxp928 <parth.psu@gmail.com> * update certifyLegal and hashEqual to generate IDs Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor certifyGood and certifyBad ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor certifyVex ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor certifyVuln ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor isDependency ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor hasMetadata ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor artifact and builder ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor license ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor occurrence ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor package, source, and vuln ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor pkgEqual ingestion and generate pkgEqual ID Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor poc ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor hasSBOM ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor scorecard ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor slsa ingestion and generate slsa ID Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor hasSourceAt ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor vulnEqual and generate ID Signed-off-by: pxp928 <parth.psu@gmail.com> * refactor vulnMetadata ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * query if ID is not present for nouns Signed-off-by: pxp928 <parth.psu@gmail.com> * re-add build tag for backend tests Signed-off-by: pxp928 <parth.psu@gmail.com> * fix package upsert bug Signed-off-by: pxp928 <parth.psu@gmail.com> * change OnConflict to ignore and ensure ID are present where needed Signed-off-by: pxp928 <parth.psu@gmail.com> * change from ignore to DoNothing Signed-off-by: pxp928 <parth.psu@gmail.com> * change to ignore when returning license, else use DoNothing for bulk ingestion Signed-off-by: pxp928 <parth.psu@gmail.com> * update vulnEqual index to be unique Signed-off-by: pxp928 <parth.psu@gmail.com> * add check hash for scorecard to ensure unique input Signed-off-by: pxp928 <parth.psu@gmail.com> * fix issues with de-referencing Signed-off-by: pxp928 <parth.psu@gmail.com> * fix issue with package version query Signed-off-by: pxp928 <parth.psu@gmail.com> * reorder package query to start with version Signed-off-by: pxp928 <parth.psu@gmail.com> * change vulnEqual to map to vuln IDs instead of a many to many edge relationship which caused an error Signed-off-by: pxp928 <parth.psu@gmail.com> * change pkgEqual and hashEqual to map to IDs instead of a many to many edge relationship which caused an error Signed-off-by: pxp928 <parth.psu@gmail.com> * fix assembler tests Signed-off-by: pxp928 <parth.psu@gmail.com> * fix integration tests Signed-off-by: pxp928 <parth.psu@gmail.com> * readd skip tests Signed-off-by: pxp928 <parth.psu@gmail.com> * change err variable to txErr for transactino error for readability Signed-off-by: pxp928 <parth.psu@gmail.com> * fix certifyBad bug Signed-off-by: pxp928 <parth.psu@gmail.com> * add helper function to generate UUID and add comment Signed-off-by: pxp928 <parth.psu@gmail.com> --------- Signed-off-by: pxp928 <parth.psu@gmail.com>
guacsec · Mar 1, 2024 · d5feab1 · d5feab1
1 parent e9e3551
commit d5feab1
Show file tree

Hide file tree

Showing 331 changed files with 24,259 additions and 39,615 deletions.
diff --git a/go.mod b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.6.0
-	github.com/google/uuid v1.6.0 // indirect
+	github.com/google/uuid v1.6.0
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/gorilla/handlers v1.5.2 // indirect
@@ -36,7 +36,7 @@ require (
 	golang.org/x/crypto v0.18.0 // indirect
 	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/oauth2 v0.16.0
-	golang.org/x/sync v0.6.0
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect

diff --git a/internal/testing/backend/helpers_test.go b/internal/testing/backend/helpers_test.go
@@ -83,13 +83,31 @@ func depTypeCmp(a, b model.DependencyType) bool {
 }
 
 func certifyVexLess(e1, e2 *model.CertifyVEXStatement) bool {
-	if e1.Vulnerability.VulnerabilityIDs[0].VulnerabilityID != e2.Vulnerability.VulnerabilityIDs[0].VulnerabilityID {
-		return e1.Vulnerability.VulnerabilityIDs[0].VulnerabilityID < e2.Vulnerability.VulnerabilityIDs[0].VulnerabilityID
-	}
+
 	if e1.VexJustification != e2.VexJustification {
 		return e1.VexJustification < e2.VexJustification
 	}
-	return false
+
+	ap, oka := e1.Subject.(*model.Package)
+	bp, okb := e2.Subject.(*model.Package)
+	if oka && !okb {
+		return false
+	}
+	if okb && !oka {
+		return true
+	}
+	if oka && okb {
+		return cmpPkg(ap, bp) < 0
+	}
+
+	if e1.Vulnerability.VulnerabilityIDs[0].VulnerabilityID != e2.Vulnerability.VulnerabilityIDs[0].VulnerabilityID {
+		return e1.Vulnerability.VulnerabilityIDs[0].VulnerabilityID < e2.Vulnerability.VulnerabilityIDs[0].VulnerabilityID
+	}
+
+	aa := e1.Subject.(*model.Artifact)
+	ba := e2.Subject.(*model.Artifact)
+	return cmpArt(aa, ba) < 0
+
 }
 
 func vulnerabilityLess(e1, e2 *model.Vulnerability) bool {

diff --git a/internal/testing/backend/main_test.go b/internal/testing/backend/main_test.go
@@ -65,7 +65,8 @@ var skipMatrix = map[string]map[string]bool{
 	// keyvalue: failing on dep package querying
 	"TestIsDependency": {ent: true, memmap: true, redis: true, tikv: true},
 	// arango errors when ID is not found
-	"TestOccurrence": {arango: true},
+	// ent errors when ID is not found
+	"TestOccurrence": {ent: true, arango: true},
 	// ent: Path/Nodes/Neighbors not implemented
 	// keyvalue: path: input: No path found up to specified length
 	// neighbors: sorting not done, testdata is only in order for arango
@@ -86,10 +87,13 @@ var skipMatrix = map[string]map[string]bool{
 	// arango: errors when ID is not found
 	// ent: query by novuln fails, query by ID fails
 	"TestVulnerability": {arango: true, ent: true},
+	// redis order issues
+	"TestVEX": {redis: true},
+	// redis order issues
+	"TestVEXBulkIngest": {redis: true},
 	// ent: query by id fails, Query_greater_than_-_no_score_value fails
 	"TestIngestVulnMetadata": {ent: true},
-
-	"TestFindSoftware": {ent: true, redis: true, arango: true},
+	"TestFindSoftware":       {ent: true, redis: true, arango: true},
 }
 
 type backend interface {