Skip to content

Commit

Permalink
[Ent] Add missing neighbor, node and path query (#1815)
Browse files Browse the repository at this point in the history 
* inital commit for querying by node ID

Signed-off-by: pxp928 <parth.psu@gmail.com>

* update query by node ID for ent

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add new globalID to determine the node type

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add type to create globalIDs

Signed-off-by: pxp928 <parth.psu@gmail.com>

* update node to check nodeType

Signed-off-by: pxp928 <parth.psu@gmail.com>

* implement nodes query for ent

Signed-off-by: pxp928 <parth.psu@gmail.com>

* output globalID during query and implement artifact neighbors

Signed-off-by: pxp928 <parth.psu@gmail.com>

* optimize nodes query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* optimize artifact neighbors query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add builder and license neighbors query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add pkgtype, pkgnamespace neighbor query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add neighbors query for packageNames

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add package neighbor queries

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add source neighbor query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add vulnerability neighbor

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add certifyBad and certifygood neighbor query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add isDep neighbors query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add hashEqual neighbors query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add hasMetadata neighbors query

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add neighbors query for occurs

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add neighbors query for pkgEquals

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add neighbors query for poc

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add neighbors query for sbom

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add neighbors query for hasSourceAt, vulnEqual and vulnMetadata

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add path query for ent

Signed-off-by: pxp928 <parth.psu@gmail.com>

* remove on delete logic in ent for occurence and artifact

Signed-off-by: pxp928 <parth.psu@gmail.com>

* re-add build tag for backend tests

Signed-off-by: pxp928 <parth.psu@gmail.com>

* fix lint issue

Signed-off-by: pxp928 <parth.psu@gmail.com>

* fix arrango edge bug and sort output

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add type filter for certifybad and certifygood

Signed-off-by: pxp928 <parth.psu@gmail.com>

---------

Signed-off-by: pxp928 <parth.psu@gmail.com>
  • Loading branch information
@pxp928
pxp928 committed Apr 5, 2024
1 parent 1381c07 commit 2180123
Show file tree
Hide file tree
Showing 71 changed files with 17,405 additions and 1,503 deletions.
6 changes: 2 additions & 4 deletions internal/testing/backend/main_test.go
View file
Expand Up @@ -43,12 +43,10 @@ var skipMatrix = map[string]map[string]bool{
"TestIsDependency": {memmap: true, redis: true, tikv: true},
// arango errors when ID is not found
"TestOccurrence": {arango: true},
// ent: Path/Nodes/Neighbors not implemented
// keyvalue: path: input: No path found up to specified length
// neighbors: sorting not done, testdata is only in order for arango
"TestPath": {ent: true, memmap: true, redis: true, tikv: true},
"TestNodes": {ent: true},
"TestNeighbors": {ent: true, memmap: true, redis: true, tikv: true},
"TestPath": {memmap: true, redis: true, tikv: true},
"TestNeighbors": {memmap: true, redis: true, tikv: true},
// keyvalue: query on both packages fail
"TestPkgEqual": {memmap: true, redis: true, tikv: true},
// keyvalue: Query_on_OSV_and_novuln_(return_nothing_as_not_valid) fails
Expand Down
43 changes: 22 additions & 21 deletions internal/testing/backend/path_test.go
View file
Expand Up @@ -52,7 +52,7 @@ func TestPath(t *testing.T) {
isDepCall *isDepCall
edges []model.Edge
want []model.Node
wantErr bool
wantPathErr bool
}{
{
name: "certifyVuln - edges not provided",
Expand Down Expand Up @@ -89,10 +89,10 @@ func TestPath(t *testing.T) {
},
},
{
name: "certifyVuln - edges not provided 2",
name: "certifyVuln - edges provided",
inVuln: []*model.VulnerabilityInputSpec{testdata.G1},
inPkg: []*model.PkgInputSpec{testdata.P2},
edges: []model.Edge{model.EdgePackageCertifyVuln, model.EdgeVulnerabilityCertifyVuln},
edges: []model.Edge{model.EdgePackageCertifyVuln, model.EdgeCertifyVulnVulnerability},
certifyVulnCall: &certifyVulnCall{
Pkg: testdata.P2,
Vuln: testdata.G1,
Expand Down Expand Up @@ -126,7 +126,7 @@ func TestPath(t *testing.T) {
name: "certifyVuln - two packages (one vulnerable)",
inVuln: []*model.VulnerabilityInputSpec{testdata.G1},
inPkg: []*model.PkgInputSpec{testdata.P2, testdata.P3},
edges: []model.Edge{model.EdgePackageCertifyVuln, model.EdgeVulnerabilityCertifyVuln},
edges: []model.Edge{model.EdgePackageCertifyVuln, model.EdgeCertifyVulnVulnerability},
certifyVulnTwoPkgsCall: &certifyVulnCall{
Pkg: testdata.P2,
Vuln: testdata.G1,
Expand Down Expand Up @@ -176,8 +176,8 @@ func TestPath(t *testing.T) {
nonVulnPkgID = pkg.PackageVersionID
}
cvID, err := b.IngestCertifyVuln(ctx, model.IDorPkgInput{PackageInput: tt.certifyVulnTwoPkgsCall.Pkg}, model.IDorVulnerabilityInput{VulnerabilityInput: tt.certifyVulnTwoPkgsCall.Vuln}, *tt.certifyVulnTwoPkgsCall.CertifyVuln)
if (err != nil) != tt.wantErr {
t.Fatalf("did not get expected ingest error, want: %v, got: %v", tt.wantErr, err)
if err != nil {
t.Fatalf("did not get expected ingest error, got: %v", err)
}
if err != nil {
return
Expand All @@ -201,8 +201,8 @@ func TestPath(t *testing.T) {
}
}
_, err := b.IngestCertifyVuln(ctx, model.IDorPkgInput{PackageInput: tt.certifyVulnCall.Pkg}, model.IDorVulnerabilityInput{VulnerabilityInput: tt.certifyVulnCall.Vuln}, *tt.certifyVulnCall.CertifyVuln)
if (err != nil) != tt.wantErr {
t.Fatalf("did not get expected ingest error, want: %v, got: %v", tt.wantErr, err)
if err != nil {
t.Fatalf("did not get expected ingest error, got: %v", err)
}
if err != nil {
return
Expand All @@ -215,8 +215,8 @@ func TestPath(t *testing.T) {
}
}
dID, err := b.IngestDependency(ctx, model.IDorPkgInput{PackageInput: tt.isDepCall.P1}, model.IDorPkgInput{PackageInput: tt.isDepCall.P2}, tt.isDepCall.MF, *tt.isDepCall.ID)
if (err != nil) != tt.wantErr {
t.Fatalf("did not get expected ingest error, want: %v, got: %v", tt.wantErr, err)
if err != nil {
t.Fatalf("did not get expected ingest error, got: %v", err)
}
if err != nil {
return
Expand All @@ -229,8 +229,8 @@ func TestPath(t *testing.T) {
stopID = found[0].DependencyPackage.Namespaces[0].Names[0].ID
}
got, err := b.Path(ctx, startID, stopID, 5, tt.edges)
if (err != nil) != tt.wantErr {
t.Errorf("node query error = %v, wantErr %v", err, tt.wantErr)
if (err != nil) != tt.wantPathErr {
t.Errorf("node query error = %v, wantErr %v", err, tt.wantPathErr)
return
}
if diff := cmp.Diff(tt.want, got, commonOpts); diff != "" {
Expand Down Expand Up @@ -1266,8 +1266,8 @@ func TestNeighbors(t *testing.T) {
},
queryPkgNameID: true,
want: []model.Node{
testdata.P2out,
testdata.P1out,
testdata.P2out,
&model.Package{
Type: "pypi",
Namespaces: []*model.PackageNamespace{{
Expand Down Expand Up @@ -1432,8 +1432,8 @@ func TestNeighbors(t *testing.T) {
},
queryPkgNameID: true,
want: []model.Node{
testdata.P2out,
testdata.P1out,
testdata.P2out,
&model.Package{
Type: "pypi",
Namespaces: []*model.PackageNamespace{{
Expand Down Expand Up @@ -2077,8 +2077,8 @@ func TestNeighbors(t *testing.T) {
},
queryPkgNameID: true,
want: []model.Node{
testdata.P2out,
testdata.P1out,
testdata.P2out,
&model.Package{
Type: "pypi",
Namespaces: []*model.PackageNamespace{{
Expand Down Expand Up @@ -2424,8 +2424,8 @@ func TestNeighbors(t *testing.T) {
},
queryPkgNameID: true,
want: []model.Node{
testdata.P2out,
testdata.P1out,
testdata.P2out,
&model.Package{
Type: "pypi",
Namespaces: []*model.PackageNamespace{{
Expand Down Expand Up @@ -2483,13 +2483,14 @@ func TestNeighbors(t *testing.T) {
}},
},
&model.HasSourceAt{
Package: testdata.P2out,
Package: testdata.P2outName,
Source: testdata.S1out,
},
&model.HasSourceAt{
Package: testdata.P2outName,
Package: testdata.P2out,
Source: testdata.S1out,
}},
},
},
}, {
name: "hasSourceAt - hasSourceAtID - pkgName",
inPkg: []*model.PkgInputSpec{testdata.P2},
Expand Down Expand Up @@ -2546,8 +2547,8 @@ func TestNeighbors(t *testing.T) {
},
queryPkgNameID: true,
want: []model.Node{
testdata.P2out,
testdata.P1out,
testdata.P2out,
&model.Package{
Type: "pypi",
Namespaces: []*model.PackageNamespace{{
Expand Down Expand Up @@ -2824,8 +2825,8 @@ func TestNeighbors(t *testing.T) {
},
queryPkgNameID: true,
want: []model.Node{
testdata.P2out,
testdata.P1out,
testdata.P2out,
&model.Package{
Type: "pypi",
Namespaces: []*model.PackageNamespace{{
Expand Down
2 changes: 1 addition & 1 deletion pkg/assembler/backends/arangodb/edgeCollections.go
View file
Expand Up @@ -216,7 +216,7 @@ var mapEdgeToArangoEdgeCollection = map[model.Edge][]string{
model.EdgeCertifyVexStatementPackage: {certifyVexPkgEdgesStr},
model.EdgeCertifyVexStatementVulnerability: {certifyVexVulnEdgesStr},
model.EdgeCertifyVulnPackage: {certifyVulnPkgEdgesStr},
model.EdgeCertifyVulnVulnerability: {certifyVulnsStr},
model.EdgeCertifyVulnVulnerability: {certifyVulnEdgesStr},
model.EdgeHashEqualArtifact: {hashEqualArtEdgesStr},
model.EdgeHasMetadataArtifact: {hasMetadataArtEdgesStr},
model.EdgeHasMetadataPackage: {hasMetadataPkgVersionEdgesStr, hasMetadataPkgNameEdgesStr},
Expand Down
1 change: 1 addition & 0 deletions pkg/assembler/backends/arangodb/pkg.go
View file
Expand Up @@ -1167,6 +1167,7 @@ func (c *arangoClient) packageNameNeighbors(ctx context.Context, nodeID string,
if err != nil {
return out, fmt.Errorf("failed to get neighbors for node ID: %s from arango cursor with error: %w", nodeID, err)
}
sort.Strings(foundIDs)
out = append(out, foundIDs...)
}
if allowedEdges[model.EdgePackageNamePackageNamespace] {
Expand Down
2 changes: 2 additions & 0 deletions pkg/assembler/backends/arangodb/src.go
View file
Expand Up @@ -18,6 +18,7 @@ package arangodb
import (
"context"
"fmt"
"sort"
"strings"

"github.com/99designs/gqlgen/graphql"
Expand Down Expand Up @@ -838,6 +839,7 @@ func (c *arangoClient) srcNameNeighbors(ctx context.Context, nodeID string, allo
if err != nil {
return out, fmt.Errorf("failed to get neighbors for node ID: %s from arango cursor with error: %w", nodeID, err)
}
sort.Strings(foundIDs)
out = append(out, foundIDs...)
}
if allowedEdges[model.EdgeSourceCertifyScorecard] {
Expand Down

0 comments on commit 2180123

Please sign in to comment.