Skip to content

Commit

Permalink
update API for bulk ingestion for CertifyBad/CertifyGood, add missing…
Browse files Browse the repository at this point in the history 
… unit tests, update collections on arango (#1115)

* add query for hashEqual

Signed-off-by: pxp928 <parth.psu@gmail.com>

* update bulk assembler

Signed-off-by: pxp928 <parth.psu@gmail.com>

* remove src and package type pre-ingest

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add bulk certifybad/certifygood for inmem

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add missing unit test for inmem and add certifyBad ingest

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add subject and artifact collections for verbs

Signed-off-by: pxp928 <parth.psu@gmail.com>

* add bulk certifyBad ingestion to test data ingestion

Signed-off-by: pxp928 <parth.psu@gmail.com>

* make pkgMatchType required

Signed-off-by: pxp928 <parth.psu@gmail.com>

* fix guacanalytics test

Signed-off-by: pxp928 <parth.psu@gmail.com>

---------

Signed-off-by: pxp928 <parth.psu@gmail.com>
  • Loading branch information
@pxp928
pxp928 committed Aug 1, 2023
1 parent 069edcb commit 15ad9f9
Show file tree
Hide file tree
Showing 42 changed files with 4,760 additions and 380 deletions.
182 changes: 172 additions & 10 deletions cmd/guacgql/cmd/ingest.go
View file
Expand Up @@ -49,6 +49,7 @@ func ingestData(port int) {
ingestVulnerability(ctx, gqlclient)
ingestPkgEqual(ctx, gqlclient)
ingestCertifyBad(ctx, gqlclient)
ingestCertifyBads(ctx, gqlclient)
ingestCertifyGood(ctx, gqlclient)
ingestHashEqual(ctx, gqlclient)
ingestHasSBOM(ctx, gqlclient)
Expand Down Expand Up @@ -741,7 +742,7 @@ func ingestCertifyBad(ctx context.Context, client graphql.Client) {
ingestCertifyBad := []struct {
name string
pkg *model.PkgInputSpec
pkgMatchType *model.MatchFlags
pkgMatchType model.MatchFlags
source *model.SourceInputSpec
artifact *model.ArtifactInputSpec
certifyBad model.CertifyBadInputSpec
Expand All @@ -755,7 +756,7 @@ func ingestCertifyBad(ctx context.Context, client graphql.Client) {
Version: &opensslVersion,
Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}},
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeSpecificVersion,
},
certifyBad: model.CertifyBadInputSpec{
Expand All @@ -771,7 +772,7 @@ func ingestCertifyBad(ctx context.Context, client graphql.Client) {
Namespace: &djangoNameSpace,
Name: "django",
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeAllVersions,
},
certifyBad: model.CertifyBadInputSpec{
Expand Down Expand Up @@ -815,7 +816,7 @@ func ingestCertifyBad(ctx context.Context, client graphql.Client) {
Version: &opensslVersion,
Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}},
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeSpecificVersion,
},
certifyBad: model.CertifyBadInputSpec{
Expand All @@ -831,7 +832,7 @@ func ingestCertifyBad(ctx context.Context, client graphql.Client) {
Namespace: &djangoNameSpace,
Name: "django",
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeAllVersions,
},
certifyBad: model.CertifyBadInputSpec{
Expand Down Expand Up @@ -895,6 +896,167 @@ func ingestCertifyBad(ctx context.Context, client graphql.Client) {
}
}

func ingestCertifyBads(ctx context.Context, client graphql.Client) {
logger := logging.FromContext(ctx)
opensslNs := "openssl.org"
opensslVersion := "3.0.3"
djangoNameSpace := ""
sourceTag := "v0.0.1"
ingestCertifyBad := []struct {
name string
pkg []model.PkgInputSpec
pkgMatchType model.MatchFlags
source []model.SourceInputSpec
artifact []model.ArtifactInputSpec
certifyBad []model.CertifyBadInputSpec
}{
{
name: "this package as this specific version has a malware",
pkg: []model.PkgInputSpec{
{
Type: "conan",
Namespace: &opensslNs,
Name: "openssl",
Version: &opensslVersion,
Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}},
},
{
Type: "conan",
Namespace: &opensslNs,
Name: "openssl",
Version: &opensslVersion,
Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}},
},
},
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeSpecificVersion,
},
certifyBad: []model.CertifyBadInputSpec{
{
Justification: "this package as this specific version has a malware",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
{
Justification: "this package as this specific version has a malware",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
},
},
{
name: "this package (all versions) is a known typo-squat",
pkg: []model.PkgInputSpec{
{
Type: "pypi",
Namespace: &djangoNameSpace,
Name: "django",
},
{
Type: "pypi",
Namespace: &djangoNameSpace,
Name: "django",
},
},
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeAllVersions,
},
certifyBad: []model.CertifyBadInputSpec{
{
Justification: "this package (all versions) is a known typo-squat",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
{
Justification: "this package (all versions) is a known typo-squat",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
},
},
{
name: "this source repo is owned by a known attacker",
source: []model.SourceInputSpec{
{
Type: "git",
Namespace: "github",
Name: "github.com/guacsec/guac",
Tag: &sourceTag,
},
{
Type: "git",
Namespace: "github",
Name: "github.com/guacsec/guac",
Tag: &sourceTag,
},
},
certifyBad: []model.CertifyBadInputSpec{
{
Justification: "this source repo is owned by a known attacker",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
{
Justification: "this source repo is owned by a known attacker",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
},
},
{
name: "artifact is associated with a malware package",
artifact: []model.ArtifactInputSpec{
{
Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf",
Algorithm: "sha256",
},
{
Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf",
Algorithm: "sha256",
},
},
certifyBad: []model.CertifyBadInputSpec{
{
Justification: "this artifact is associated with a malware package",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
{
Justification: "this artifact is associated with a malware package",
Origin: "Demo ingestion",
Collector: "Demo ingestion",
},
},
},
}
for _, ingest := range ingestCertifyBad {
if ingest.pkg != nil {
if _, err := model.IngestPackages(ctx, client, ingest.pkg); err != nil {
logger.Errorf("Error in ingesting package: %v\n", err)
}
if _, err := model.CertifyBadPkgs(ctx, client, ingest.pkg, ingest.pkgMatchType, ingest.certifyBad); err != nil {
logger.Errorf("Error in ingesting: %v\n", err)
}
} else if ingest.source != nil {
if _, err := model.IngestSources(ctx, client, ingest.source); err != nil {
logger.Errorf("Error in ingesting source: %v\n", err)
}
if _, err := model.CertifyBadSrcs(ctx, client, ingest.source, ingest.certifyBad); err != nil {
logger.Errorf("Error in ingesting: %v\n", err)
}
} else if ingest.artifact != nil {
if _, err := model.IngestArtifacts(ctx, client, ingest.artifact); err != nil {
logger.Errorf("Error in ingesting artifact: %v\n", err)
}
if _, err := model.CertifyBadArtifacts(ctx, client, ingest.artifact, ingest.certifyBad); err != nil {
logger.Errorf("Error in ingesting: %v\n", err)
}
} else {
fmt.Printf("input missing for cve, osv or ghsa")
}
}
}

func ingestCertifyGood(ctx context.Context, client graphql.Client) {
logger := logging.FromContext(ctx)
opensslNs := "openssl.org"
Expand All @@ -904,7 +1066,7 @@ func ingestCertifyGood(ctx context.Context, client graphql.Client) {
ingestCertifyGood := []struct {
name string
pkg *model.PkgInputSpec
pkgMatchType *model.MatchFlags
pkgMatchType model.MatchFlags
source *model.SourceInputSpec
artifact *model.ArtifactInputSpec
certifyGood model.CertifyGoodInputSpec
Expand All @@ -918,7 +1080,7 @@ func ingestCertifyGood(ctx context.Context, client graphql.Client) {
Version: &opensslVersion,
Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}},
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeSpecificVersion,
},
certifyGood: model.CertifyGoodInputSpec{
Expand All @@ -934,7 +1096,7 @@ func ingestCertifyGood(ctx context.Context, client graphql.Client) {
Namespace: &djangoNameSpace,
Name: "django",
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeAllVersions,
},
certifyGood: model.CertifyGoodInputSpec{
Expand Down Expand Up @@ -978,7 +1140,7 @@ func ingestCertifyGood(ctx context.Context, client graphql.Client) {
Version: &opensslVersion,
Qualifiers: []model.PackageQualifierInputSpec{{Key: "user", Value: "bincrafters"}, {Key: "channel", Value: "stable"}},
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeSpecificVersion,
},
certifyGood: model.CertifyGoodInputSpec{
Expand All @@ -994,7 +1156,7 @@ func ingestCertifyGood(ctx context.Context, client graphql.Client) {
Namespace: &djangoNameSpace,
Name: "django",
},
pkgMatchType: &model.MatchFlags{
pkgMatchType: model.MatchFlags{
Pkg: model.PkgMatchTypeAllVersions,
},
certifyGood: model.CertifyGoodInputSpec{
Expand Down

0 comments on commit 15ad9f9

Please sign in to comment.