Skip to content

Commit

Permalink
Ingestor/Assembler and SPDX Parser for Legal information. (#1244)
Browse files Browse the repository at this point in the history 
* Ingestor/Assembler and SPDX Parser for Legal information.

Signed-off-by: Jeff Mendoza <jlm@jlm.name>

* Add license unit test to assembler.

Signed-off-by: Jeff Mendoza <jlm@jlm.name>

---------

Signed-off-by: Jeff Mendoza <jlm@jlm.name>
  • Loading branch information
@jeffmendoza
jeffmendoza committed Sep 14, 2023
1 parent 2b44e51 commit ebd91bb
Show file tree
Hide file tree
Showing 9 changed files with 751 additions and 14 deletions.
77 changes: 77 additions & 0 deletions internal/testing/testdata/testdata.go
View file
Expand Up @@ -734,10 +734,77 @@ var (
},
}

SpdxCertifyLegal = []assembler.CertifyLegalIngest{
{
Pkg: baselayoutPack,
Declared: []model.LicenseInputSpec{
{
Name: "GPL-2.0-only",
ListVersion: ptrfrom.String("3.18"),
},
},
Discovered: []model.LicenseInputSpec{
{
Name: "GPL-2.0-only",
ListVersion: ptrfrom.String("3.18"),
},
},
CertifyLegal: &model.CertifyLegalInputSpec{
DeclaredLicense: "GPL-2.0-only",
DiscoveredLicense: "GPL-2.0-only",
Justification: "Found in SPDX document.",
TimeScanned: parseRfc3339("2022-09-24T17:27:55.556104Z"),
},
},
{
Pkg: baselayoutdataPack,
Declared: []model.LicenseInputSpec{
{
Name: "GPL-2.0-only",
ListVersion: ptrfrom.String("3.18"),
},
},
Discovered: []model.LicenseInputSpec{
{
Name: "GPL-2.0-only",
ListVersion: ptrfrom.String("3.18"),
},
},
CertifyLegal: &model.CertifyLegalInputSpec{
DeclaredLicense: "GPL-2.0-only",
DiscoveredLicense: "GPL-2.0-only",
Justification: "Found in SPDX document.",
TimeScanned: parseRfc3339("2022-09-24T17:27:55.556104Z"),
},
},
{
Pkg: keysPack,
Declared: []model.LicenseInputSpec{
{
Name: "MIT",
ListVersion: ptrfrom.String("3.18"),
},
},
Discovered: []model.LicenseInputSpec{
{
Name: "MIT",
ListVersion: ptrfrom.String("3.18"),
},
},
CertifyLegal: &model.CertifyLegalInputSpec{
DeclaredLicense: "MIT",
DiscoveredLicense: "MIT",
Justification: "Found in SPDX document.",
TimeScanned: parseRfc3339("2022-09-24T17:27:55.556104Z"),
},
},
}

SpdxIngestionPredicates = assembler.IngestPredicates{
IsDependency: SpdxDeps,
IsOccurrence: SpdxOccurences,
HasSBOM: SpdxHasSBOM,
CertifyLegal: SpdxCertifyLegal,
}

// CycloneDX Testdata
Expand Down Expand Up @@ -2613,6 +2680,8 @@ var IngestPredicatesCmpOpts = []cmp.Option{
cmpopts.SortSlices(packageQualifierInputSpecLess),
cmpopts.SortSlices(psaInputSpecLess),
cmpopts.SortSlices(slsaPredicateInputSpecLess),
cmpopts.SortSlices(certifyLegalInputSpecLess),
cmpopts.SortSlices(licenseInputSpecLess),
}

func certifyScorecardLess(e1, e2 assembler.CertifyScorecardIngest) bool {
Expand All @@ -2639,6 +2708,14 @@ func slsaPredicateInputSpecLess(e1, e2 model.SLSAPredicateInputSpec) bool {
return gLess(e1, e2)
}

func certifyLegalInputSpecLess(e1, e2 assembler.CertifyLegalIngest) bool {
return gLess(e1, e2)
}

func licenseInputSpecLess(e1, e2 generated.LicenseInputSpec) bool {
return gLess(e1, e2)
}

func gLess(e1, e2 any) bool {
s1, _ := json.Marshal(e1)
s2, _ := json.Marshal(e2)
Expand Down
57 changes: 57 additions & 0 deletions pkg/assembler/assembler.go
View file
Expand Up @@ -47,6 +47,7 @@ type IngestPredicates struct {
PointOfContact []PointOfContactIngest `json:"contact,omitempty"`
VulnMetadata []VulnMetadataIngest `json:"vulnMetadata,omitempty"`
HasMetadata []HasMetadataIngest `json:"hasMetadata,omitempty"`
CertifyLegal []CertifyLegalIngest `json:"certifyLegal,omitempty"`
}

type CertifyScorecardIngest struct {
Expand Down Expand Up @@ -184,6 +185,16 @@ type PkgEqualIngest struct {
PkgEqual *generated.PkgEqualInputSpec `json:"pkgEqual,omitempty"`
}

type CertifyLegalIngest struct {
Pkg *generated.PkgInputSpec `json:"pkg,omitempty"`
Src *generated.SourceInputSpec `json:"src,omitempty"`

Declared []generated.LicenseInputSpec `json:"declared,omitempty"`
Discovered []generated.LicenseInputSpec `json:"discovered,omitempty"`

CertifyLegal *generated.CertifyLegalInputSpec `json:"certifyLegal,omitempty"`
}

func (i IngestPredicates) GetPackages(ctx context.Context) []*generated.PkgInputSpec {
packageMap := make(map[string]*generated.PkgInputSpec)
for _, dep := range i.IsDependency {
Expand Down Expand Up @@ -286,6 +297,14 @@ func (i IngestPredicates) GetPackages(ctx context.Context) []*generated.PkgInput
}
}
}
for _, cl := range i.CertifyLegal {
if cl.Pkg != nil {
pkgPurl := helpers.PkgInputSpecToPurl(cl.Pkg)
if _, ok := packageMap[pkgPurl]; !ok {
packageMap[pkgPurl] = cl.Pkg
}
}
}
packages := make([]*generated.PkgInputSpec, 0, len(packageMap))

for _, pkg := range packageMap {
Expand Down Expand Up @@ -352,6 +371,14 @@ func (i IngestPredicates) GetSources(ctx context.Context) []*generated.SourceInp
}
}
}
for _, cl := range i.CertifyLegal {
if cl.Src != nil {
sourceString := concatenateSourceInput(cl.Src)
if _, ok := sourceMap[sourceString]; !ok {
sourceMap[sourceString] = cl.Src
}
}
}
sources := make([]*generated.SourceInputSpec, 0, len(sourceMap))

for _, source := range sourceMap {
Expand Down Expand Up @@ -527,6 +554,29 @@ func (i IngestPredicates) GetVulnerabilities(ctx context.Context) []*generated.V
return vulns
}

func (i IngestPredicates) GetLicenses(ctx context.Context) []generated.LicenseInputSpec {
licenseMap := make(map[string]*generated.LicenseInputSpec)
for _, cl := range i.CertifyLegal {
for i := range cl.Declared {
k := licenseKey(&cl.Declared[i])
if _, ok := licenseMap[k]; !ok {
licenseMap[k] = &cl.Declared[i]
}
}
for i := range cl.Discovered {
k := licenseKey(&cl.Discovered[i])
if _, ok := licenseMap[k]; !ok {
licenseMap[k] = &cl.Discovered[i]
}
}
}
licenses := make([]generated.LicenseInputSpec, 0, len(licenseMap))
for _, license := range licenseMap {
licenses = append(licenses, *license)
}
return licenses
}

func concatenateSourceInput(source *generated.SourceInputSpec) string {
var sourceElements []string
sourceElements = append(sourceElements, source.Type, source.Namespace, source.Name)
Expand All @@ -539,5 +589,12 @@ func concatenateSourceInput(source *generated.SourceInputSpec) string {
return strings.Join(sourceElements, "/")
}

func licenseKey(l *generated.LicenseInputSpec) string {
if l.ListVersion != nil && *l.ListVersion != "" {
return strings.Join([]string{l.Name, *l.ListVersion}, ":")
}
return l.Name
}

// AssemblerInput represents the inputs to add to the graph
type AssemblerInput = IngestPredicates
75 changes: 75 additions & 0 deletions pkg/assembler/assembler_test.go
View file
Expand Up @@ -84,6 +84,7 @@ func TestIngestPredicates(t *testing.T) {
wantMaterials []generated.ArtifactInputSpec
wantBuilder []*generated.BuilderInputSpec
wantVuln []*generated.VulnerabilityInputSpec
wantLicense []generated.LicenseInputSpec
}{{
name: "get nouns",
field: IngestPredicates{
Expand Down Expand Up @@ -627,6 +628,60 @@ func TestIngestPredicates(t *testing.T) {
},
},
},
CertifyLegal: []CertifyLegalIngest{
{
Pkg: maven,
Declared: []generated.LicenseInputSpec{
{
Name: "asdf",
ListVersion: ptrfrom.String("1.2.3"),
},
},
Discovered: []generated.LicenseInputSpec{
{
Name: "asdf",
ListVersion: ptrfrom.String("1.2.3"),
},
{
Name: "qwer",
ListVersion: ptrfrom.String("1.2.3"),
},
},
CertifyLegal: &generated.CertifyLegalInputSpec{
DeclaredLicense: "asdf",
DiscoveredLicense: "asdf AND qwer",
Attribution: "Copyright Jeff",
Justification: "Scanner foo",
TimeScanned: toTime("2022-10-06"),
},
},
{
Pkg: openSSL,
Declared: []generated.LicenseInputSpec{
{
Name: "qwer",
ListVersion: ptrfrom.String("1.2.3"),
},
},
Discovered: []generated.LicenseInputSpec{
{
Name: "qwer",
ListVersion: ptrfrom.String("1.2.3"),
},
{
Name: "LicenseRef-123",
Inline: ptrfrom.String("This is the license text."),
},
},
CertifyLegal: &generated.CertifyLegalInputSpec{
DeclaredLicense: "qwer",
DiscoveredLicense: "qwer AND LicenseRef-123",
Attribution: "Copyright Jeff",
Justification: "Scanner foo",
TimeScanned: toTime("2022-10-06"),
},
},
},
},
wantPkg: []*generated.PkgInputSpec{rootFilePack, maven, openSSL, openSSLWithQualifier, topLevelPack, baselayoutPack, baselayoutdataPack, worldFilePack},
wantSource: []*generated.SourceInputSpec{k8sSource},
Expand Down Expand Up @@ -733,6 +788,20 @@ func TestIngestPredicates(t *testing.T) {
VulnerabilityID: "ghsa-p6xc-xr62-6r2g",
},
},
wantLicense: []generated.LicenseInputSpec{
{
Name: "qwer",
ListVersion: ptrfrom.String("1.2.3"),
},
{
Name: "asdf",
ListVersion: ptrfrom.String("1.2.3"),
},
{
Name: "LicenseRef-123",
Inline: ptrfrom.String("This is the license text."),
},
},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down Expand Up @@ -776,6 +845,12 @@ func TestIngestPredicates(t *testing.T) {
if diff := cmp.Diff(tt.wantVuln, gotVulns, cmpopts.SortSlices(vulnSort)); diff != "" {
t.Errorf("Unexpected gotVulns results. (-want +got):\n%s", diff)
}

gotLicenses := i.GetLicenses(ctx)
licSort := func(a, b generated.LicenseInputSpec) bool { return a.Name < b.Name }
if diff := cmp.Diff(tt.wantLicense, gotLicenses, cmpopts.SortSlices(licSort)); diff != "" {
t.Errorf("Unexpected GetLicenses results. (-want +got):\n%s", diff)
}
})
}
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/inmem/certifyLegal.go
View file
Expand Up @@ -131,7 +131,7 @@ func (c *demoClient) ingestCertifyLegal(ctx context.Context, subject model.Packa
for _, lis := range declaredLicenses {
l, ok := c.licenses[licenseKey(lis.Name, lis.ListVersion)]
if !ok {
return nil, gqlerror.Errorf("%v :: License not found %s", funcName, lis.Name)
return nil, gqlerror.Errorf("%v :: License not found %q", funcName, licenseKey(lis.Name, lis.ListVersion))
}
dec = append(dec, l.id)
}
Expand All @@ -140,7 +140,7 @@ func (c *demoClient) ingestCertifyLegal(ctx context.Context, subject model.Packa
for _, lis := range discoveredLicenses {
l, ok := c.licenses[licenseKey(lis.Name, lis.ListVersion)]
if !ok {
return nil, gqlerror.Errorf("%v :: License not found %s", funcName, lis.Name)
return nil, gqlerror.Errorf("%v :: License not found %q", funcName, licenseKey(lis.Name, lis.ListVersion))
}
dis = append(dis, l.id)
}
Expand Down
36 changes: 36 additions & 0 deletions pkg/assembler/clients/helpers/assembler.go
View file
Expand Up @@ -76,6 +76,14 @@ func GetAssembler(ctx context.Context, gqlclient graphql.Client) func([]assemble
}
}

licenses := p.GetLicenses(ctx)
logger.Infof("assembling License: %v", len(licenses))
for _, v := range licenses {
if err := ingestLicense(ctx, gqlclient, &v); err != nil {
return err
}
}

logger.Infof("assembling CertifyScorecard: %v", len(p.CertifyScorecard))
for _, v := range p.CertifyScorecard {
if err := ingestCertifyScorecard(ctx, gqlclient, v); err != nil {
Expand Down Expand Up @@ -187,6 +195,13 @@ func GetAssembler(ctx context.Context, gqlclient graphql.Client) func([]assemble
return err
}
}

logger.Infof("assembling CertifyLegal : %v", len(p.CertifyLegal))
for _, cl := range p.CertifyLegal {
if err := ingestCertifyLegal(ctx, gqlclient, cl); err != nil {
return err
}
}
}
return nil
}
Expand Down Expand Up @@ -217,6 +232,11 @@ func ingestVulnerability(ctx context.Context, client graphql.Client, v *model.Vu
return err
}

func ingestLicense(ctx context.Context, client graphql.Client, l *model.LicenseInputSpec) error {
_, err := model.IngestLicense(ctx, client, *l)
return err
}

func ingestCertifyScorecard(ctx context.Context, client graphql.Client, v assembler.CertifyScorecardIngest) error {
_, err := model.CertifyScorecard(ctx, client, *v.Source, *v.Scorecard)
return err
Expand Down Expand Up @@ -409,6 +429,22 @@ func ingestHashEqual(ctx context.Context, client graphql.Client, v assembler.Has
return err
}

func ingestCertifyLegal(ctx context.Context, client graphql.Client, v assembler.CertifyLegalIngest) error {
if v.Pkg != nil && v.Src != nil {
return fmt.Errorf("unable to create CertifyLegal with both Src and Pkg subject specified")
}
if v.Pkg == nil && v.Src == nil {
return fmt.Errorf("unable to create CertifyLegal without either Src and Pkg subject specified")
}

if v.Src != nil {
_, err := model.CertifyLegalSrc(ctx, client, *v.Src, v.Declared, v.Discovered, *v.CertifyLegal)
return err
}
_, err := model.CertifyLegalPkg(ctx, client, *v.Pkg, v.Declared, v.Discovered, *v.CertifyLegal)
return err
}

func validatePackageSourceOrArtifactInput(pkg *model.PkgInputSpec, src *model.SourceInputSpec, artifact *model.ArtifactInputSpec, path string) error {
valuesDefined := 0
if pkg != nil {
Expand Down

0 comments on commit ebd91bb

Please sign in to comment.