Skip to content

Commit

Permalink
Update key methods to use a non-cryptographic hash function. (#1559)
Browse files Browse the repository at this point in the history 
Some key-value stores (TiKV) have a size limit on keys. So just concatinating
all the fields for each node will overflow it. Use a NCHF to hash the key.

Signed-off-by: Jeff Mendoza <jlm@jlm.name>
  • Loading branch information
@jeffmendoza
jeffmendoza committed Dec 7, 2023
1 parent 18ad0d0 commit 2b1e1ae
Show file tree
Hide file tree
Showing 34 changed files with 144 additions and 128 deletions.
2 changes: 1 addition & 1 deletion pkg/assembler/backends/keyvalue/artifact.go
View file
Expand Up @@ -120,7 +120,7 @@ func (n *artStruct) setPointOfContactLinks(ctx context.Context, ID string, c *de
}

func (n *artStruct) Key() string {
return strings.Join([]string{n.Algorithm, n.Digest}, ":")
return hashKey(strings.Join([]string{n.Algorithm, n.Digest}, ":"))
}

func (c *demoClient) artifactByInput(ctx context.Context, a *model.ArtifactInputSpec) (*artStruct, error) {
Expand Down
16 changes: 16 additions & 0 deletions pkg/assembler/backends/keyvalue/backend.go
View file
Expand Up @@ -19,6 +19,7 @@ import (
"context"
"errors"
"fmt"
"hash/fnv"
"math"
"reflect"
"slices"
Expand Down Expand Up @@ -329,3 +330,18 @@ func (c *demoClient) getPackageVersionAndArtifacts(ctx context.Context, pkgOrArt

return pkgs, arts, nil
}

func hashKey(in string) string {
h := fnv.New128a()
h.Write([]byte(in))
bts := h.Sum(nil)
return fmtBytes(bts)
}

func fmtBytes(s []byte) string {
strs := make([]string, len(s))
for i, v := range s {
strs[i] = fmt.Sprintf("%x", v)
}
return strings.Join(strs, "")
}
2 changes: 1 addition & 1 deletion pkg/assembler/backends/keyvalue/builder.go
View file
Expand Up @@ -32,7 +32,7 @@ type builderStruct struct {
}

func (n *builderStruct) Key() string {
return n.URI
return hashKey(n.URI)
}

func (b *builderStruct) ID() string { return b.ThisID }
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/certifyBad.go
View file
Expand Up @@ -44,15 +44,15 @@ type badLink struct {
func (n *badLink) ID() string { return n.ThisID }

func (n *badLink) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.PackageID,
n.ArtifactID,
n.SourceID,
n.Justification,
n.Origin,
n.Collector,
timeKey(n.KnownSince),
}, ":")
}, ":"))
}

func (n *badLink) Neighbors(allowedEdges edgeMap) []string {
Expand Down
8 changes: 4 additions & 4 deletions pkg/assembler/backends/keyvalue/certifyBad_test.go
View file
Expand Up @@ -408,11 +408,11 @@ func TestCertifyBad(t *testing.T) {
},
ExpCB: []*model.CertifyBad{
{
Subject: s1out,
Subject: s2out,
Justification: "test justification",
},
{
Subject: s2out,
Subject: s1out,
Justification: "test justification",
},
},
Expand Down Expand Up @@ -464,11 +464,11 @@ func TestCertifyBad(t *testing.T) {
},
ExpCB: []*model.CertifyBad{
{
Subject: p1outName,
Subject: p2out,
Justification: "test justification",
},
{
Subject: p2out,
Subject: p1outName,
Justification: "test justification",
},
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/certifyGood.go
View file
Expand Up @@ -42,15 +42,15 @@ type goodLink struct {
func (n *goodLink) ID() string { return n.ThisID }

func (n *goodLink) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.PackageID,
n.ArtifactID,
n.SourceID,
n.Justification,
n.Origin,
n.Collector,
timeKey(n.KnownSince),
}, ":")
}, ":"))
}

func (n *goodLink) Neighbors(allowedEdges edgeMap) []string {
Expand Down
8 changes: 4 additions & 4 deletions pkg/assembler/backends/keyvalue/certifyGood_test.go
View file
Expand Up @@ -408,11 +408,11 @@ func TestCertifyGood(t *testing.T) {
},
ExpCG: []*model.CertifyGood{
{
Subject: s1out,
Subject: s2out,
Justification: "test justification",
},
{
Subject: s2out,
Subject: s1out,
Justification: "test justification",
},
},
Expand Down Expand Up @@ -464,11 +464,11 @@ func TestCertifyGood(t *testing.T) {
},
ExpCG: []*model.CertifyGood{
{
Subject: p1outName,
Subject: p2out,
Justification: "test justification",
},
{
Subject: p2out,
Subject: p1outName,
Justification: "test justification",
},
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/certifyLegal.go
View file
Expand Up @@ -47,7 +47,7 @@ type certifyLegalStruct struct {

func (n *certifyLegalStruct) ID() string { return n.ThisID }
func (n *certifyLegalStruct) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.Pkg,
n.Source,
n.DeclaredLicense,
Expand All @@ -59,7 +59,7 @@ func (n *certifyLegalStruct) Key() string {
timeKey(n.TimeScanned),
n.Origin,
n.Collector,
}, ":")
}, ":"))
}

func (n *certifyLegalStruct) Neighbors(allowedEdges edgeMap) []string {
Expand Down
8 changes: 4 additions & 4 deletions pkg/assembler/backends/keyvalue/certifyLegal_test.go
View file
Expand Up @@ -431,12 +431,12 @@ func TestLegal(t *testing.T) {
},
ExpLegal: []*model.CertifyLegal{
{
Subject: p1out,
Subject: p2out,
DeclaredLicenses: []*model.License{l1out},
Justification: "test justification",
},
{
Subject: p2out,
Subject: p1out,
DeclaredLicenses: []*model.License{l1out},
Justification: "test justification",
},
Expand Down Expand Up @@ -570,12 +570,12 @@ func TestLegals(t *testing.T) {
},
ExpLegal: []*model.CertifyLegal{
{
Subject: p1out,
Subject: p2out,
DeclaredLicenses: []*model.License{l1out},
Justification: "test justification",
},
{
Subject: p2out,
Subject: p1out,
DeclaredLicenses: []*model.License{l1out},
Justification: "test justification",
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/certifyScorecard.go
View file
Expand Up @@ -44,7 +44,7 @@ type scorecardLink struct {

func (n *scorecardLink) ID() string { return n.ThisID }
func (n *scorecardLink) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.SourceID,
timeKey(n.TimeScanned),
fmt.Sprint(n.AggregateScore),
Expand All @@ -53,7 +53,7 @@ func (n *scorecardLink) Key() string {
n.ScorecardCommit,
n.Origin,
n.Collector,
}, ":")
}, ":"))
}

func (n *scorecardLink) Neighbors(allowedEdges edgeMap) []string {
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/certifyVEXStatement.go
View file
Expand Up @@ -46,7 +46,7 @@ type vexLink struct {
func (n *vexLink) ID() string { return n.ThisID }

func (n *vexLink) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.PackageID,
n.ArtifactID,
n.VulnerabilityID,
Expand All @@ -57,7 +57,7 @@ func (n *vexLink) Key() string {
string(n.Justification),
n.Origin,
n.Collector,
}, ":")
}, ":"))
}

func (n *vexLink) Neighbors(allowedEdges edgeMap) []string {
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/certifyVuln.go
View file
Expand Up @@ -45,7 +45,7 @@ type certifyVulnerabilityLink struct {

func (n *certifyVulnerabilityLink) ID() string { return n.ThisID }
func (n *certifyVulnerabilityLink) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.PackageID,
n.VulnerabilityID,
timeKey(n.TimeScanned),
Expand All @@ -55,7 +55,7 @@ func (n *certifyVulnerabilityLink) Key() string {
n.ScannerVersion,
n.Origin,
n.Collector,
}, ":")
}, ":"))
}

func (n *certifyVulnerabilityLink) Neighbors(allowedEdges edgeMap) []string {
Expand Down
24 changes: 12 additions & 12 deletions pkg/assembler/backends/keyvalue/certifyVuln_test.go
View file
Expand Up @@ -501,14 +501,6 @@ func TestIngestCertifyVulnerability(t *testing.T) {
Vulnerability: &model.VulnerabilitySpec{},
},
ExpVuln: []*model.CertifyVuln{
{
Package: p2out,
Vulnerability: &model.Vulnerability{
Type: "novuln",
VulnerabilityIDs: []*model.VulnerabilityID{noVulnOut},
},
Metadata: vmd1,
},
{
Package: p1out,
Vulnerability: &model.Vulnerability{
Expand All @@ -525,6 +517,14 @@ func TestIngestCertifyVulnerability(t *testing.T) {
},
Metadata: vmd1,
},
{
Package: p2out,
Vulnerability: &model.Vulnerability{
Type: "novuln",
VulnerabilityIDs: []*model.VulnerabilityID{noVulnOut},
},
Metadata: vmd1,
},
},
},
{
Expand Down Expand Up @@ -701,18 +701,18 @@ func TestIngestCertifyVulns(t *testing.T) {
},
ExpVuln: []*model.CertifyVuln{
{
Package: p1out,
Package: p2out,
Vulnerability: &model.Vulnerability{
Type: "cve",
VulnerabilityIDs: []*model.VulnerabilityID{c2out},
VulnerabilityIDs: []*model.VulnerabilityID{c1out},
},
Metadata: vmd1,
},
{
Package: p2out,
Package: p1out,
Vulnerability: &model.Vulnerability{
Type: "cve",
VulnerabilityIDs: []*model.VulnerabilityID{c1out},
VulnerabilityIDs: []*model.VulnerabilityID{c2out},
},
Metadata: vmd1,
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/hasMetadata.go
View file
Expand Up @@ -42,7 +42,7 @@ type hasMetadataLink struct {

func (n *hasMetadataLink) ID() string { return n.ThisID }
func (n *hasMetadataLink) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.PackageID,
n.ArtifactID,
n.SourceID,
Expand All @@ -52,7 +52,7 @@ func (n *hasMetadataLink) Key() string {
n.Justification,
n.Origin,
n.Collector,
}, ":")
}, ":"))
}

func (n *hasMetadataLink) Neighbors(allowedEdges edgeMap) []string {
Expand Down
12 changes: 6 additions & 6 deletions pkg/assembler/backends/keyvalue/hasMetadata_test.go
View file
Expand Up @@ -243,14 +243,14 @@ func TestHasMetadata(t *testing.T) {
ExpHM: []*model.HasMetadata{
{
Subject: p1out,
Key: "key1",
Value: "value1",
Key: "key2",
Value: "value2",
Justification: "test justification",
},
{
Subject: p1out,
Key: "key2",
Value: "value2",
Key: "key1",
Value: "value1",
Justification: "test justification",
},
},
Expand Down Expand Up @@ -546,11 +546,11 @@ func TestHasMetadata(t *testing.T) {
},
ExpHM: []*model.HasMetadata{
{
Subject: p1outName,
Subject: p2out,
Justification: "test justification",
},
{
Subject: p2out,
Subject: p1outName,
Justification: "test justification",
},
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/hasSBOM.go
View file
Expand Up @@ -46,7 +46,7 @@ type hasSBOMStruct struct {

func (n *hasSBOMStruct) ID() string { return n.ThisID }
func (n *hasSBOMStruct) Key() string {
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.Pkg,
n.Artifact,
n.URI,
Expand All @@ -59,7 +59,7 @@ func (n *hasSBOMStruct) Key() string {
fmt.Sprint(n.IncludedSoftware),
fmt.Sprint(n.IncludedDependencies),
fmt.Sprint(n.IncludedOccurrences),
}, ":")
}, ":"))
}

func (n *hasSBOMStruct) Neighbors(allowedEdges edgeMap) []string {
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/hasSBOM_test.go
View file
Expand Up @@ -798,11 +798,11 @@ func TestHasSBOM(t *testing.T) {
},
ExpHS: []*model.HasSbom{
{
Subject: p1out,
Subject: p2out,
DownloadLocation: "location two",
},
{
Subject: p2out,
Subject: p1out,
DownloadLocation: "location two",
},
},
Expand Down
4 changes: 2 additions & 2 deletions pkg/assembler/backends/keyvalue/hasSLSA.go
View file
Expand Up @@ -55,7 +55,7 @@ func (n *hasSLSAStruct) Key() string {
if n.Finish != nil {
fn = timeKey(*n.Finish)
}
return strings.Join([]string{
return hashKey(strings.Join([]string{
n.Subject,
fmt.Sprint(n.BuiltFrom),
n.BuiltBy,
Expand All @@ -66,7 +66,7 @@ func (n *hasSLSAStruct) Key() string {
fn,
n.Origin,
n.Collector,
}, ":")
}, ":"))
}

func (n *hasSLSAStruct) Neighbors(allowedEdges edgeMap) []string {
Expand Down

0 comments on commit 2b1e1ae

Please sign in to comment.