utils: remove string.go

database/pgsql/complex_test.go

@@ -29,7 +29,6 @@ import (
 	"github.com/coreos/clair"
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/versionfmt/dpkg"
-	"github.com/coreos/clair/utils"
 )
 
 const (
@@ -157,7 +156,7 @@ func TestRaceAffects(t *testing.T) {
 			}
 		}
 
-		assert.Len(t, utils.CompareStringLists(expectedAffectedNames, actualAffectedNames), 0)
-		assert.Len(t, utils.CompareStringLists(actualAffectedNames, expectedAffectedNames), 0)
+		assert.Len(t, compareStringLists(expectedAffectedNames, actualAffectedNames), 0)
+		assert.Len(t, compareStringLists(actualAffectedNames, expectedAffectedNames), 0)
 	}
 }

database/pgsql/layer.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/pkg/commonerr"
-	"github.com/coreos/clair/utils"
 )
 
 func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) {
@@ -362,8 +361,8 @@ func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *
 		parentLayerFeaturesMapNV, parentLayerFeaturesNV := createNV(layer.Parent.Features)
 
 		// Calculate the added and deleted FeatureVersions name:version.
-		addNV := utils.CompareStringLists(layerFeaturesNV, parentLayerFeaturesNV)
-		delNV := utils.CompareStringLists(parentLayerFeaturesNV, layerFeaturesNV)
+		addNV := compareStringLists(layerFeaturesNV, parentLayerFeaturesNV)
+		delNV := compareStringLists(parentLayerFeaturesNV, layerFeaturesNV)
 
 		// Fill the structures containing the added and deleted FeatureVersions.
 		for _, nv := range addNV {

database/pgsql/vulnerability.go

@@ -23,10 +23,48 @@ import (
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/ext/versionfmt"
 	"github.com/coreos/clair/pkg/commonerr"
-	"github.com/coreos/clair/utils"
 	"github.com/guregu/null/zero"
 )
 
+// compareStringLists returns the strings that are present in X but not in Y.
+func compareStringLists(X, Y []string) []string {
+	m := make(map[string]bool)
+
+	for _, y := range Y {
+		m[y] = true
+	}
+
+	diff := []string{}
+	for _, x := range X {
+		if m[x] {
+			continue
+		}
+
+		diff = append(diff, x)
+		m[x] = true
+	}
+
+	return diff
+}
+
+func compareStringListsInBoth(X, Y []string) []string {
+	m := make(map[string]struct{})
+
+	for _, y := range Y {
+		m[y] = struct{}{}
+	}
+
+	diff := []string{}
+	for _, x := range X {
+		if _, e := m[x]; e {
+			diff = append(diff, x)
+			delete(m, x)
+		}
+	}
+
+	return diff
+}
+
 func (pgSQL *pgSQL) ListVulnerabilities(namespaceName string, limit int, startID int) ([]database.Vulnerability, int, error) {
 	defer observeQueryTime("listVulnerabilities", "all", time.Now())
 
@@ -341,8 +379,8 @@ func applyFixedInDiff(currentList, diff []database.FeatureVersion) ([]database.F
 	currentMap, currentNames := createFeatureVersionNameMap(currentList)
 	diffMap, diffNames := createFeatureVersionNameMap(diff)
 
-	addedNames := utils.CompareStringLists(diffNames, currentNames)
-	inBothNames := utils.CompareStringListsInBoth(diffNames, currentNames)
+	addedNames := compareStringLists(diffNames, currentNames)
+	inBothNames := compareStringListsInBoth(diffNames, currentNames)
 
 	different := false
 

database/pgsql/vulnerability_test.go

@@ -282,3 +282,16 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
 		}
 	}
 }
+
+func TestStringComparison(t *testing.T) {
+	cmp := compareStringLists([]string{"a", "b", "b", "a"}, []string{"a", "c"})
+	assert.Len(t, cmp, 1)
+	assert.NotContains(t, cmp, "a")
+	assert.Contains(t, cmp, "b")
+
+	cmp = compareStringListsInBoth([]string{"a", "a", "b", "c"}, []string{"a", "c", "c"})
+	assert.Len(t, cmp, 2)
+	assert.NotContains(t, cmp, "b")
+	assert.Contains(t, cmp, "a")
+	assert.Contains(t, cmp, "c")
+}

worker/worker.go

@@ -17,6 +17,8 @@
 package worker
 
 import (
+	"regexp"
+
 	"github.com/coreos/pkg/capnslog"
 
 	"github.com/coreos/clair/database"
@@ -25,7 +27,6 @@ import (
 	"github.com/coreos/clair/ext/imagefmt"
 	"github.com/coreos/clair/pkg/commonerr"
 	"github.com/coreos/clair/pkg/tarutil"
-	"github.com/coreos/clair/utils"
 )
 
 const (
@@ -44,8 +45,15 @@ var (
 	// ErrParentUnknown is the error that should be raised when a parent layer
 	// has yet to be processed for the current layer.
 	ErrParentUnknown = commonerr.NewBadRequestError("worker: parent layer is unknown, it must be processed first")
+
+	urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`)
 )
 
+// cleanURL removes all parameters from an URL.
+func cleanURL(str string) string {
+	return urlParametersRegexp.ReplaceAllString(str, "")
+}
+
 // Process detects the Namespace of a layer, the features it adds/removes, and
 // then stores everything in the database.
 // TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
@@ -65,7 +73,7 @@ func Process(datastore database.Datastore, imageFormat, name, parentName, path s
 	}
 
 	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)",
-		name, utils.CleanURL(path), Version, parentName, imageFormat)
+		name, cleanURL(path), Version, parentName, imageFormat)
 
 	// Check to see if the layer is already in the database.
 	layer, err := datastore.FindLayer(name, false, false)
@@ -118,7 +126,7 @@ func detectContent(imageFormat, name, path string, headers map[string]string, pa
 	totalRequiredFiles := append(featurefmt.RequiredFilenames(), featurens.RequiredFilenames()...)
 	files, err := imagefmt.Extract(imageFormat, path, headers, totalRequiredFiles)
 	if err != nil {
-		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
+		log.Errorf("layer %s: failed to extract data from %s: %s", name, cleanURL(path), err)
 		return
 	}