Update Database and Worker implementation for layer-wise feature

Feature extraction algorithm is changed to associate features with
ancestry layer. Database is updated to keep the relationship.

api/v3/clairpb/clair.proto

@@ -99,9 +99,8 @@ message GetAncestryResponse {
     // The layer's information.
     Layer layer = 1;
     // The features detected in this layer.
-    repeated Feature detectedFeatures = 2;
+    repeated Feature detected_features = 2;
   }
-
   message Ancestry {
     // The name of the desired ancestry.
     string name = 1;

api/v3/clairpb/clair.swagger.json

@@ -206,13 +206,15 @@
       "type": "object",
       "properties": {
         "layer": {
-          "$ref": "#/definitions/clairLayer"
+          "$ref": "#/definitions/clairLayer",
+          "description": "The layer's information."
         },
-        "detectedFeatures": {
+        "detected_features": {
           "type": "array",
           "items": {
             "$ref": "#/definitions/clairFeature"
-          }
+          },
+          "description": "The features detected in this layer."
         }
       }
     },
@@ -419,7 +421,7 @@
           "items": {
             "$ref": "#/definitions/PostAncestryRequestPostLayer"
           },
-          "description": "The layers to be scanned for this Ancestry, ordered in the way that i th\nlayer is the i + 1 th layer's parent."
+          "description": "The layers to be scanned for this Ancestry, ordered in the way that i th\nlayer is the parent of i + 1 th layer."
         }
       }
     },

api/v3/clairpb/convert.go

@@ -124,11 +124,17 @@ func VulnerabilityWithFixedInFromDatabaseModel(dbVuln database.VulnerabilityWith
 
 // AncestryFromDatabaseModel converts database ancestry to api ancestry.
 func AncestryFromDatabaseModel(dbAncestry database.Ancestry) *GetAncestryResponse_Ancestry {
-	ancestry := &GetAncestryResponse_Ancestry{Name: dbAncestry.Name}
+	ancestry := &GetAncestryResponse_Ancestry{
+		Name:             dbAncestry.Name,
+		ScannedDetectors: dbAncestry.ProcessedBy.Detectors,
+		ScannedListers:   dbAncestry.ProcessedBy.Listers,
+	}
+
 	for _, layer := range dbAncestry.Layers {
-		ancestryLayer := &GetAncestryResponse_AncestryLayer{}
-		ancestryLayer.Layer = LayerFromDatabaseModel(layer)
-		ancestry.Layers = append(ancestry.Layers, ancestryLayer)
+		ancestry.Layers = append(ancestry.Layers,
+			&GetAncestryResponse_AncestryLayer{
+				Layer: LayerFromDatabaseModel(layer),
+			})
 	}
 
 	return ancestry

api/v3/rpc.go

@@ -130,13 +130,18 @@ func (s *AncestryServer) GetAncestry(ctx context.Context, req *pb.GetAncestryReq
 			return nil, status.Error(codes.NotFound, fmt.Sprintf("requested ancestry '%s' is not found", req.GetAncestryName()))
 		}
 
-		respAncestry = &pb.GetAncestryResponse_Ancestry{Name: name}
-		respAncestry.ScannedDetectors = ancestry.ProcessedBy.Detectors
-		respAncestry.ScannedListers = ancestry.ProcessedBy.Listers
-		respAncestry.Layers = []*pb.GetAncestryResponse_AncestryLayer{}
+		respAncestry = &pb.GetAncestryResponse_Ancestry{
+			Name:             name,
+			ScannedDetectors: ancestry.ProcessedBy.Detectors,
+			ScannedListers:   ancestry.ProcessedBy.Listers,
+		}
 
 		for _, layer := range ancestry.Layers {
-			ancestryLayer := &pb.GetAncestryResponse_AncestryLayer{}
+			ancestryLayer := &pb.GetAncestryResponse_AncestryLayer{
+				Layer: &pb.Layer{
+					Hash: layer.Hash,
+				},
+			}
 
 			if req.GetWithVulnerabilities() {
 				featureVulnerabilities, err := tx.FindAffectedNamespacedFeatures(layer.DetectedFeatures)

database/database.go

@@ -91,18 +91,18 @@ type Session interface {
 
 	// UpsertAncestry inserts or replaces an ancestry and its namespaced
 	// features and processors used to scan the ancestry.
-	UpsertAncestry(ancestry Ancestry, features []NamespacedFeature, processedBy Processors) error
+	UpsertAncestry(AncestryWithContent) error
 
 	// FindAncestry retrieves an ancestry with processors used to scan the
 	// ancestry. If the ancestry is not found, return false.
 	//
 	// The ancestry's processors are returned to short cut processing ancestry
 	// if it has been processed by all processors in the current Clair instance.
-	FindAncestry(name string) (ancestry Ancestry, processedBy Processors, found bool, err error)
+	FindAncestry(name string) (ancestry Ancestry, found bool, err error)
 
-	// FindAncestryFeatures retrieves an ancestry with all detected namespaced
-	// features. If the ancestry is not found, return false.
-	FindAncestryFeatures(name string) (ancestry AncestryWithFeatures, found bool, err error)
+	// FindAncestryWithContent retrieves an ancestry with all detected
+	// namespaced features. If the ancestry is not found, return false.
+	FindAncestryWithContent(name string) (ancestry AncestryWithContent, found bool, err error)
 
 	// PersistFeatures inserts a set of features if not in the database.
 	PersistFeatures(features []Feature) error
@@ -125,8 +125,8 @@ type Session interface {
 	// PersistNamespaces inserts a set of namespaces if not in the database.
 	PersistNamespaces([]Namespace) error
 
-	// PersistLayer inserts a layer if not in the datastore.
-	PersistLayer(Layer) error
+	// PersistLayer creates a layer using the blob Sum hash.
+	PersistLayer(hash string) error
 
 	// PersistLayerContent persists a layer's content in the database. The given
 	// namespaces and features can be partial content of this layer.
@@ -135,8 +135,8 @@ type Session interface {
 	// in the database.
 	PersistLayerContent(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
 
-	// FindLayer retrieves a layer and the processors scanned the layer.
-	FindLayer(hash string) (layer Layer, processedBy Processors, found bool, err error)
+	// FindLayer retrieves the metadata of a layer.
+	FindLayer(hash string) (layer Layer, found bool, err error)
 
 	// FindLayerWithContent returns a layer with all detected features and
 	// namespaces.

database/mock.go

@@ -21,17 +21,17 @@ import "time"
 type MockSession struct {
 	FctCommit                           func() error
 	FctRollback                         func() error
-	FctUpsertAncestry                   func(Ancestry, []NamespacedFeature, Processors) error
-	FctFindAncestry                     func(name string) (Ancestry, Processors, bool, error)
-	FctFindAncestryFeatures             func(name string) (AncestryWithFeatures, bool, error)
+	FctUpsertAncestry                   func(AncestryWithContent) error
+	FctFindAncestry                     func(name string) (Ancestry, bool, error)
+	FctFindAncestryWithContent          func(name string) (AncestryWithContent, bool, error)
 	FctFindAffectedNamespacedFeatures   func(features []NamespacedFeature) ([]NullableAffectedNamespacedFeature, error)
 	FctPersistNamespaces                func([]Namespace) error
 	FctPersistFeatures                  func([]Feature) error
 	FctPersistNamespacedFeatures        func([]NamespacedFeature) error
 	FctCacheAffectedNamespacedFeatures  func([]NamespacedFeature) error
-	FctPersistLayer                     func(Layer) error
+	FctPersistLayer                     func(hash string) error
 	FctPersistLayerContent              func(hash string, namespaces []Namespace, features []Feature, processedBy Processors) error
-	FctFindLayer                        func(name string) (Layer, Processors, bool, error)
+	FctFindLayer                        func(name string) (Layer, bool, error)
 	FctFindLayerWithContent             func(name string) (LayerWithContent, bool, error)
 	FctInsertVulnerabilities            func([]VulnerabilityWithAffected) error
 	FctFindVulnerabilities              func([]VulnerabilityID) ([]NullableVulnerability, error)
@@ -63,23 +63,23 @@ func (ms *MockSession) Rollback() error {
 	panic("required mock function not implemented")
 }
 
-func (ms *MockSession) UpsertAncestry(ancestry Ancestry, features []NamespacedFeature, processedBy Processors) error {
+func (ms *MockSession) UpsertAncestry(ancestry AncestryWithContent) error {
 	if ms.FctUpsertAncestry != nil {
-		return ms.FctUpsertAncestry(ancestry, features, processedBy)
+		return ms.FctUpsertAncestry(ancestry)
 	}
 	panic("required mock function not implemented")
 }
 
-func (ms *MockSession) FindAncestry(name string) (Ancestry, Processors, bool, error) {
+func (ms *MockSession) FindAncestry(name string) (Ancestry, bool, error) {
 	if ms.FctFindAncestry != nil {
 		return ms.FctFindAncestry(name)
 	}
 	panic("required mock function not implemented")
 }
 
-func (ms *MockSession) FindAncestryFeatures(name string) (AncestryWithFeatures, bool, error) {
-	if ms.FctFindAncestryFeatures != nil {
-		return ms.FctFindAncestryFeatures(name)
+func (ms *MockSession) FindAncestryWithContent(name string) (AncestryWithContent, bool, error) {
+	if ms.FctFindAncestryWithContent != nil {
+		return ms.FctFindAncestryWithContent(name)
 	}
 	panic("required mock function not implemented")
 }
@@ -119,7 +119,7 @@ func (ms *MockSession) CacheAffectedNamespacedFeatures(namespacedFeatures []Name
 	panic("required mock function not implemented")
 }
 
-func (ms *MockSession) PersistLayer(layer Layer) error {
+func (ms *MockSession) PersistLayer(layer string) error {
 	if ms.FctPersistLayer != nil {
 		return ms.FctPersistLayer(layer)
 	}
@@ -133,7 +133,7 @@ func (ms *MockSession) PersistLayerContent(hash string, namespaces []Namespace,
 	panic("required mock function not implemented")
 }
 
-func (ms *MockSession) FindLayer(name string) (Layer, Processors, bool, error) {
+func (ms *MockSession) FindLayer(name string) (Layer, bool, error) {
 	if ms.FctFindLayer != nil {
 		return ms.FctFindLayer(name)
 	}

database/models.go

@@ -20,7 +20,7 @@ import (
 	"time"
 )
 
-// Processors are extentions to scan layer's content.
+// Processors are extentions to scan a layer's content.
 type Processors struct {
 	Listers   []string
 	Detectors []string
@@ -29,34 +29,48 @@ type Processors struct {
 // Ancestry is a manifest that keeps all layers in an image in order.
 type Ancestry struct {
 	Name string
+	// ProcessedBy contains the processors that are used when computing the
+	// content of this ancestry.
+	ProcessedBy Processors
 	// Layers should be ordered and i_th layer is the parent of i+1_th layer in
 	// the slice.
 	Layers []Layer
 }
 
-// AncestryWithFeatures is an ancestry with namespaced features detected in the
-// ancestry, which is processed by `ProcessedBy`.
-type AncestryWithFeatures struct {
+// AncestryWithContent has the ancestry's name and the Ancestry Layers
+// associated with it.
+type AncestryWithContent struct {
 	Ancestry
 
-	ProcessedBy Processors
-	Features    []NamespacedFeature
+	// TODO(sidchen) deduplicate the Layers here and the Layers in
+	// Ancestry.Layers.
+	// AncestryLayers should have the same order as Ancestry.Layers.
+	Layers []AncestryLayer
+}
+
+// AncestryLayer is a layer with all detected namespaced features.
+type AncestryLayer struct {
+	Layer
+
+	// DetectedFeatures are the features introduced by this layer.
+	DetectedFeatures []NamespacedFeature
 }
 
-// Layer corresponds to a layer in an image processed by `ProcessedBy`.
+// Layer contains the metadata of a layer.
 type Layer struct {
 	// Hash is content hash of the layer.
 	Hash string
+	// ProcessedBy contains the processors that processed this layer.
+	ProcessedBy Processors
 }
 
 // LayerWithContent is a layer with its detected namespaces and features by
 // ProcessedBy.
 type LayerWithContent struct {
 	Layer
 
-	ProcessedBy Processors
-	Namespaces  []Namespace
-	Features    []Feature
+	Namespaces []Namespace
+	Features   []Feature
 }
 
 // Namespace is the contextual information around features.
@@ -198,6 +212,7 @@ type VulnerabilityNotificationWithVulnerable struct {
 // PageNumber is used to do pagination.
 type PageNumber string
 
+// MetadataMap is for storing the metadata returned by vulnerability database.
 type MetadataMap map[string]interface{}
 
 // NullableAffectedNamespacedFeature is an affectednamespacedfeature with