Merge pull request #205 from m-lab/sandbox-reload

Sandbox reload

README.md

@@ -82,3 +82,28 @@ each distinct date, using the most recent Annotator from each list prior to that
 ### Directory
 
 Directory wraps a []api.Annotator, and provides the GetAnnotator(date time.Time) function.
+
+### CachingLoader
+
+CachingLoader specifies the interface provided by loaders that load and cache a list of annotators.
+
+1. Maintains list of loaded Annotator objects.
+1. Refreshes the list of loaded objects on demand.
+
+```
+type CachingLoader interface {
+  UpdateCache() error
+  Fetch() []Annotator
+}
+```
+
+TODO: Need to implement the CachingLoaders.
+
+### Generator
+```
+func NewGenerator(v4, v6, g2 *CachingLoader)
+
+func (gen *Generator) Update() error         // Reloads all lists
+func (gen *Generator) Generate() []Annotator // constructs list of CompositeAnnotators
+
+```

api/api.go

@@ -108,3 +108,18 @@ func ExtractDateFromFilename(filename string) (time.Time, error) {
 	}
 	return time.Parse(time.RFC3339, filedate[0][0:4]+"-"+filedate[0][4:6]+"-"+filedate[0][6:8]+"T00:00:00Z")
 }
+
+/*************************************************************************
+*                            Annotator Loader                            *
+*************************************************************************/
+
+// CachingLoader keeps a cache of loaded annotators, updates the cache on request, and returns a copy
+// of the cache on request.
+type CachingLoader interface {
+	// UpdateCache causes the loader to load any new annotators and add them to the cached list.
+	UpdateCache() error
+
+	// Fetch returns a copy of the current list of annotators.
+	// May return an empty slice, but must not return nil.
+	Fetch() []Annotator
+}

directory/directory.go

@@ -1,5 +1,6 @@
 // Package directory provides the GetAnnotator function, which returns an appropriate annotator for
 // requests with a particular target date.
+// TODO - rename this cache?  Or is there a better name?
 package directory
 
 // A directory entry points to an appropriate CompositeAnnotator.
@@ -13,28 +14,17 @@ package directory
 // until then, we only have a different CA for each date where a new v4 or v6, or a new GL2
 // annotator is available.
 //
-// To construct the directory, we begin with lists of Annotator objects for each type of annotation.
-// We first merge the v4 and v6 annotators into a list of CompositeAnnotators, using MergeAnnotators.
-// We then append all the GeoLite2 annotators to this list.
-// Then, we merge the Geo annotation list with the ASN annotator list.
-// Finally, we use Build to create a Directory based on this list.
-
-// Example use (simplified, with some functions that don't exist yet):
-// v4, _ = geoloader.LoadLegacyV4(nil)
-// v6, _ = geoloader.LoadLegacyV6(nil)
-// legacy := directory.MergeAnnotators(v4, v6)  // Creates annotators that will handle v4 or v6
-// g2, _ = geoloader.LoadGeolite2(nil)
-// combo := make([]api.Annotator, len(g2)+len(legacy))
-// combo = append(combo, g2...)
-// combo = append(combo, legacy...)
-// annotatorDirectory = directory.Build(combo)
-
-// TODO delete this line.  Just here to allow comments in #198
+// Generator is used to construct directories. It collects lists of Annotator objects for each type of
+// annotator from CachingLoader objects.
+// It then merges the v4 and v6 annotators into a list of CompositeAnnotators, using MergeAnnotators.
+// It then appends all the GeoLite2 annotators to this list.
+// Then, we merge the Geo annotation list with the ASN annotator list (when available).
 
 import (
 	"errors"
 	"log"
 	"sort"
+	"sync"
 	"time"
 
 	"github.com/m-lab/annotation-service/api"
@@ -45,8 +35,6 @@ var (
 
 	// ErrEmptyDirectory is returned by GetAnnotator if a Directory has no entries.
 	ErrEmptyDirectory = errors.New("Directory is empty")
-	// ErrNilAnnotator is returned if GetAnnotator encounters a nil Directory entry.
-	ErrNilAnnotator = errors.New("Annotator is nil")
 )
 
 // CompositeAnnotator wraps several annotators, and calls to Annotate() are forwarded to all of them.
@@ -118,6 +106,7 @@ func NewCompositeAnnotator(annotators []api.Annotator) api.Annotator {
 }
 
 // Directory allows searching a list of annotators
+// TODO not crazy about this name.
 type Directory struct {
 	annotators []api.Annotator
 }
@@ -246,3 +235,78 @@ func (d *Directory) lastEarlierThan(date time.Time) api.Annotator {
 	}
 	return d.annotators[index-1]
 }
+
+/*************************************************************************
+*                          Directory Builder                             *
+*************************************************************************/
+
+// Generator wraps a set of CachingLoaders, and creates a set of merged Annotators on request.
+// TODO - not crazy about this name.
+type Generator struct {
+	legacyV4 api.CachingLoader // loader for legacy v4 annotators
+	legacyV6 api.CachingLoader // loader for legacy v6 annotators
+	geolite2 api.CachingLoader // loader for geolite2 annotators
+	asn      api.CachingLoader // loader for asn annotators (currently nil)
+}
+
+// NewGenerator initializes a Generator object, and preloads the CachingLoaders
+func NewGenerator(v4, v6, g2 api.CachingLoader) *Generator {
+	if v4 == nil || v6 == nil || g2 == nil {
+		return nil
+	}
+	wg := sync.WaitGroup{}
+	go func() {
+		v4.UpdateCache()
+		wg.Done()
+	}()
+	go func() {
+		v6.UpdateCache()
+		wg.Done()
+	}()
+	go func() {
+		g2.UpdateCache()
+		wg.Done()
+	}()
+	wg.Wait()
+	return &Generator{v4, v6, g2, nil}
+}
+
+// Update updates the (dynamic) CachingLoaders
+func (gen *Generator) Update() error {
+	// v4 and v6 are static, so we  don't have to reload them.
+	return gen.geolite2.UpdateCache()
+}
+
+// Generate creates a complete list of CompositeAnnotators from the cached annotators
+// from the CachingLoaders.
+func (gen *Generator) Generate() []api.Annotator {
+	v4 := gen.legacyV4.Fetch()
+	v6 := gen.legacyV6.Fetch()
+
+	var legacy []api.Annotator
+	if len(v4)*len(v6) < 1 {
+		log.Println("empty legacy v4 or v6 annotator list - skipping legacy")
+		legacy = make([]api.Annotator, 0)
+	} else {
+		legacy = MergeAnnotators(v4, v6)
+		// TODO logAnnotatorDates("legacy", legacy)
+	}
+
+	// Now append the Geolite2 annotators
+	g2 := gen.geolite2.Fetch()
+
+	combo := make([]api.Annotator, 0, len(g2)+len(legacy))
+	combo = append(combo, legacy...)
+	combo = append(combo, g2...)
+
+	// Sort them just in case there are some out of order.
+	combo = SortSlice(combo)
+	// TODO logAnnotatorDates("combo", combo)
+
+	if len(combo) < 1 {
+		log.Println("No annotators available")
+		return nil
+	}
+
+	return combo
+}

geoloader/geoloader_test.go

@@ -38,6 +38,9 @@ func fakeLoader(obj *storage.ObjectAttrs) (api.Annotator, error) {
 }
 
 func TestLoad(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test that depends on GCS")
+	}
 	// The downloader-mlab-testing bucket has a snapshot of the datasets
 	// as of Sept 22, 2018.  If we ever update it, the numbers here may
 	// need to be adjusted.