Merge 4ed47fc into 4801d04

.travis.yml

@@ -61,20 +61,19 @@ script:
 
 # First run all tests, to ensure that we don't miss any new tests not included in
 # the coverage collection.
-- go test ./...
+- GCLOUD_PROJECT=mlab-testing go test ./...
 # Also run benchmarks
 - go test -bench . ./search/...
 
 # Run all tests, collecting coverage data.
-- go test -covermode=count -coverprofile=handler.cov -v github.com/m-lab/annotation-service/handler
-- go test -covermode=count -coverprofile=dataset.cov -v github.com/m-lab/annotation-service/handler/dataset
+- GCLOUD_PROJECT=mlab-testing go test -covermode=count -coverprofile=handler.cov -v github.com/m-lab/annotation-service/handler
 - go test -covermode=count -coverprofile=loader.cov -v github.com/m-lab/annotation-service/loader
 - go test -covermode=count -coverprofile=metrics.cov -v github.com/m-lab/annotation-service/metrics
 - go test -covermode=count -coverprofile=parser.cov -v github.com/m-lab/annotation-service/parser
 - go test -covermode=count -coverprofile=search.cov -v github.com/m-lab/annotation-service/search
 
 # Coveralls
-- $HOME/gopath/bin/gocovmerge handler.cov dataset.cov loader.cov parser.cov search.cov > merge.cov
+- $HOME/gopath/bin/gocovmerge handler.cov loader.cov parser.cov search.cov > merge.cov
 - $HOME/gopath/bin/goveralls -coverprofile=merge.cov -service=travis-ci
 
 # Build and prepare for deployment

annotator.yaml

@@ -7,17 +7,17 @@ service: annotator
 
 # TODO(dev): adjust CPU and memory based on actual requirements.
 resources:
-  cpu: 2
+  cpu: 8
   # Instances support between [(cpu * 0.9) - 0.4, (cpu * 6.5) - 0.4]
   # Actual memory available is exposed via GAE_MEMORY_MB environment variable.
-  memory_gb: 6
+  memory_gb: 40
 
   # TODO - Do we need any disk? Adjust once we understand requirements.
   disk_size_gb: 10
 
 automatic_scaling:
   # We expect negligible load, so this is unlikely to trigger.
-  min_num_instances: 2
+  min_num_instances: 10
   max_num_instances: 20
   cool_down_period_sec: 1800
   cpu_utilization:

handler/data.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"regexp"
 
-	"github.com/m-lab/annotation-service/loader"
 	"github.com/m-lab/annotation-service/parser"
 
 	"google.golang.org/api/iterator"
@@ -17,6 +16,10 @@ import (
 // This is the regex used to filter for which files we want to consider acceptable for using with Geolite2
 var GeoLite2Regex = regexp.MustCompile(`Maxmind/\d{4}/\d{2}/\d{2}/\d{8}T\d{6}Z-GeoLite2-City-CSV\.zip`)
 
+// This is the regex used to filter for which files we want to consider acceptable for using with Geolite2
+var GeoLegacyRegex = regexp.MustCompile(`.*-GeoLiteCity.dat.*`)
+var GeoLegacyv6Regex = regexp.MustCompile(`.*-GeoLiteCityv6.dat.*`)
+
 var BucketName = "downloader-" + os.Getenv("GCLOUD_PROJECT") // This is the bucket containing maxmind files
 
 const (
@@ -34,9 +37,11 @@ func PopulateLatestData() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	currentDataMutex.Lock()
-	CurrentGeoDataset = data
-	currentDataMutex.Unlock()
+
+	CurrentGeoDataset.SetCurrentDataset(data)
+	// clear the data in memeory for historical data.
+	Geolite2DatasetInMemory.Init()
+	LegacyDatasetInMemory.Init()
 }
 
 // DetermineFilenameOfLatestGeolite2File will get a list of filenames
@@ -70,9 +75,5 @@ func LoadLatestGeolite2File() (*parser.GeoDataset, error) {
 	if err != nil {
 		return nil, err
 	}
-	zip, err := loader.CreateZipReader(context.Background(), BucketName, filename)
-	if err != nil {
-		return nil, err
-	}
-	return parser.LoadGeoLite2(zip)
+	return LoadGeoLite2Dataset(filename, BucketName)
 }

handler/geoip/ex/geoip-demo.go

@@ -10,22 +10,22 @@ func main() {
 
 	file6 := "./GeoIPv6.dat"
 
-	gi6, err := geoip.Open(file6)
+	gi6, err := geoip.Open(file6, "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIPv6 database: %s\n", err)
 	}
 
-	gi, err := geoip.Open("./GeoLiteCity.dat")
+	gi, err := geoip.Open("./GeoLiteCity.dat", "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIP database: %s\n", err)
 	}
 
-	giasn, err := geoip.Open("./GeoIPASNum.dat")
+	giasn, err := geoip.Open("./GeoIPASNum.dat", "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIPASN database: %s\n", err)
 	}
 
-	giasn6, err := geoip.Open("./GeoIPASNumv6.dat")
+	giasn6, err := geoip.Open("./GeoIPASNumv6.dat", "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIPASN database: %s\n", err)
 	}
@@ -38,7 +38,7 @@ func main() {
 	}
 
 	if gi != nil {
-		record := gi.GetRecord("207.171.7.51")
+		record := gi.GetRecord("207.171.7.51", true)
 		fmt.Printf("%v\n", record)
 	}
 	if gi6 != nil {

handler/geoip/geoip.go

@@ -16,15 +16,17 @@ import "C"
 
 import (
 	"fmt"
+	"log"
 	"os"
-	"runtime"
+	//"runtime"
 	"sync"
 	"unsafe"
 )
 
 type GeoIP struct {
 	db *C.GeoIP
 
+	name string
 	// We don't use GeoIP's thread-safe API calls, which means there is a
 	// single global netmask variable that gets clobbered in the main
 	// lookup routine.  Any calls which have _GeoIP_seek_record_gl need to
@@ -33,30 +35,31 @@ type GeoIP struct {
 	mu sync.Mutex
 }
 
-func (gi *GeoIP) free() {
+func (gi *GeoIP) Free() {
 	if gi == nil {
 		return
 	}
 	if gi.db == nil {
 		gi = nil
 		return
 	}
+	log.Println("free memory for legacy dataset " + gi.name)
 	C.GeoIP_delete(gi.db)
 	gi = nil
 	return
 }
 
 // Default convenience wrapper around OpenDb
-func Open(filename string) (*GeoIP, error) {
-	return OpenDb(filename, GEOIP_MEMORY_CACHE)
+func Open(filename string, datasetName string) (*GeoIP, error) {
+	return OpenDb(filename, GEOIP_MEMORY_CACHE, datasetName)
 }
 
 // Opens a GeoIP database by filename with specified GeoIPOptions flag.
 // All formats supported by libgeoip are supported though there are only
 // functions to access some of the databases in this API.
-func OpenDb(file string, flag int) (*GeoIP, error) {
+func OpenDb(file string, flag int, datasetName string) (*GeoIP, error) {
 	g := &GeoIP{}
-	runtime.SetFinalizer(g, (*GeoIP).free)
+	//runtime.SetFinalizer(g, (*GeoIP).Free)
 
 	var err error
 
@@ -78,6 +81,7 @@ func OpenDb(file string, flag int) (*GeoIP, error) {
 	}
 
 	C.GeoIP_set_charset(g.db, C.GEOIP_CHARSET_UTF8)
+	g.name = datasetName
 	return g, nil
 }
 
@@ -95,7 +99,7 @@ func SetCustomDirectory(dir string) {
 // (for example GEOIP_COUNTRY_EDITION).
 func OpenTypeFlag(dbType int, flag int) (*GeoIP, error) {
 	g := &GeoIP{}
-	runtime.SetFinalizer(g, (*GeoIP).free)
+	//runtime.SetFinalizer(g, (*GeoIP).Free)
 
 	var err error
 
@@ -167,16 +171,24 @@ type GeoIPRecord struct {
 
 // Returns the "City Record" for an IP address. Requires the GeoCity(Lite)
 // database - http://www.maxmind.com/en/city
-func (gi *GeoIP) GetRecord(ip string) *GeoIPRecord {
+func (gi *GeoIP) GetRecord(ip string, isIP4 bool) *GeoIPRecord {
 	if gi.db == nil {
 		return nil
 	}
 
+	if len(ip) == 0 {
+		return nil
+	}
 	cip := C.CString(ip)
 	defer C.free(unsafe.Pointer(cip))
 
+	var record *C.GeoIPRecord
 	gi.mu.Lock()
-	record := C.GeoIP_record_by_addr(gi.db, cip)
+	if isIP4 {
+		record = C.GeoIP_record_by_addr(gi.db, cip)
+	} else {
+		record = C.GeoIP_record_by_addr_v6(gi.db, cip)
+	}
 	gi.mu.Unlock()
 
 	if record == nil {