Merge pull request #96 from m-lab/sandbox-legacy

Fix 3rd-party C wrapper code for legacy Maxmind API

.travis.yml

@@ -68,13 +68,14 @@ script:
 # Run all tests, collecting coverage data.
 - go test -covermode=count -coverprofile=handler.cov -v github.com/m-lab/annotation-service/handler
 - go test -covermode=count -coverprofile=dataset.cov -v github.com/m-lab/annotation-service/handler/dataset
+- go test -covermode=count -coverprofile=geoip.cov -v github.com/m-lab/annotation-service/handler/geoip
 - go test -covermode=count -coverprofile=loader.cov -v github.com/m-lab/annotation-service/loader
 - go test -covermode=count -coverprofile=metrics.cov -v github.com/m-lab/annotation-service/metrics
 - go test -covermode=count -coverprofile=parser.cov -v github.com/m-lab/annotation-service/parser
 - go test -covermode=count -coverprofile=search.cov -v github.com/m-lab/annotation-service/search
 
 # Coveralls
-- $HOME/gopath/bin/gocovmerge handler.cov dataset.cov loader.cov parser.cov search.cov > merge.cov
+- $HOME/gopath/bin/gocovmerge handler.cov dataset.cov geoip.cov loader.cov parser.cov search.cov > merge.cov
 - $HOME/gopath/bin/goveralls -coverprofile=merge.cov -service=travis-ci
 
 # Build and prepare for deployment

handler/dataset/legacy-data.go

@@ -83,14 +83,17 @@ gs://downloader-mlab-oti/Maxmind/2018/02/08/20180208T013555Z-GeoLite2-City-CSV.z
 
 
 */
+// TODO: remove dataset package and move this file to handler package to avoid circular dependancy.
 import (
 	"context"
 	"errors"
 	"fmt"
 	"regexp"
+	"strconv"
 	"time"
 
 	"cloud.google.com/go/storage"
+	"github.com/m-lab/annotation-service/common"
 	"github.com/m-lab/annotation-service/handler"
 	"github.com/m-lab/annotation-service/handler/geoip"
 	"github.com/m-lab/annotation-service/loader"
@@ -189,26 +192,18 @@ func SelectGeoLegacyFile(requestDate time.Time, bucketName string) (string, erro
 // LoadGeoliteDataset will check GCS for the matching dataset, download
 // it, process it, and load it into memory so that it can be easily
 // searched, then it will return a pointer to that GeoDataset or an error.
-func LoadLegacyGeoliteDataset(requestDate time.Time, bucketName string) (*geoip.GeoIP, error) {
-	CutOffDate, _ := time.Parse("January 2, 2006", GeoLite2CutOffDate)
-	if requestDate.Before(CutOffDate) {
-		filename, err := SelectGeoLegacyFile(requestDate, bucketName)
-		if err != nil {
-			return nil, err
-		}
-		// load the legacy binary dataset
-		dataFileName := "GeoLiteCity.dat"
-		err = loader.UncompressGzFile(context.Background(), bucketName, filename, dataFileName)
-		if err != nil {
-			return nil, err
-		}
-		gi, err := geoip.Open(dataFileName)
-		if err != nil {
-			return nil, errors.New("could not open GeoIP database")
-		}
-		return gi, nil
+func LoadLegacyGeoliteDataset(filename string, bucketname string) (*geoip.GeoIP, error) {
+	// load the legacy binary dataset
+	dataFileName := "GeoLiteCity.dat"
+	err := loader.UncompressGzFile(context.Background(), bucketname, filename, dataFileName)
+	if err != nil {
+		return nil, err
+	}
+	gi, err := geoip.Open(dataFileName, filename)
+	if err != nil {
+		return nil, errors.New("could not open GeoIP database")
 	}
-	return nil, errors.New("should call LoadGeoLite2Dataset with input date")
+	return gi, nil
 }
 
 func LoadGeoLite2Dataset(requestDate time.Time, bucketName string) (*parser.GeoDataset, error) {
@@ -228,9 +223,37 @@ func LoadGeoLite2Dataset(requestDate time.Time, bucketName string) (*parser.GeoD
 	return nil, errors.New("should call LoadLegacyGeoliteDataset with input date")
 }
 
-func GetRecordFromLegacyDataset(gi *geoip.GeoIP, ip string) {
-	if gi != nil {
-		record := gi.GetRecord(ip)
-		fmt.Printf("%v\n", record)
+func round(x float32) float64 {
+	i, err := strconv.ParseFloat(fmt.Sprintf("%.3f", x), 64)
+	if err != nil {
+		return float64(0)
+	}
+	return i
+}
+
+func GetRecordFromLegacyDataset(ip string, gi *geoip.GeoIP, isIP4 bool) *common.GeoData {
+	if gi == nil {
+		return nil
 	}
+	record := gi.GetRecord(ip, isIP4)
+	// It is very possible that the record missed some fields in legacy dataset.
+	if record != nil {
+		return &common.GeoData{
+			Geo: &common.GeolocationIP{
+				Continent_code: record.ContinentCode,
+				Country_code:   record.CountryCode,
+				Country_code3:  record.CountryCode3,
+				Country_name:   record.CountryName,
+				Region:         record.Region,
+				Metro_code:     int64(record.MetroCode),
+				City:           record.City,
+				Area_code:      int64(record.AreaCode),
+				Postal_code:    record.PostalCode,
+				Latitude:       round(record.Latitude),
+				Longitude:      round(record.Longitude),
+			},
+			ASN: &common.IPASNData{},
+		}
+	}
+	return nil
 }

handler/dataset/legacy-data_test.go

@@ -1,7 +1,6 @@
 package dataset_test
 
 import (
-	"fmt"
 	"log"
 	"testing"
 	"time"
@@ -64,7 +63,15 @@ func TestSelectGeoLegacyFile(t *testing.T) {
 		t.Errorf("Did not select correct dataset. Expected %s, got %s, %+v.",
 			"Maxmind/2017/08/15/20170815T200946Z-GeoLite2-City-CSV.zip", filename4, err)
 	}
-
+	/*
+		// before the cutoff date, IPv6
+		date6, _ := time.Parse("January 2, 2006", "April 4, 2016")
+		filename6, err := dataset.SelectGeoLegacyFile(date6, testBucket, false)
+		if filename6 != "Maxmind/2016/03/08/20160308T080000Z-GeoLiteCityv6.dat.gz" || err != nil {
+			t.Errorf("Did not select correct dataset. Expected %s, got %s, %+v.",
+				"Maxmind/2016/03/08/20160308T080000Z-GeoLiteCityv6.dat.gz", filename6, err)
+		}
+	*/
 	// return the latest available dataset.
 	date5, _ := time.Parse("January 2, 2006", "August 15, 2037")
 	filename5, err := dataset.SelectGeoLegacyFile(date5, testBucket)
@@ -83,11 +90,12 @@ type GeoIPSuite struct {
 var _ = check.Suite(&GeoIPSuite{})
 
 func (s *GeoIPSuite) TestLoadLegacyGeoliteDataset(c *check.C) {
-	date1, _ := time.Parse("January 2, 2006", "February 3, 2014")
-	gi, err := dataset.LoadLegacyGeoliteDataset(date1, "downloader-mlab-testing")
-	fmt.Printf("%v", err)
+	gi, err := dataset.LoadLegacyGeoliteDataset("Maxmind/2014/03/07/20140307T160000Z-GeoLiteCity.dat.gz", "downloader-mlab-testing")
+	if err != nil {
+		log.Printf("Did not load legacy dataset correctly %v", err)
+	}
 	if gi != nil {
-		record := gi.GetRecord("207.171.7.51")
+		record := gi.GetRecord("207.171.7.51", true)
 		c.Assert(record, check.NotNil)
 		c.Check(
 			*record,
@@ -109,3 +117,32 @@ func (s *GeoIPSuite) TestLoadLegacyGeoliteDataset(c *check.C) {
 		)
 	}
 }
+
+func (s *GeoIPSuite) TestLoadLegacyGeoliteV6Dataset(c *check.C) {
+	gi, err := dataset.LoadLegacyGeoliteDataset("Maxmind/2014/03/07/20140307T160000Z-GeoLiteCityv6.dat.gz", "downloader-mlab-testing")
+	if err != nil {
+		log.Printf("Did not load legacy dataset correctly %v", err)
+	}
+	if gi != nil {
+		record := gi.GetRecord("2620:0:1003:415:fa1e:73f3:ec68:7709", false)
+		c.Assert(record, check.NotNil)
+		c.Check(
+			*record,
+			check.Equals,
+			geoip.GeoIPRecord{
+				CountryCode:   "US",
+				CountryCode3:  "USA",
+				CountryName:   "United States",
+				Region:        "",
+				City:          "",
+				PostalCode:    "",
+				Latitude:      38,
+				Longitude:     -97,
+				AreaCode:      00,
+				MetroCode:     0,
+				CharSet:       1,
+				ContinentCode: "NA",
+			},
+		)
+	}
+}

handler/geoip/ex/geoip-demo.go

@@ -1,3 +1,6 @@
+/* demo code to show how to use the C-Wrapper of legacy Maxmind API.
+   originally forked from github.com/abh/geoip
+*/
 package main
 
 import (
@@ -10,22 +13,22 @@ func main() {
 
 	file6 := "./GeoIPv6.dat"
 
-	gi6, err := geoip.Open(file6)
+	gi6, err := geoip.Open(file6, "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIPv6 database: %s\n", err)
 	}
 
-	gi, err := geoip.Open("./GeoLiteCity.dat")
+	gi, err := geoip.Open("./GeoLiteCity.dat", "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIP database: %s\n", err)
 	}
 
-	giasn, err := geoip.Open("./GeoIPASNum.dat")
+	giasn, err := geoip.Open("./GeoIPASNum.dat", "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIPASN database: %s\n", err)
 	}
 
-	giasn6, err := geoip.Open("./GeoIPASNumv6.dat")
+	giasn6, err := geoip.Open("./GeoIPASNumv6.dat", "default")
 	if err != nil {
 		fmt.Printf("Could not open GeoIPASN database: %s\n", err)
 	}
@@ -38,7 +41,7 @@ func main() {
 	}
 
 	if gi != nil {
-		record := gi.GetRecord("207.171.7.51")
+		record := gi.GetRecord("207.171.7.51", true)
 		fmt.Printf("%v\n", record)
 	}
 	if gi6 != nil {
@@ -52,4 +55,10 @@ func main() {
 		fmt.Printf("%s: %s/%d %s/%d\n", ip, country, netmask, asn, asn_netmask)
 	}
 
+	gi6.Free()
+
+	if gi6.Check() {
+		fmt.Printf("Free() did not release gi6 memory correctly.\n")
+	}
+
 }