Add internal DensityIncome database and new range for Precision City

maxmind · Sep 19, 2016 · e318308 · e318308
1 parent 7515690
commit e318308
Show file tree

Hide file tree

Showing 33 changed files with 129 additions and 50 deletions.
diff --git a/source-data/GeoIP2-DensityIncome-Test.json b/source-data/GeoIP2-DensityIncome-Test.json
@@ -0,0 +1,14 @@
+[
+   {
+      "::5.83.124.0/118" : {
+         "average_income" : 32323,
+         "population_density" : 1232
+      }
+   },
+   {
+      "::216.160.83.0/120" : {
+         "average_income" : 24626,
+         "population_density" : 1341
+      }
+   }
+]
diff --git a/source-data/GeoIP2-ISP-Test.json b/source-data/GeoIP2-ISP-Test.json
@@ -374,8 +374,8 @@
    },
    {
       "::16.0.0.0/104" : {
-         "$" : "Hewlett-Packard Company",
          "autonomous_system_number" : 71,
+         "autonomous_system_organization" : "Hewlett-Packard Company",
          "isp" : "Hewlett-Packard Company",
          "organization" : "Hewlett-Packard Company"
       }

diff --git a/source-data/GeoIP2-Precision-City-Test.json b/source-data/GeoIP2-Precision-City-Test.json
@@ -12099,7 +12099,7 @@
          },
          "subdivisions" : [
             {
-         "confidence": 70,
+               "confidence" : 70,
                "geoname_id" : 6269131,
                "iso_code" : "ENG",
                "names" : {
@@ -12668,6 +12668,27 @@
          ]
       }
    },
+   {
+      "::5.83.124.0/118" : {
+         "registered_country" : {
+            "geoname_id" : 798549,
+            "iso_code" : "RO",
+            "names" : {
+               "de" : "Rumänien",
+               "en" : "Romania",
+               "es" : "Rumanía",
+               "fr" : "Roumanie",
+               "ja" : "ルーマニア",
+               "pt-BR" : "Romênia",
+               "ru" : "Румыния",
+               "zh-CN" : "罗马尼亚"
+            }
+         },
+         "traits" : {
+            "is_anonymous_proxy" : 1
+         }
+      }
+   },
    {
       "::67.43.156.0/120" : {
          "continent" : {

diff --git a/source-data/GeoIP2-Precision-ISP-Test.json b/source-data/GeoIP2-Precision-ISP-Test.json
@@ -374,8 +374,8 @@
    },
    {
       "::16.0.0.0/104" : {
-         "$" : "Hewlett-Packard Company",
          "autonomous_system_number" : 71,
+         "autonomous_system_organization" : "Hewlett-Packard Company",
          "isp" : "Hewlett-Packard Company",
          "organization" : "Hewlett-Packard Company"
       }

diff --git a/test-data/GeoIP2-Anonymous-IP-Test.mmdb b/test-data/GeoIP2-Anonymous-IP-Test.mmdb
diff --git a/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb b/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb
diff --git a/test-data/GeoIP2-City-Test-Invalid-Node-Count.mmdb b/test-data/GeoIP2-City-Test-Invalid-Node-Count.mmdb
diff --git a/test-data/GeoIP2-City-Test.mmdb b/test-data/GeoIP2-City-Test.mmdb
diff --git a/test-data/GeoIP2-Connection-Type-Test.mmdb b/test-data/GeoIP2-Connection-Type-Test.mmdb
diff --git a/test-data/GeoIP2-Country-Test.mmdb b/test-data/GeoIP2-Country-Test.mmdb
diff --git a/test-data/GeoIP2-DensityIncome-Test.mmdb b/test-data/GeoIP2-DensityIncome-Test.mmdb
diff --git a/test-data/GeoIP2-Domain-Test.mmdb b/test-data/GeoIP2-Domain-Test.mmdb
diff --git a/test-data/GeoIP2-Enterprise-Test.mmdb b/test-data/GeoIP2-Enterprise-Test.mmdb
diff --git a/test-data/GeoIP2-ISP-Test.mmdb b/test-data/GeoIP2-ISP-Test.mmdb
diff --git a/test-data/GeoIP2-Precision-City-Test.mmdb b/test-data/GeoIP2-Precision-City-Test.mmdb
diff --git a/test-data/GeoIP2-Precision-ISP-Test.mmdb b/test-data/GeoIP2-Precision-ISP-Test.mmdb
diff --git a/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb b/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb
diff --git a/test-data/MaxMind-DB-string-value-entries.mmdb b/test-data/MaxMind-DB-string-value-entries.mmdb
diff --git a/test-data/MaxMind-DB-test-broken-pointers-24.mmdb b/test-data/MaxMind-DB-test-broken-pointers-24.mmdb
diff --git a/test-data/MaxMind-DB-test-broken-search-tree-24.mmdb b/test-data/MaxMind-DB-test-broken-search-tree-24.mmdb
diff --git a/test-data/MaxMind-DB-test-decoder.mmdb b/test-data/MaxMind-DB-test-decoder.mmdb
diff --git a/test-data/MaxMind-DB-test-ipv4-24.mmdb b/test-data/MaxMind-DB-test-ipv4-24.mmdb
diff --git a/test-data/MaxMind-DB-test-ipv4-28.mmdb b/test-data/MaxMind-DB-test-ipv4-28.mmdb
diff --git a/test-data/MaxMind-DB-test-ipv4-32.mmdb b/test-data/MaxMind-DB-test-ipv4-32.mmdb
diff --git a/test-data/MaxMind-DB-test-ipv6-24.mmdb b/test-data/MaxMind-DB-test-ipv6-24.mmdb
diff --git a/test-data/MaxMind-DB-test-ipv6-28.mmdb b/test-data/MaxMind-DB-test-ipv6-28.mmdb
diff --git a/test-data/MaxMind-DB-test-ipv6-32.mmdb b/test-data/MaxMind-DB-test-ipv6-32.mmdb
diff --git a/test-data/MaxMind-DB-test-metadata-pointers.mmdb b/test-data/MaxMind-DB-test-metadata-pointers.mmdb
diff --git a/test-data/MaxMind-DB-test-mixed-24.mmdb b/test-data/MaxMind-DB-test-mixed-24.mmdb
diff --git a/test-data/MaxMind-DB-test-mixed-28.mmdb b/test-data/MaxMind-DB-test-mixed-28.mmdb
diff --git a/test-data/MaxMind-DB-test-mixed-32.mmdb b/test-data/MaxMind-DB-test-mixed-32.mmdb
diff --git a/test-data/MaxMind-DB-test-nested.mmdb b/test-data/MaxMind-DB-test-nested.mmdb
diff --git a/test-data/write-test-data.pl b/test-data/write-test-data.pl
@@ -335,6 +335,7 @@ sub write_geoip2_dbs {
         [ 'City',            0 ],
         [ 'Connection-Type', 0 ],
         [ 'Country',         0 ],
+        [ 'DensityIncome',   0 ],
         [ 'Domain',          0 ],
         [ 'Enterprise',      0 ],
         [ 'ISP',             0 ],
@@ -367,83 +368,126 @@ sub write_invalid_node_count {
     _write_geoip2_db( 'City', 0, 'Test Invalid Node Count' );
 }
 
-{
-    my %type_map = (
+sub _universal_map_key_type_callback {
+    my $map = {
+
+        # languages
+        de      => 'utf8_string',
+        en      => 'utf8_string',
+        es      => 'utf8_string',
+        fr      => 'utf8_string',
+        ja      => 'utf8_string',
+        'pt-BR' => 'utf8_string',
+        ru      => 'utf8_string',
+        'zh-CN' => 'utf8_string',
+
+        # production
         accuracy_radius                => 'uint16',
         autonomous_system_number       => 'uint32',
         autonomous_system_organization => 'utf8_string',
-        cellular                       => 'uint16',
+        average_income                 => 'uint32',
         city                           => 'map',
+        code                           => 'utf8_string',
         confidence                     => 'uint16',
+        connection_type                => 'utf8_string',
         continent                      => 'map',
         country                        => 'map',
+        domain                         => 'utf8_string',
         geoname_id                     => 'uint32',
+        is_anonymous                   => 'boolean',
+        is_anonymous_proxy             => 'boolean',
+        is_anonymous_vpn               => 'boolean',
+        is_hosting_provider            => 'boolean',
+        is_legitimate_proxy            => 'boolean',
+        is_public_proxy                => 'boolean',
+        is_satellite_provider          => 'boolean',
+        is_tor_exit_node               => 'boolean',
+        iso_code                       => 'utf8_string',
         isp                            => 'utf8_string',
         latitude                       => 'double',
         location                       => 'map',
         longitude                      => 'double',
         metro_code                     => 'uint16',
         names                          => 'map',
         organization                   => 'utf8_string',
+        population_density             => 'uint32',
         postal                         => 'map',
         registered_country             => 'map',
         represented_country            => 'map',
         subdivisions                   => [ 'array', 'map' ],
+        time_zone                      => 'utf8_string',
         traits                         => 'map',
-    );
+        traits                         => 'map',
+        type                           => 'utf8_string',
+        user_type                      => 'utf8_string',
+
+        # for testing only
+        foo       => 'utf8_string',
+        bar       => 'utf8_string',
+        buzz      => 'utf8_string',
+        our_value => 'utf8_string',
+    };
+
+    my $callback = sub {
+        my $key = shift;
 
-    my $type_cb = sub {
-        return 'boolean' if $_[0] =~ /^is_/;
-        return $type_map{ $_[0] } // 'utf8_string';
+        return $map->{$key} || die <<"ERROR";
+Unknown tree key '$key'.
+
+The universal_map_key_type_callback doesn't know what type to use for the passed
+key.  If you are adding a new key that will be used in a frozen tree / mmdb then
+you should update the mapping in MM::GeoIP2::Util to provide a type for that
+value.
+ERROR
     };
 
-    sub _write_geoip2_db {
-        my $type                  = shift;
-        my $populate_all_networks = shift;
-        my $description           = shift;
+    return $callback;
+}
+
+sub _write_geoip2_db {
+    my $type                  = shift;
+    my $populate_all_networks = shift;
+    my $description           = shift;
 
-        my $writer = MaxMind::DB::Writer::Tree->new(
-            ip_version    => 6,
-            record_size   => 28,
-            ip_version    => 6,
-            database_type => "GeoIP2-$type",
-            languages     => [ 'en', $type eq 'City' ? ('zh') : () ],
-            description   => {
-                en =>
-                    "GeoIP2 $type $description Database (fake GeoIP2 data, for example purposes only)",
-                $type eq 'City' ? ( zh => '小型数据库' ) : (),
-            },
-            alias_ipv6_to_ipv4    => 1,
-            map_key_type_callback => $type_cb,
-        );
+    my $writer = MaxMind::DB::Writer::Tree->new(
+        ip_version    => 6,
+        record_size   => 28,
+        ip_version    => 6,
+        database_type => "GeoIP2-$type",
+        languages     => [ 'en', $type eq 'City' ? ('zh') : () ],
+        description   => {
+            en =>
+                "GeoIP2 $type $description Database (fake GeoIP2 data, for example purposes only)",
+            $type eq 'City' ? ( zh => '小型数据库' ) : (),
+        },
+        alias_ipv6_to_ipv4    => 1,
+        map_key_type_callback => _universal_map_key_type_callback(),
+    );
 
-        _populate_all_networks($writer) if $populate_all_networks;
+    _populate_all_networks($writer) if $populate_all_networks;
 
-        my $nodes = decode_json(
-            read_file(
-                "$Dir/../source-data/GeoIP2-$type-Test.json",
-                binmode => ':raw'
-            )
-        );
+    my $nodes = decode_json(
+        read_file(
+            "$Dir/../source-data/GeoIP2-$type-Test.json",
+            binmode => ':raw'
+        )
+    );
 
-        for my $node (@$nodes) {
-            for my $network ( keys %$node ) {
-                $writer->insert_network(
-                    Net::Works::Network->new_from_string(
-                        string => $network
-                    ),
-                    $node->{$network}
-                );
-            }
+    for my $node (@$nodes) {
+        for my $network ( keys %$node ) {
+            $writer->insert_network(
+                Net::Works::Network->new_from_string( string => $network ),
+                $node->{$network}
+            );
         }
+    }
 
-        my $suffix = $description =~ s/ /-/gr;
-        open my $output_fh, '>', "$Dir/GeoIP2-$type-$suffix.mmdb";
-        $writer->write_tree($output_fh);
-        close $output_fh;
+    my $suffix = $description =~ s/ /-/gr;
+    open my $output_fh, '>', "$Dir/GeoIP2-$type-$suffix.mmdb";
+    $writer->write_tree($output_fh);
+    close $output_fh;
 
-        return;
-    }
+    return;
 }
 
 sub _populate_all_networks {