diff --git a/docs/howto/ingest_geoip.md b/docs/howto/ingest_geoip.md new file mode 100644 index 0000000000..51c956fc32 --- /dev/null +++ b/docs/howto/ingest_geoip.md @@ -0,0 +1,27 @@ +# HOWTO: Use MaxMind's GeoIP database in tests + +Elasticsearch provides default GeoIP databases that can be downloaded in runtime and which weights ~70 MB. This can be +a root cause of flakiness of package tests, so elastic-package embeds small samples of GeoIP databases, that can identify +accurately only few ranges of IP addresses: + +``` +1.128.3.4 +175.16.199.1 +216.160.83.57 +216.160.83.61 +67.43.156.12 +81.2.69.143 +81.2.69.144 +81.2.69.145 +81.2.69.193 +89.160.20.112 +89.160.20.156 +67.43.156.12 +67.43.156.13 +67.43.156.14 +67.43.156.15 +2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 +``` + +If you want the ingest pipeline to include a "geo" section in the event, feel free to use one of above IP addresses. +Embedded databases contain information about: cities, countries and ASNs. \ No newline at end of file diff --git a/internal/install/_static/GeoLite2-ASN.mmdb b/internal/install/_static/GeoLite2-ASN.mmdb new file mode 100644 index 0000000000..fd4a733ba0 Binary files /dev/null and b/internal/install/_static/GeoLite2-ASN.mmdb differ diff --git a/internal/install/_static/GeoLite2-City.mmdb b/internal/install/_static/GeoLite2-City.mmdb new file mode 100644 index 0000000000..0809201619 Binary files /dev/null and b/internal/install/_static/GeoLite2-City.mmdb differ diff --git a/internal/install/_static/GeoLite2-Country.mmdb b/internal/install/_static/GeoLite2-Country.mmdb new file mode 100644 index 0000000000..aa81cbe8a2 Binary files /dev/null and b/internal/install/_static/GeoLite2-Country.mmdb differ diff --git a/internal/install/install.go b/internal/install/install.go index a6d29406d9..fab0ec297e 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -147,7 +147,6 @@ func createElasticPackageDirectory(elasticPackagePath *locations.LocationManager } func writeStackResources(elasticPackagePath *locations.LocationManager) error { - err := os.MkdirAll(elasticPackagePath.PackagesDir(), 0755) if err != nil { return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir()) @@ -158,10 +157,44 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error { return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir()) } - resourcePath := filepath.Join(elasticPackagePath.StackDir(), "healthcheck.sh") - err = writeStaticResource(err, resourcePath, kibanaHealthcheckSh) + kibanaHealthcheckPath := filepath.Join(elasticPackagePath.StackDir(), "healthcheck.sh") + err = writeStaticResource(err, kibanaHealthcheckPath, kibanaHealthcheckSh) + if err != nil { + return errors.Wrapf(err, "copying healthcheck script failed (%s)", kibanaHealthcheckPath) + } + + // Install GeoIP database + ingestGeoIPDir := filepath.Join(elasticPackagePath.StackDir(), "ingest-geoip") + + // This directory is intended to be empty as we include GeoIP databases only in the 8x stack family. + ingestGeoIPDefaultDir := filepath.Join(ingestGeoIPDir, "default") + err = os.MkdirAll(ingestGeoIPDefaultDir, 0755) + if err != nil { + return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIPDefaultDir) + } + + ingestGeoIP8xDir := filepath.Join(ingestGeoIPDir, "8x") + err = os.MkdirAll(ingestGeoIP8xDir, 0755) + if err != nil { + return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIP8xDir) + } + + geoIpAsnMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-ASN.mmdb") + err = writeStaticResource(err, geoIpAsnMmdbPath, geoIpAsnMmdb) + if err != nil { + return errors.Wrapf(err, "copying GeoIP ASN database failed (%s)", geoIpAsnMmdbPath) + } + + geoIpCityMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-City.mmdb") + err = writeStaticResource(err, geoIpCityMmdbPath, geoIpCityMmdb) if err != nil { - return errors.Wrapf(err, "copying healthcheck script failed (%s)", resourcePath) + return errors.Wrapf(err, "copying GeoIP city database failed (%s)", geoIpCityMmdbPath) + } + + geoIpCountryMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-Country.mmdb") + err = writeStaticResource(err, geoIpCountryMmdbPath, geoIpCountryMmdb) + if err != nil { + return errors.Wrapf(err, "copying GeoIP country database failed (%s)", geoIpCountryMmdbPath) } options := profile.Options{ @@ -170,7 +203,6 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error { OverwriteExisting: false, } return profile.CreateProfile(options) - } func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error { diff --git a/internal/install/static.go b/internal/install/static.go index 20ad26ec63..540a566eb7 100644 --- a/internal/install/static.go +++ b/internal/install/static.go @@ -17,3 +17,12 @@ var terraformDeployerYml string //go:embed _static/terraform_deployer_run.sh var terraformDeployerRun string + +//go:embed _static/GeoLite2-ASN.mmdb +var geoIpAsnMmdb string + +//go:embed _static/GeoLite2-City.mmdb +var geoIpCityMmdb string + +//go:embed _static/GeoLite2-Country.mmdb +var geoIpCountryMmdb string diff --git a/internal/profile/_static/docker-compose-stack.yml b/internal/profile/_static/docker-compose-stack.yml index 97ba44268c..19109c27e2 100644 --- a/internal/profile/_static/docker-compose-stack.yml +++ b/internal/profile/_static/docker-compose-stack.yml @@ -11,6 +11,7 @@ services: - "ELASTIC_PASSWORD=changeme" volumes: - "./elasticsearch.config.${STACK_VERSION_VARIANT}.yml:/usr/share/elasticsearch/config/elasticsearch.yml" + - "../../../stack/ingest-geoip/${STACK_VERSION_VARIANT}/:/usr/share/elasticsearch/config/ingest-geoip" ports: - "127.0.0.1:9200:9200"