Use embedded GeoIP database

docs/howto/ingest_geoip.md

@@ -0,0 +1,27 @@
+# HOWTO: Use MaxMind's GeoIP database in tests
+
+Elasticsearch provides default GeoIP databases that can be downloaded in runtime and which weights ~70 MB. This can be
+a root cause of flakiness of package tests, so elastic-package embeds small samples of GeoIP databases, that can identify
+accurately only few ranges of IP addresses:
+
+```
+1.128.3.4
+175.16.199.1
+216.160.83.57
+216.160.83.61
+67.43.156.12
+81.2.69.143
+81.2.69.144
+81.2.69.145
+81.2.69.193
+89.160.20.112
+89.160.20.156
+67.43.156.12
+67.43.156.13
+67.43.156.14
+67.43.156.15
+2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6
+```
+
+If you want the ingest pipeline to include a "geo" section in the event, feel free to use one of above IP addresses.
+Embedded databases contain information about: cities, countries and ASNs.

internal/install/install.go

@@ -147,7 +147,6 @@ func createElasticPackageDirectory(elasticPackagePath *locations.LocationManager
 }
 
 func writeStackResources(elasticPackagePath *locations.LocationManager) error {
-
 	err := os.MkdirAll(elasticPackagePath.PackagesDir(), 0755)
 	if err != nil {
 		return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir())
@@ -158,10 +157,44 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error {
 		return errors.Wrapf(err, "creating directory failed (path: %s)", elasticPackagePath.PackagesDir())
 	}
 
-	resourcePath := filepath.Join(elasticPackagePath.StackDir(), "healthcheck.sh")
-	err = writeStaticResource(err, resourcePath, kibanaHealthcheckSh)
+	kibanaHealthcheckPath := filepath.Join(elasticPackagePath.StackDir(), "healthcheck.sh")
+	err = writeStaticResource(err, kibanaHealthcheckPath, kibanaHealthcheckSh)
+	if err != nil {
+		return errors.Wrapf(err, "copying healthcheck script failed (%s)", kibanaHealthcheckPath)
+	}
+
+	// Install GeoIP database
+	ingestGeoIPDir := filepath.Join(elasticPackagePath.StackDir(), "ingest-geoip")
+
+	// This directory is intended to be empty as we include GeoIP databases only in the 8x stack family.
+	ingestGeoIPDefaultDir := filepath.Join(ingestGeoIPDir, "default")
+	err = os.MkdirAll(ingestGeoIPDefaultDir, 0755)
+	if err != nil {
+		return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIPDefaultDir)
+	}
+
+	ingestGeoIP8xDir := filepath.Join(ingestGeoIPDir, "8x")
+	err = os.MkdirAll(ingestGeoIP8xDir, 0755)
+	if err != nil {
+		return errors.Wrapf(err, "creating directory failed (path: %s)", ingestGeoIP8xDir)
+	}
+
+	geoIpAsnMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-ASN.mmdb")
+	err = writeStaticResource(err, geoIpAsnMmdbPath, geoIpAsnMmdb)
+	if err != nil {
+		return errors.Wrapf(err, "copying GeoIP ASN database failed (%s)", geoIpAsnMmdbPath)
+	}
+
+	geoIpCityMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-City.mmdb")
+	err = writeStaticResource(err, geoIpCityMmdbPath, geoIpCityMmdb)
 	if err != nil {
-		return errors.Wrapf(err, "copying healthcheck script failed (%s)", resourcePath)
+		return errors.Wrapf(err, "copying GeoIP city database failed (%s)", geoIpCityMmdbPath)
+	}
+
+	geoIpCountryMmdbPath := filepath.Join(ingestGeoIP8xDir, "GeoLite2-Country.mmdb")
+	err = writeStaticResource(err, geoIpCountryMmdbPath, geoIpCountryMmdb)
+	if err != nil {
+		return errors.Wrapf(err, "copying GeoIP country database failed (%s)", geoIpCountryMmdbPath)
 	}
 
 	options := profile.Options{
@@ -170,7 +203,6 @@ func writeStackResources(elasticPackagePath *locations.LocationManager) error {
 		OverwriteExisting: false,
 	}
 	return profile.CreateProfile(options)
-
 }
 
 func writeTerraformDeployerResources(elasticPackagePath *locations.LocationManager) error {

internal/install/static.go

@@ -17,3 +17,12 @@ var terraformDeployerYml string
 
 //go:embed _static/terraform_deployer_run.sh
 var terraformDeployerRun string
+
+//go:embed _static/GeoLite2-ASN.mmdb
+var geoIpAsnMmdb string
+
+//go:embed _static/GeoLite2-City.mmdb
+var geoIpCityMmdb string
+
+//go:embed _static/GeoLite2-Country.mmdb
+var geoIpCountryMmdb string

internal/profile/_static/docker-compose-stack.yml

@@ -11,6 +11,7 @@ services:
       - "ELASTIC_PASSWORD=changeme"
     volumes:
       - "./elasticsearch.config.${STACK_VERSION_VARIANT}.yml:/usr/share/elasticsearch/config/elasticsearch.yml"
+      - "../../../stack/ingest-geoip/${STACK_VERSION_VARIANT}/:/usr/share/elasticsearch/config/ingest-geoip"
     ports:
       - "127.0.0.1:9200:9200"