Support per-module docker compose files for metricbeat tests

libbeat/tests/compose/wrapper.go

@@ -300,21 +300,44 @@ func (d *wrapperDriver) containers(ctx context.Context, projectFilter Filter, fi
 		}
 	}
 
+	serviceNames, err := d.serviceNames(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get container list")
+	}
+
 	var containers []types.Container
 	for _, f := range serviceFilters {
-		c, err := d.client.ContainerList(ctx, types.ContainerListOptions{
+		list, err := d.client.ContainerList(ctx, types.ContainerListOptions{
 			All:     true,
 			Filters: f,
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to get container list")
 		}
-		containers = append(containers, c...)
+		for _, container := range list {
+			serviceName, ok := container.Labels[labelComposeService]
+			if !ok || !contains(serviceNames, serviceName) {
+				// Service is not defined in current docker compose file, ignore it
+				continue
+			}
+			containers = append(containers, container)
+		}
 	}
 
 	return containers, nil
 }
 
+func (d *wrapperDriver) serviceNames(ctx context.Context) ([]string, error) {
+	var stdout bytes.Buffer
+	cmd := d.cmd(ctx, "config", "--services")
+	cmd.Stdout = &stdout
+	err := cmd.Run()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get list of service names")
+	}
+	return strings.Fields(stdout.String()), nil
+}
+
 func makeFilter(project, service string, projectFilter Filter) filters.Args {
 	f := filters.NewArgs()
 	f.Add("label", fmt.Sprintf("%s=%s", labelComposeProject, project))

libbeat/tests/system/beat/compose.py

@@ -22,8 +22,8 @@ class ComposeMixin(object):
     # List of required services to run INTEGRATION_TESTS
     COMPOSE_SERVICES = []
 
-    # docker-compose.yml dir path
-    COMPOSE_PROJECT_DIR = '.'
+    # Additional build arguments for docker compose build
+    COMPOSE_BUILD_ARGS = {}
 
     # timeout waiting for health (seconds)
     COMPOSE_TIMEOUT = 300
@@ -51,10 +51,13 @@ def is_healthy(container):
             return container.inspect()['State']['Health']['Status'] == 'healthy'
 
         project = cls.compose_project()
+        project.build(
+            service_names=cls.COMPOSE_SERVICES,
+            build_args=cls.COMPOSE_BUILD_ARGS)
         project.up(
             strategy=ConvergenceStrategy.always,
             service_names=cls.COMPOSE_SERVICES,
-            do_build=BuildAction.force,
+            do_build=BuildAction.skip,
             timeout=30)
 
         # Wait for them to be healthy
@@ -126,7 +129,11 @@ def compose_down(cls):
             return
 
         if INTEGRATION_TESTS and cls.COMPOSE_SERVICES:
-            cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
+            # Use down on per-module scenarios to release network pools too
+            if os.path.basename(os.path.dirname(cls.find_compose_path())) == "module":
+                cls.compose_project().down(remove_image_type=None, include_volumes=True)
+            else:
+                cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
 
     @classmethod
     def get_hosts(cls):
@@ -182,6 +189,25 @@ def compose_host(cls, service=None, port=None):
             return cls._private_host(info, port)
         return cls._exposed_host(info, port)
 
+    @classmethod
+    def compose_project_name(cls):
+        basename = os.path.basename(cls.find_compose_path())
+
+        def positivehash(x):
+            return hash(x) % ((sys.maxsize+1) * 2)
+
+        return "%s_%X" % (basename, positivehash(frozenset(cls.COMPOSE_BUILD_ARGS.items())))
+
     @classmethod
     def compose_project(cls):
-        return get_project(cls.COMPOSE_PROJECT_DIR, project_name=os.environ.get('DOCKER_COMPOSE_PROJECT_NAME'))
+        return get_project(cls.find_compose_path(), project_name=cls.compose_project_name())
+
+    @classmethod
+    def find_compose_path(cls):
+        class_dir = os.path.abspath(os.path.dirname(sys.modules[cls.__module__].__file__))
+        while True:
+            if os.path.exists(os.path.join(class_dir, "docker-compose.yml")):
+                return class_dir
+            class_dir, current = os.path.split(class_dir)
+            if current == '':  # We have reached root
+                raise Exception("failed to find a docker-compose.yml file")

metricbeat/docker-compose.yml

@@ -19,19 +19,6 @@ services:
     ports:
       - 3000
 
-  apache:
-    build: ./module/apache/_meta
-    ports:
-      - 80
-
-  apache_2_4_12:
-    build:
-      context: ./module/apache/_meta
-      args:
-        APACHE_VERSION: 2.4.12
-    ports:
-      - 80
-
   metricbeat:
     build: ./module/beat/_meta
     ports:

metricbeat/module/apache/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '2.3'
+
+services:
+  apache:
+    build: ./_meta
+    ports:
+      - 80

metricbeat/tests/system/test_apache.py → metricbeat/module/apache/test_apache.py

@@ -1,9 +1,12 @@
 import os
-import metricbeat
 import unittest
 from nose.plugins.attrib import attr
 import urllib2
 import time
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system'))
+import metricbeat
 
 APACHE_FIELDS = metricbeat.COMMON_FIELDS + ["apache"]
 
@@ -83,7 +86,7 @@ def get_hosts(self):
 
 class ApacheOldStatusTest(ApacheStatusTest):
 
-    COMPOSE_SERVICES = ['apache_2_4_12']
+    COMPOSE_BUILD_ARGS = {'APACHE_VERSION': '2.4.12'}
 
     def verify_fields(self, evt):
         self.assertItemsEqual(self.de_dot(APACHE_FIELDS), evt.keys())