HDDS-2588. Consolidate compose environments

hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml

@@ -18,30 +18,30 @@ version: "3"
 services:
    datanode:
       image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
-      privileged: true #required by the profiler
+      privileged: true # required by the profiler
       volumes:
         - ../..:/opt/hadoop
       ports:
         - 9864
         - 9882
-      command: ["/opt/hadoop/bin/ozone","datanode"]
+      command: ["ozone","datanode"]
       env_file:
         - ./docker-config
    om:
       image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
-      privileged: true #required by the profiler
+      privileged: true
       volumes:
-         - ../..:/opt/hadoop
+        - ../..:/opt/hadoop
       ports:
          - 9874:9874
       environment:
          ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
       env_file:
           - ./docker-config
-      command: ["/opt/hadoop/bin/ozone","om"]
+      command: ["ozone","om"]
    scm:
       image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
-      privileged: true #required by the profiler
+      privileged: true # required by the profiler
       volumes:
          - ../..:/opt/hadoop
       ports:
@@ -50,4 +50,48 @@ services:
           - ./docker-config
       environment:
           ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
-      command: ["/opt/hadoop/bin/ozone","scm"]
+      command: ["ozone","scm"]
+   jaeger:
+     image: jaegertracing/all-in-one:latest
+     environment:
+       COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+     ports:
+       - 16686:16686
+   prometheus:
+     image: prom/prometheus
+     volumes:
+       - "./prometheus.yml:/etc/prometheus.yml"
+     command: ["--config.file","/etc/prometheus.yml"]
+     ports:
+        - 9090:9090
+   grafana:
+      image: grafana/grafana
+      volumes:
+         - "../common/grafana/dashboards:/var/lib/grafana/dashboards"
+         - "../common/grafana/provisioning:/etc/grafana/provisioning"
+         - "../common/grafana/conf/grafana.ini:/etc/grafana/grafana.ini"
+      command: ["-config","/etc/grafana/grafana.ini"]
+      ports:
+         - 3000:3000
+   s3g:
+     image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+     privileged: true # required by the profiler
+     volumes:
+       - ../..:/opt/hadoop
+     ports:
+       - 9878:9878
+     env_file:
+       - ./docker-config
+     command: ["ozone","s3g"]
+   recon:
+      image: apache/ozone-runner:${OZONE_RUNNER_VERSION}
+      privileged: true # required by the profiler
+      volumes:
+         - ../..:/opt/hadoop
+      ports:
+         - 9888:9888
+      env_file:
+         - ./docker-config
+      environment:
+         WAITFOR: om:9874
+      command: ["ozone","recon"]

hadoop-ozone/dist/src/main/compose/ozone/docker-config

@@ -20,14 +20,22 @@ OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
 OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
+OZONE-SITE.XML_ozone.recon.om.db.dir=/data/metadata/recon
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=3
 OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
+OZONE-SITE.XML_ozone.replication=3
+OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true
 OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+
+ASYNC_PROFILER_HOME=/opt/profiler
+
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-ASYNC_PROFILER_HOME=/opt/profiler
+
+JAEGER_SAMPLER_PARAM=1
+JAEGER_SAMPLER_TYPE=const
+JAEGER_AGENT_HOST=jaeger
 
 #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
 #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm

hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml → hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml

@@ -23,13 +23,13 @@ scrape_configs:
      - targets:
         - "scm:9876"
         - "om:9874"
-        - "ozoneperf_datanode_1:9882"
-        - "ozoneperf_datanode_2:9882"
-        - "ozoneperf_datanode_3:9882"
-        - "ozoneperf_datanode_4:9882"
-        - "ozoneperf_datanode_5:9882"
-        - "ozoneperf_datanode_6:9882"
-        - "ozoneperf_datanode_7:9882"
-        - "ozoneperf_datanode_8:9882"
-        - "ozoneperf_datanode_9:9882"
-        - "ozoneperf_datanode_10:9882"
+        - "ozone_datanode_1:9882"
+        - "ozone_datanode_2:9882"
+        - "ozone_datanode_3:9882"
+        - "ozone_datanode_4:9882"
+        - "ozone_datanode_5:9882"
+        - "ozone_datanode_6:9882"
+        - "ozone_datanode_7:9882"
+        - "ozone_datanode_8:9882"
+        - "ozone_datanode_9:9882"
+        - "ozone_datanode_10:9882"

hadoop-ozone/dist/src/main/compose/ozone/test.sh

@@ -18,6 +18,8 @@
 COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 export COMPOSE_DIR
 
+export SECURITY_ENABLED=false
+
 # shellcheck source=/dev/null
 source "$COMPOSE_DIR/../testlib.sh"
 
@@ -29,9 +31,11 @@ start_docker_env
 #Disabling for now, audit parser tool during parse getting exception.
 #execute_robot_test om auditparser
 
-execute_robot_test scm basic/basic.robot
+execute_robot_test scm basic
+
+execute_robot_test scm gdpr
 
-execute_robot_test scm gdpr/gdpr.robot
+execute_robot_test scm s3
 
 stop_docker_env