[dbnode] Remove namespaces from example config and integration tests (#…

…2866)
m3db · Nov 10, 2020 · 5b5c050 · 5b5c050
1 parent a0a4c61
commit 5b5c050
Show file tree

Hide file tree

Showing 33 changed files with 270 additions and 241 deletions.
diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml
@@ -12,15 +12,7 @@ carbon:
             retention: 6h
 
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        resolution: 10s
-        retention: 6h
-      - namespace: unagg
-        type: unaggregated
-        retention: 1s
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh
@@ -29,7 +29,7 @@ function defer {
 trap defer EXIT
 
 echo "Setup DB node"
-setup_single_m3db_node
+AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node
 
 echo "Initializing aggregator topology"
 curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{

diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
@@ -12,15 +12,7 @@ carbon:
             retention: 6h
 
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        resolution: 10s
-        retention: 6h
-      - namespace: unagg
-        type: unaggregated
-        retention: 1s
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh
@@ -22,7 +22,7 @@ function defer {
 trap defer EXIT
 
 echo "Setup DB node"
-setup_single_m3db_node
+AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node
 
 echo "Initializing aggregator topology"
 curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{

diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml
@@ -1,13 +1,5 @@
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        retention: 10h
-        resolution: 5s
-      - namespace: unagg
-        type: unaggregated
-        retention: 10m
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh
@@ -19,7 +19,7 @@ function defer {
 }
 trap defer EXIT
 
-setup_single_m3db_node
+AGG_RESOLUTION=5s setup_single_m3db_node
 
 function read_carbon {
   target=$1

diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
@@ -3,15 +3,7 @@ limits:
     maxFetchedSeries: 100
 
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        retention: 10h
-        resolution: 15s
-      - namespace: unagg
-        type: unaggregated
-        retention: 10h
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/common.sh b/scripts/docker-integration-tests/common.sh
@@ -74,10 +74,32 @@ function setup_single_m3db_node_long_namespaces {
     ]
   }'
 
+  echo "Updating aggregation options for agg namespace"
+  curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
+    "name": "agg",
+    "options": {
+      "aggregationOptions": {
+        "aggregations": [
+           {
+            "aggregated": true,
+            "attributes": {
+              "resolutionDuration": "30s",
+              "downsampleOptions": { "all": false }
+            }
+          }
+        ]
+      }
+    }
+  }'
+
   echo "Wait until placement is init'd"
   ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
     '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'
 
+  echo "Wait until agg namespace is ready"
+  ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
+
   wait_for_namespaces
 
   echo "Adding agg2d namespace"
@@ -86,10 +108,31 @@ function setup_single_m3db_node_long_namespaces {
     "retentionTime": "48h"
   }'
 
+  echo "Updating aggregation options for agg namespace"
+  curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
+    "name": "agg2d",
+    "options": {
+      "aggregationOptions": {
+        "aggregations": [
+          {
+            "aggregated": true,
+            "attributes": {
+              "resolutionDuration": "1m",
+              "downsampleOptions": { "all": false }
+            }
+          }
+        ]
+      }
+    }
+  }'
+
   echo "Wait until agg2d namespace is init'd"
   ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
     '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.agg2d.indexOptions.enabled)" == true ]'
 
+  echo "Wait until agg2d namespace is ready"
+  ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg2d\"}" | grep -c true)" -eq 1 ]'
 
   echo "Wait until bootstrapped"
   ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
@@ -103,6 +146,8 @@ function setup_single_m3db_node {
   local dbnode_id=${DBNODE_ID:-m3db_local}
   local coordinator_port=${COORDINATOR_PORT:-7201}
   local zone=${ZONE:-embedded}
+  local agg_resolution=${AGG_RESOLUTION:-15s}
+  local agg_retention=${AGG_RETENTION:-10h}
 
   echo "Wait for API to be available"
   ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
@@ -112,7 +157,7 @@ function setup_single_m3db_node {
   curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
     "type": "cluster",
     "namespaceName": "agg",
-    "retentionTime": "6h",
+    "retentionTime": "'${agg_retention}'",
     "num_shards": 4,
     "replicationFactor": 1,
     "hosts": [
@@ -127,10 +172,31 @@ function setup_single_m3db_node {
     ]
   }'
 
+  echo "Updating aggregation options for agg namespace"
+  curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
+    "name": "agg",
+    "options": {
+      "aggregationOptions": {
+        "aggregations": [
+          {
+            "aggregated": true,
+            "attributes": {
+              "resolutionDuration": "'${agg_resolution}'"
+            }
+          }
+        ]
+      }
+    }
+  }'
+
   echo "Wait until placement is init'd"
   ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
     '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'${dbnode_id}'.id)" == \"'${dbnode_id}'\" ]'
 
+  echo "Wait until agg namespace is ready"
+  ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
+
   wait_for_namespaces
 
   echo "Wait until bootstrapped"
@@ -147,6 +213,8 @@ function setup_two_m3db_nodes {
   local dbnode_host_1_health_port=${DBNODE_HEALTH_PORT_01:-9012}
   local dbnode_host_2_health_port=${DBNODE_HEALTH_PORT_02:-9022}
   local coordinator_port=${COORDINATOR_PORT:-7201}
+  local agg_resolution=${AGG_RESOLUTION:-15s}
+  local agg_retention=${AGG_RETENTION:-10h}
 
   echo "Wait for API to be available"
   ATTEMPTS=100 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
@@ -156,7 +224,7 @@ function setup_two_m3db_nodes {
   curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/create -d '{
     "type": "cluster",
     "namespaceName": "agg",
-    "retentionTime": "6h",
+    "retentionTime": "'${agg_retention}'",
     "num_shards": 2,
     "replicationFactor": 2,
     "hosts": [
@@ -179,10 +247,31 @@ function setup_two_m3db_nodes {
     ]
   }'
 
+  echo "Updating aggregation options for agg namespace"
+  curl -vvvsSf -X PUT 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
+    "name": "agg",
+    "options": {
+      "aggregationOptions": {
+        "aggregations": [
+          {
+            "aggregated": true,
+            "attributes": {
+              "resolutionDuration": "'${agg_resolution}'"
+            }
+          }
+        ]
+      }
+    }
+  }'
+
   echo "Wait until placement is init'd"
   ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
     '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/placement | jq .placement.instances.'"${dbnode_id_1}"'.id)" == \"'"${dbnode_id_1}"'\" ]'
 
+  echo "Wait until agg namespace is ready"
+  ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"agg\"}" | grep -c true)" -eq 1 ]'
+
   wait_for_namespaces
 
   echo "Wait until bootstrapped"
@@ -194,6 +283,7 @@ function setup_two_m3db_nodes {
 
 function wait_for_namespaces {
   local coordinator_port=${COORDINATOR_PORT:-7201}
+  local unagg_retention=${UNAGG_RETENTION:-10h}
 
   echo "Wait until agg namespace is init'd"
   ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
@@ -202,13 +292,17 @@ function wait_for_namespaces {
   echo "Adding unagg namespace"
   curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/database/namespace/create -d '{
     "namespaceName": "unagg",
-    "retentionTime": "6h"
+    "retentionTime": "'${unagg_retention}'"
   }'
 
   echo "Wait until unagg namespace is init'd"
   ATTEMPTS=10 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
     '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]'
 
+  echo "Wait until unagg namespace is ready"
+  ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff  \
+    '[ "$(curl -sSf 0.0.0.0:'"${coordinator_port}"'/api/v1/services/m3db/namespace/ready -d "{ \"name\": \"unagg\"}" | grep -c true)" -eq 1 ]'
+
   echo "Adding coldWritesRepairAndNoIndex namespace"
   curl -vvvsSf -X POST 0.0.0.0:${coordinator_port}/api/v1/services/m3db/namespace -d '{
     "name": "coldWritesRepairAndNoIndex",
@@ -227,6 +321,11 @@ function wait_for_namespaces {
         "bufferPastDuration": "10m",
         "blockDataExpiry": true,
         "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
+      },
+      "aggregationOptions": {
+        "aggregations": [
+          { "aggregated": false }
+        ]
       }
     }
   }'

diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml
@@ -1,21 +1,5 @@
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        retention: 24h
-        resolution: 30s
-        downsample:
-          all: false
-      - namespace: agg2d
-        type: aggregated
-        retention: 48h
-        resolution: 1m
-        downsample:
-          all: false
-      - namespace: unagg
-        type: unaggregated
-        retention: 10m
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml
@@ -3,15 +3,7 @@ limits:
     maxFetchedSeries: 100
 
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        retention: 10h
-        resolution: 15s
-      - namespace: unagg
-        type: unaggregated
-        retention: 10h
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml
@@ -3,15 +3,7 @@ limits:
     maxFetchedSeries: 100
 
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        retention: 10h
-        resolution: 15s
-      - namespace: unagg
-        type: unaggregated
-        retention: 10h
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml
@@ -3,15 +3,7 @@ limits:
     maxFetchedSeries: 100
 
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        retention: 10h
-        resolution: 15s
-      - namespace: unagg
-        type: unaggregated
-        retention: 10m
-    client:
+  - client:
       config:
         service:
           env: default_env

diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml
@@ -4,15 +4,7 @@ writeForwarding:
     - url: http://coordinator02:7201/api/v1/prom/remote/write
 
 clusters:
-  - namespaces:
-      - namespace: agg
-        type: aggregated
-        retention: 10h
-        resolution: 15s
-      - namespace: unagg
-        type: unaggregated
-        retention: 10m
-    client:
+  - client:
       config:
         service:
           env: default_env