Change default backing index naming scheme (#57721)

elastic · Jun 8, 2020 · 904bdae · 904bdae
1 parent 76c7e32
commit 904bdae
Show file tree

Hide file tree

Showing 34 changed files with 192 additions and 157 deletions.
diff --git a/docs/reference/data-streams.asciidoc b/docs/reference/data-streams.asciidoc
@@ -20,7 +20,7 @@ template>> containing:
 
 * A name or wildcard pattern for the data stream in the `index_patterns` property.
 * A `data_stream` definition that contains the `timestamp_field` property.
-  The `timestamp_field` must be the primary timestamp field 
+  The `timestamp_field` must be the primary timestamp field
    for the data source. This field must be included in every
    document indexed to the data stream.
 
@@ -77,7 +77,7 @@ Response:
         "failed" : 0,
         "successful" : 1
     },
-    "_index" : "logs-foobar-000001",
+    "_index" : ".ds-logs-foobar-000001",
     "_id" : "W0tpsmIBdwcYyG50zbta",
     "_version" : 1,
     "_seq_no" : 0,

diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc
@@ -76,11 +76,11 @@ The API returns the following response:
     "timestamp_field" : "@timestamp", <2>
     "indices" : [ <3>
       {
-        "index_name" : "my-data-stream-000001",
+        "index_name" : ".ds-my-data-stream-000001",
         "index_uuid" : "DXAE-xcCQTKF93bMm9iawA"
       },
       {
-        "index_name" : "my-data-stream-000002",
+        "index_name" : ".ds-my-data-stream-000002",
         "index_uuid" : "Wzxq0VhsQKyPxHhaK3WYAg"
       }
     ],

diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
@@ -254,7 +254,7 @@ POST /my-data-stream/_rollover <2>
 --------------------------------------------------
 // TEST[continued]
 // TEST[setup:huge_twitter]
-// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"my-data-stream-000001"}}/]
+// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":".ds-my-data-stream-000001"}}/]
 <1> Creates a data stream called `my-data-stream` with one initial backing index
 named `my-data-stream-000001`.
 <2> This request creates a new backing index, `my-data-stream-000002`, and adds
@@ -274,8 +274,8 @@ The API returns the following response:
 {
   "acknowledged": true,
   "shards_acknowledged": true,
-  "old_index": "my-data-stream-000001", <1>
-  "new_index": "my-data-stream-000002", <2>
+  "old_index": ".ds-my-data-stream-000001", <1>
+  "new_index": ".ds-my-data-stream-000002", <2>
   "rolled_over": true, <3>
   "dry_run": false, <4>
   "conditions": { <5>

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
@@ -23,8 +23,8 @@ setup:
 ---
 "Create data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
 
   - do:
       indices.create_data_stream:
@@ -43,12 +43,12 @@ setup:
   - match: { 0.timestamp_field: '@timestamp' }
   - match: { 0.generation: 1 }
   - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: 'simple-data-stream1-000001' }
+  - match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
   - match: { 1.name: simple-data-stream2 }
   - match: { 1.timestamp_field: '@timestamp2' }
   - match: { 0.generation: 1 }
   - length: { 1.indices: 1 }
-  - match: { 1.indices.0.index_name: 'simple-data-stream2-000001' }
+  - match: { 1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
 
   - do:
       index:
@@ -64,7 +64,7 @@ setup:
         index: simple-data-stream1
         body: { query: { match_all: {} } }
   - length:   { hits.hits: 1  }
-  - match: { hits.hits.0._index: simple-data-stream1-000001 }
+  - match: { hits.hits.0._index: .ds-simple-data-stream1-000001 }
   - match: { hits.hits.0._source.foo: 'bar' }
 
   - do:
@@ -95,8 +95,8 @@ setup:
 ---
 "Get data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
 
   - do:
       indices.create_data_stream:
@@ -160,8 +160,8 @@ setup:
 ---
 "Delete data stream with backing indices":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
 
   - do:
       indices.create_data_stream:
@@ -178,18 +178,18 @@ setup:
 
   - do:
       indices.get:
-        index: ['simple-data-stream1-000001', 'test_index']
+        index: ['.ds-simple-data-stream1-000001', 'test_index']
 
   - is_true: test_index.settings
-  - is_true: simple-data-stream1-000001.settings
+  - is_true: \.ds-simple-data-stream1-000001.settings
 
   - do:
       indices.get_data_stream: {}
   - match: { 0.name: simple-data-stream1 }
   - match: { 0.timestamp_field: '@timestamp' }
   - match: { 0.generation: 1 }
   - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: 'simple-data-stream1-000001' }
+  - match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
 
   - do:
       indices.delete_data_stream:
@@ -199,4 +199,4 @@ setup:
   - do:
       catch: missing
       indices.get:
-        index: "simple-data-stream1-000001"
+        index: ".ds-simple-data-stream1-000001"
diff --git a/...pi-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml b/...pi-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml
@@ -1,8 +1,8 @@
 ---
 "Test apis that do not supported data streams":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
       features: allowed_warnings
 
   - do:
@@ -25,14 +25,14 @@
         index:  logs-foobar
         refresh: true
         body:   { foo: bar }
-  - match: {_index: logs-foobar-000001}
+  - match: {_index: .ds-logs-foobar-000001}
 
   - do:
       search:
         index: logs-foobar
         body: { query: { match_all: {} } }
   - length:   { hits.hits: 1  }
-  - match: { hits.hits.0._index: logs-foobar-000001 }
+  - match: { hits.hits.0._index: .ds-logs-foobar-000001 }
   - match: { hits.hits.0._source.foo: 'bar' }
 
   - do:

diff --git a/.../src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml b/.../src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml
@@ -1,8 +1,8 @@
 ---
 "Put index template":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
       features: allowed_warnings
 
   - do:
@@ -34,7 +34,7 @@
         index: logs-foobar
         body: { query: { match_all: {} } }
   - length:   { hits.hits: 1  }
-  - match: { hits.hits.0._index: logs-foobar-000001 }
+  - match: { hits.hits.0._index: .ds-logs-foobar-000001 }
   - match: { hits.hits.0._source.foo: 'bar' }
 
   - do:
@@ -43,7 +43,7 @@
   - match: { 0.name: logs-foobar }
   - match: { 0.timestamp_field: 'timestamp' }
   - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: 'logs-foobar-000001' }
+  - match: { 0.indices.0.index_name: '.ds-logs-foobar-000001' }
 
   - do:
       indices.delete_data_stream:

diff --git a/...-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml b/...-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml
@@ -31,9 +31,9 @@ teardown:
   - do:
       indices.get:
         index: logs-foobar
-  - is_true: logs-foobar-000001
+  - is_true: \.ds-logs-foobar-000001
   - is_false: logs-foobar
-  - match: { logs-foobar-000001.settings.index.number_of_shards: '1' }
+  - match: { \.ds-logs-foobar-000001.settings.index.number_of_shards: '1' }
 
 ---
 "Verify get mapping api":
@@ -44,5 +44,5 @@ teardown:
   - do:
       indices.get_mapping:
         index: logs-foobar
-  - is_true: logs-foobar-000001.mappings
-  - is_false: logs-foobar.mappings
+  - is_true: \.ds-logs-foobar-000001.mappings
+  - is_false: \.ds-logs-foobar.mappings
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml
@@ -14,8 +14,8 @@ setup:
 ---
 "Delete backing index on data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
 
   - do:
       indices.create_data_stream:
@@ -27,25 +27,25 @@ setup:
       indices.rollover:
         alias: "simple-data-stream"
 
-  - match: { old_index: simple-data-stream-000001 }
-  - match: { new_index: simple-data-stream-000002 }
+  - match: { old_index: .ds-simple-data-stream-000001 }
+  - match: { new_index: .ds-simple-data-stream-000002 }
   - match: { rolled_over: true }
   - match: { dry_run: false }
 
   # ensure new index is created
   - do:
       indices.exists:
-        index: simple-data-stream-000002
+        index: .ds-simple-data-stream-000002
 
   - is_true: ''
 
   - do:
       indices.delete:
-        index: simple-data-stream-000001
+        index: .ds-simple-data-stream-000001
 
   - do:
       indices.exists:
-        index: simple-data-stream-000001
+        index: .ds-simple-data-stream-000001
 
   - is_false: ''
 
@@ -56,7 +56,7 @@ setup:
   - match: { 0.timestamp_field: '@timestamp' }
   - match: { 0.generation: 2 }
   - length: { 0.indices: 1 }
-  - match: { 0.indices.0.index_name: 'simple-data-stream-000002' }
+  - match: { 0.indices.0.index_name: '.ds-simple-data-stream-000002' }
 
   - do:
       indices.delete_data_stream:
@@ -79,26 +79,26 @@ setup:
       indices.rollover:
         alias: "simple-data-stream"
 
-  - match: { old_index: simple-data-stream-000001 }
-  - match: { new_index: simple-data-stream-000002 }
+  - match: { old_index: .ds-simple-data-stream-000001 }
+  - match: { new_index: .ds-simple-data-stream-000002 }
   - match: { rolled_over: true }
   - match: { dry_run: false }
 
   # ensure new index is created
   - do:
       indices.exists:
-        index: simple-data-stream-000002
+        index: .ds-simple-data-stream-000002
 
   - is_true: ''
 
   - do:
       catch: bad_request
       indices.delete:
-        index: simple-data-stream-000002
+        index: .ds-simple-data-stream-000002
 
   - do:
       indices.exists:
-        index: simple-data-stream-000002
+        index: .ds-simple-data-stream-000002
 
   - is_true: ''
 

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml
@@ -1,8 +1,8 @@
 ---
 "Get backing indices for data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
       features: allowed_warnings
 
   - do:
@@ -30,11 +30,11 @@
 
   - do:
       indices.get:
-        index: ['data-stream1-000001', 'test_index']
+        index: ['.ds-data-stream1-000001', 'test_index']
 
-  - is_true: data-stream1-000001.settings
-  - is_true: data-stream1-000001.data_stream
-  - match: { data-stream1-000001.data_stream: data-stream1 }
+  - is_true: \.ds-data-stream1-000001.settings
+  - is_true: \.ds-data-stream1-000001.data_stream
+  - match: { \.ds-data-stream1-000001.data_stream: data-stream1 }
   - is_true: test_index.settings
   - is_false: test_index.data_stream
 

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml
@@ -141,7 +141,7 @@
   - do:
       catch: bad_request
       indices.close:
-        index: "simple-data-stream1-000001"
+        index: ".ds-simple-data-stream1-000001"
 
   - do:
       indices.delete_data_stream:

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml
@@ -1,8 +1,8 @@
 ---
 "Roll over a data stream":
   - skip:
-      version: " - 7.8.99"
-      reason: "data streams only supported in 7.9+"
+      version: " - 7.99.99"
+      reason: "change to 7.8.99 after backport"
       features: allowed_warnings
 
   - do:
@@ -25,15 +25,15 @@
       indices.rollover:
         alias: "data-stream-for-rollover"
 
-  - match: { old_index: data-stream-for-rollover-000001 }
-  - match: { new_index: data-stream-for-rollover-000002 }
+  - match: { old_index: .ds-data-stream-for-rollover-000001 }
+  - match: { new_index: .ds-data-stream-for-rollover-000002 }
   - match: { rolled_over: true }
   - match: { dry_run: false }
 
   # ensure new index is created
   - do:
       indices.exists:
-        index: data-stream-for-rollover-000002
+        index: .ds-data-stream-for-rollover-000002
 
   - is_true: ''
 
@@ -44,8 +44,8 @@
   - match: { 0.timestamp_field: '@timestamp' }
   - match: { 0.generation: 2 }
   - length: { 0.indices: 2 }
-  - match: { 0.indices.0.index_name: 'data-stream-for-rollover-000001' }
-  - match: { 0.indices.1.index_name: 'data-stream-for-rollover-000002' }
+  - match: { 0.indices.0.index_name: '.ds-data-stream-for-rollover-000001' }
+  - match: { 0.indices.1.index_name: '.ds-data-stream-for-rollover-000002' }
 
   - do:
       indices.delete_data_stream: