From 471789e99a72c4dd3691c7fc380120fe84445846 Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Thu, 31 Jan 2019 16:22:57 +0000 Subject: [PATCH 1/2] [ML] Automatically migrate saved objects created by data recognizer --- .../ml/server/models/data_recognizer/data_recognizer.js | 7 ++++++- .../ML-Apache2-Access-Remote-IP-Count-Explorer.json | 1 - .../dashboard/ML-Apache2-Remote-IP-URL-Explorer.json | 1 - .../apache2/kibana/search/ML-Filebeat-Apache2-Access.json | 1 - .../kibana/visualization/ML-Apache2-Access-Map.json | 1 - .../ML-Apache2-Access-Remote-IP-Timechart.json | 1 - .../ML-Apache2-Access-Response-Code-Timechart.json | 1 - .../ML-Apache2-Access-Top-Remote-IPs-Table.json | 1 - .../visualization/ML-Apache2-Access-Top-URLs-Table.json | 1 - .../ML-Apache2-Access-Unique-Count-URL-Timechart.json | 1 - 10 files changed, 6 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js index 5f97fc652108b4..646e64ffc175c7 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js @@ -385,7 +385,12 @@ export class DataRecognizer { let results = { saved_objects: [] }; const filteredSavedObjects = objectExistResults.filter(o => o.exists === false).map(o => o.savedObject); if (filteredSavedObjects.length) { - results = await this.savedObjectsClient.bulkCreate(filteredSavedObjects); + results = await this.savedObjectsClient.bulkCreate( + // Add an empty migrationVersion attribute to each saved object to ensure + // it is automatically migrated to the 7.0+ format with a references attribute. + // TODO: update module kibana objects to be in the 7.0+ format + filteredSavedObjects.map(doc => ({ ...doc, migrationVersion: doc.migrationVersion || { } })) + ); } return results.saved_objects; } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Access-Remote-IP-Count-Explorer.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Access-Remote-IP-Count-Explorer.json index 24b87e39b35dba..5fc696c6c702d3 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Access-Remote-IP-Count-Explorer.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Access-Remote-IP-Count-Explorer.json @@ -7,7 +7,6 @@ "panelsJSON": "[{\"size_x\":6,\"size_y\":3,\"panelIndex\":1,\"type\":\"visualization\",\"id\":\"ML-Apache2-Access-Remote-IP-Timechart\",\"col\":1,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":2,\"type\":\"visualization\",\"id\":\"ML-Apache2-Access-Response-Code-Timechart\",\"col\":7,\"row\":1},{\"size_x\":6,\"size_y\":3,\"panelIndex\":3,\"type\":\"visualization\",\"id\":\"ML-Apache2-Access-Top-Remote-IPs-Table\",\"col\":1,\"row\":4},{\"size_x\":6,\"size_y\":3,\"panelIndex\":4,\"type\":\"visualization\",\"id\":\"ML-Apache2-Access-Map\",\"col\":7,\"row\":4},{\"size_x\":12,\"size_y\":9,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Apache2-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]", "optionsJSON": "{}", "version": 1, - "migrationVersion": {}, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Remote-IP-URL-Explorer.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Remote-IP-URL-Explorer.json index d4ef153201bf23..b04050ceb6e19a 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Remote-IP-URL-Explorer.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/dashboard/ML-Apache2-Remote-IP-URL-Explorer.json @@ -7,7 +7,6 @@ "panelsJSON": "[{\"col\":1,\"id\":\"ML-Apache2-Access-Unique-Count-URL-Timechart\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Apache2-Access-Response-Code-Timechart\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"ML-Apache2-Access-Top-Remote-IPs-Table\",\"panelIndex\":3,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"ML-Apache2-Access-Map\",\"panelIndex\":4,\"row\":4,\"size_x\":6,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":8,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"ML-Apache2-Access-Top-URLs-Table\",\"col\":1,\"row\":7}]", "optionsJSON": "{}", "version": 1, - "migrationVersion": {}, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/search/ML-Filebeat-Apache2-Access.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/search/ML-Filebeat-Apache2-Access.json index 7c6124295aae3b..edb54752c2ffe2 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/search/ML-Filebeat-Apache2-Access.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/search/ML-Filebeat-Apache2-Access.json @@ -7,7 +7,6 @@ "description": "Filebeat Apache2 Access Data", "title": "ML Apache2 Access Data", "version": 1, - "migrationVersion": {}, "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"query\":{\"query_string\":{\"query\":\"_exists_:apache2.access\",\"analyze_wildcard\":true}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}" }, diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Map.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Map.json index c2df807f189855..cf95b968169413 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Map.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Map.json @@ -4,7 +4,6 @@ "title": "ML Apache2 Access Map", "uiStateJSON": "{\n \"mapCenter\": [\n 12.039320557540572,\n -0.17578125\n ]\n}", "version": 1, - "migrationVersion": {}, "savedSearchId": "ML-Filebeat-Apache2-Access", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"filter\":[]}" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Remote-IP-Timechart.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Remote-IP-Timechart.json index 0789ee1e03f6fd..c2db71b8c96679 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Remote-IP-Timechart.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Remote-IP-Timechart.json @@ -4,7 +4,6 @@ "title": "ML Apache2 Access Remote IP Timechart", "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", "version": 1, - "migrationVersion": {}, "savedSearchId": "ML-Filebeat-Apache2-Access", "kibanaSavedObjectMeta": { "searchSourceJSON": "{}" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Response-Code-Timechart.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Response-Code-Timechart.json index bc0a22685d5cfb..496d15dcda861b 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Response-Code-Timechart.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Response-Code-Timechart.json @@ -4,7 +4,6 @@ "title": "ML Apache2 Access Response Code Timechart", "uiStateJSON": "{\n \"vis\": {\n \"colors\": {\n \"200\": \"#7EB26D\",\n \"404\": \"#614D93\"\n }\n }\n}", "version": 1, - "migrationVersion": {}, "savedSearchId": "ML-Filebeat-Apache2-Access", "kibanaSavedObjectMeta": { "searchSourceJSON": "{\"filter\":[]}" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-Remote-IPs-Table.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-Remote-IPs-Table.json index 38943fd9ee6acc..856dec47b6b7b1 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-Remote-IPs-Table.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-Remote-IPs-Table.json @@ -4,7 +4,6 @@ "title": "ML Apache2 Access Top Remote IPs Table", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "version": 1, - "migrationVersion": {}, "savedSearchId": "ML-Filebeat-Apache2-Access", "kibanaSavedObjectMeta": { "searchSourceJSON": "{}" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-URLs-Table.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-URLs-Table.json index 406c314787c72a..b698cd12b303ed 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-URLs-Table.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Top-URLs-Table.json @@ -4,7 +4,6 @@ "title": "ML Apache2 Access Top URLs Table", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "version": 1, - "migrationVersion": {}, "savedSearchId": "ML-Filebeat-Apache2-Access", "kibanaSavedObjectMeta": { "searchSourceJSON": "{}" diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Unique-Count-URL-Timechart.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Unique-Count-URL-Timechart.json index 54d324434925c2..03ffb27206bd2e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Unique-Count-URL-Timechart.json +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/apache2/kibana/visualization/ML-Apache2-Access-Unique-Count-URL-Timechart.json @@ -4,7 +4,6 @@ "title": "ML Apache2 Access Unique Count URL Timechart", "uiStateJSON": "{}", "version": 1, - "migrationVersion": {}, "savedSearchId": "ML-Filebeat-Apache2-Access", "kibanaSavedObjectMeta": { "searchSourceJSON": "{}" From 5b9fa0c4447a7072786a6125d54a988f0c169bfa Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Thu, 31 Jan 2019 16:34:43 +0000 Subject: [PATCH 2/2] [ML] Remove TODO from saved object import step --- .../plugins/ml/server/models/data_recognizer/data_recognizer.js | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js index 646e64ffc175c7..817176cb787959 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.js @@ -388,7 +388,6 @@ export class DataRecognizer { results = await this.savedObjectsClient.bulkCreate( // Add an empty migrationVersion attribute to each saved object to ensure // it is automatically migrated to the 7.0+ format with a references attribute. - // TODO: update module kibana objects to be in the 7.0+ format filteredSavedObjects.map(doc => ({ ...doc, migrationVersion: doc.migrationVersion || { } })) ); }