Support Kibana Spaces (#8045)

* Support Kibana Spaces via setup.kibana.space.id setting

* Adding CHANGELOG entry

* Add space-id option to export_dashboards script

* Add integration tests for importing and exporting dashboards with space ID

* Adding -space-id option for export_dashboards.go to documentation

* Adding entry to dev CHANGELOG

* Prefix space path with /

* Create Kibana space before trying to use it

* Make spaces tests conditional on version

* Add semver to requirements.txt

* Add kbn-xsrf header to POST request

* Don't recreate space in second space test

CHANGELOG-developer.asciidoc

@@ -53,3 +53,4 @@ The list below covers the major changes between 6.3.0 and master only.
   `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850]
 - Set current year in generator templates. {pull}8396[8396]
 - You can now override default settings of libbeat by using instance.Settings. {pull}8449[8449]
+- Add `-space-id` option to `export_dashboards.go` script to support Kibana Spaces {pull}7942[7942]

CHANGELOG.asciidoc

@@ -110,6 +110,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Added the `add_process_metadata` processor to enrich events with process information. {pull}6789[6789]
 - Report number of open file handles on Windows. {pull}8329[8329]
 - Support for Kafka 2.0.0 in kafka output {pull}8399[8399]
+- Add setting `setup.kibana.space.id` to support Kibana Spaces {pull}7942[7942]
 
 *Auditbeat*
 

auditbeat/auditbeat.yml

@@ -95,6 +95,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using auditbeat with the Elastic Cloud (https://cloud.elastic.co/).

dev-tools/cmd/dashboards/export_dashboards.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -48,11 +49,14 @@ func makeURL(url, path string, params url.Values) string {
 	return strings.Join([]string{url, path, "?", params.Encode()}, "")
 }
 
-func Export(client *http.Client, conn string, dashboard string, out string) error {
+func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error {
 	params := url.Values{}
 
 	params.Add("dashboard", dashboard)
 
+	if spaceID != "" {
+		exportAPI = path.Join("/s", spaceID, exportAPI)
+	}
 	fullURL := makeURL(conn, exportAPI, params)
 	if !quiet {
 		log.Printf("Calling HTTP GET %v\n", fullURL)
@@ -138,6 +142,7 @@ var quiet = false
 
 func main() {
 	kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL")
+	spaceID := flag.String("space-id", "", "Space ID")
 	dashboard := flag.String("dashboard", "", "Dashboard ID")
 	fileOutput := flag.String("output", "output.json", "Output file")
 	ymlFile := flag.String("yml", "", "Path to the module.yml file containing the dashboards")
@@ -171,7 +176,7 @@ func main() {
 			if err != nil {
 				log.Fatalf("fail to create directory %s: %v", directory, err)
 			}
-			err = Export(client, *kibanaURL, dashboard["id"], filepath.Join(directory, dashboard["file"]))
+			err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"]))
 			if err != nil {
 				log.Fatalf("fail to export the dashboards: %s", err)
 			}
@@ -180,7 +185,7 @@ func main() {
 	}
 
 	if len(*dashboard) > 0 {
-		err := Export(client, *kibanaURL, *dashboard, *fileOutput)
+		err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput)
 		if err != nil {
 			log.Fatalf("fail to export the dashboards: %s", err)
 		}

docs/devguide/newdashboards.asciidoc

@@ -283,6 +283,15 @@ go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system
 -------------------
 
 
+===== Export dashboards from a Kibana Space
+
+If you are using the Kibana Spaces feature and want to export dashboards from a specific Space, pass the Space ID to the `export_dashboards.go` script:
+
+[source,shell]
+-------------------
+go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options]
+-------------------
+
 
 ==== Exporting Kibana 5.x dashboards
 

filebeat/filebeat.yml

@@ -122,6 +122,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).

heartbeat/heartbeat.yml

@@ -69,6 +69,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using heartbeat with the Elastic Cloud (https://cloud.elastic.co/).

libbeat/_meta/config.yml

@@ -39,6 +39,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using beatname with the Elastic Cloud (https://cloud.elastic.co/).

libbeat/kibana/client.go

@@ -25,6 +25,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -88,7 +89,11 @@ func NewKibanaClient(cfg *common.Config) (*Client, error) {
 
 // NewClientWithConfig creates and returns a kibana client using the given config
 func NewClientWithConfig(config *ClientConfig) (*Client, error) {
-	kibanaURL, err := common.MakeURL(config.Protocol, config.Path, config.Host, 5601)
+	p := config.Path
+	if config.SpaceID != "" {
+		p = path.Join(p, "s", config.SpaceID)
+	}
+	kibanaURL, err := common.MakeURL(config.Protocol, p, config.Host, 5601)
 	if err != nil {
 		return nil, fmt.Errorf("invalid Kibana host: %v", err)
 	}

libbeat/kibana/client_config.go

@@ -28,6 +28,7 @@ type ClientConfig struct {
 	Protocol string            `config:"protocol"`
 	Host     string            `config:"host"`
 	Path     string            `config:"path"`
+	SpaceID  string            `config:"space.id"`
 	Username string            `config:"username"`
 	Password string            `config:"password"`
 	TLS      *tlscommon.Config `config:"ssl"`
@@ -39,6 +40,7 @@ var (
 		Protocol: "http",
 		Host:     "localhost:5601",
 		Path:     "",
+		SpaceID:  "",
 		Username: "",
 		Password: "",
 		Timeout:  90 * time.Second,

libbeat/tests/system/requirements.txt

@@ -29,3 +29,4 @@ urllib3==1.22
 websocket-client==0.47.0
 parameterized==0.6.1
 jsondiff==1.1.2
+semver==2.8.1

libbeat/tests/system/test_dashboard.py

@@ -4,7 +4,8 @@
 import subprocess
 from nose.plugins.attrib import attr
 import unittest
-
+import requests
+import semver
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 
@@ -36,6 +37,40 @@ def test_load_dashboard(self):
 
         assert self.log_contains("Kibana dashboards successfully loaded") is True
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_load_dashboard_into_space(self, create_space=True):
+        """
+        Test loading dashboards into Kibana space
+        """
+        version = self.get_version()
+        if semver.compare(version, "6.5.0") == -1:
+            # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available
+            raise SkipTest
+
+        self.render_config_template()
+        if create_space:
+            self.create_kibana_space()
+
+        beat = self.start_beat(
+            logging_args=["-e", "-d", "*"],
+            extra_args=["setup",
+                        "--dashboards",
+                        "-E", "setup.dashboards.file=" +
+                        os.path.join(self.beat_path, "tests", "files", "testbeat-dashboards.zip"),
+                        "-E", "setup.dashboards.beat=testbeat",
+                        "-E", "setup.kibana.protocol=http",
+                        "-E", "setup.kibana.host=" + self.get_kibana_host(),
+                        "-E", "setup.kibana.port=" + self.get_kibana_port(),
+                        "-E", "setup.kibana.space.id=foo-bar",
+                        "-E", "output.elasticsearch.hosts=['" + self.get_host() + "']",
+                        "-E", "output.file.enabled=false"]
+        )
+
+        beat.check_wait(exit_code=0)
+
+        assert self.log_contains("Kibana dashboards successfully loaded") is True
+
     @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
     @attr('integration')
     def test_load_only_index_patterns(self):
@@ -88,6 +123,36 @@ def test_export_dashboard(self):
 
         os.remove("output.json")
 
+    @unittest.skipUnless(INTEGRATION_TESTS, "integration test")
+    @attr('integration')
+    def test_export_dashboard_from_space(self):
+        """
+        Test export dashboards from Kibana space and remove unsupported characters
+        """
+        version = self.get_version()
+        if semver.compare(version, "6.5.0") == -1:
+            # Skip for Kibana versions < 6.5.0 as Kibana Spaces not available
+            raise SkipTest
+
+        self.test_load_dashboard_into_space(False)
+
+        path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go")
+        command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port()
+        command = "go run " + command + " -dashboard Metricbeat-system-overview -space-id foo-bar"
+
+        p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        content, err = p.communicate()
+
+        assert p.returncode == 0
+
+        assert os.path.isfile("output.json") is True
+
+        with open('output.json') as f:
+            content = f.read()
+            assert "Metricbeat-system-overview" in content
+
+        os.remove("output.json")
+
     def get_host(self):
         return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')
 
@@ -96,3 +161,28 @@ def get_kibana_host(self):
 
     def get_kibana_port(self):
         return os.getenv('KIBANA_PORT', '5601')
+
+    def create_kibana_space(self):
+        url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \
+            "/api/spaces/space"
+        data = {
+            "id": "foo-bar",
+            "name": "Foo bar space"
+        }
+
+        headers = {
+            "kbn-xsrf": "1"
+        }
+
+        r = requests.post(url, json=data, headers=headers)
+        assert r.status_code == 200
+
+    def get_version(self):
+        url = "http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + \
+            "/api/status"
+
+        r = requests.get(url)
+        body = r.json()
+        version = body["version"]["number"]
+
+        return version

metricbeat/metricbeat.yml

@@ -66,6 +66,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/).

packetbeat/packetbeat.yml

@@ -149,6 +149,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using packetbeat with the Elastic Cloud (https://cloud.elastic.co/).

winlogbeat/winlogbeat.yml

@@ -70,6 +70,11 @@ setup.kibana:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
   #host: "localhost:5601"
 
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
 #============================= Elastic Cloud ==================================
 
 # These settings simplify using winlogbeat with the Elastic Cloud (https://cloud.elastic.co/).