more updates

.gitignore

@@ -10,6 +10,6 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
-
+.history
 # Dependency directories (remove the comment below to include it)
 # vendor/

README.md

@@ -1,62 +1,59 @@
-mkp-kibana-monitors
 
-# Overview
-This is a command line tool to sync and update the `Kibana object configs` - see below the help section. This tool is run as part of the scheduled pipeline to run every hour. It downloads the monitors config from kiban cluster
-then push the latest configuration into Gitlab - this project. The kibana objects supported are:
+![go](go.png)
+# odfe-kibana-sync 
+## Overview
+This is a command line tool to sync and cfeate the `Kibana object configs` - see below the help section. This tool assumes that Kibana cluster holds the source of the truth in relation to the configuration file. The sole purpose of this tool is to be run periodically to sync these configuration files with the a given Git repository.
+
+Idelly, this tool should be run in a CI pipeline for a given project that hosts Kibana configuration files - see example Github project.
+
+The following kibana configuration files can be synched and created in a given kiban instance:
 1. Monitors
 2. Dashboards
 3. Saved search
 4. Destinations(slack, email etc)
+5. Email accounts
+6. Group email
 
-```./kibana-sync -h
+```./odfe-kibana-sync -h
 This tool performs the followings:
-                1. Fetches configured monitors for the given kibana cluster and store them locally as json files.
+                1. Fetches configured monitors, dashboards, alert destinations for the given kibana cluster and store them locally as json files.
                 2. Pushes the changes done to the monitor's config to Kiban cluster
 
 Usage:
-  kibana-sync [flags]
-  kibana-sync [command]
+  odfe-kibana-sync [flags]
+  odfe-kibana-sync [command]
 
 Available Commands:
+  create      create all kiban objects (monitors, dashbaor, etc) present in the config folder
   help        Help about any command
-  sync        Fetch all configured monitors from Kibana cluster
-  push      pushes the monitor's config to Kibana cluster
+  sync        Fetches Kiban objects (monitor, dashbaord, etc) from Kibana cluster
 
 Flags:
-  -h, --help              help for kibana-sync
+  -h, --help              help for odfe-kibana-sync
       --password string   The kibana cluster password. This is a required argument to connect to the ELK cluster
       --url string        The kibana cluster url. This is required argument to connect to the ELk cluster
       --username string   The kibana cluster username. This is required argument to connect to the ELK cluster
+      --workdir string    The working directory where the kibana configuration files will be stored (default "config")
 
-Use "kibana-sync [command] --help" for more information about a command.
+Use "odfe-kibana-sync [command] --help" for more information about a command.
 ```
 
-## The pipeline
-
-The pipeline of this project is scheduled to run every hour, it's not triggered by push to master. So any changes to Kibana monitor config will get picked up every hour or so.
-
-
-## Limitations
 
-The limitations of this tool are highlighted below. Hopefully, subsequent releases of this tool will address some of them
-
-1. The push command push all the configuration files present in the config folder regardless if there is a change or no. This is not an issue at all, but we can improve.
 
 ## Invoking the sync command
 The sync command fetches all the monitors config defined in the given kibana cluster and store them locally in the `./config folder`
 
 ```
-./kibana-sync sync --username VF_Kibana_EMEA --password Vfkibana***** --url https://vpc-vf-sysint-emea-es-ir-jiqfwlmrnmrkm7ydovqpsvqueu.eu-west-1.es.amazonaws.com
+./odfe-kibana-sync sync --username admin --password admin --url https://localhost:9200
 ```
 
-## Invoking the push command
+## Invoking the create command
 The push command read all the monitor configs present in the local `./config folder` and push them into Kiban cluster
 
 ```
-./kibana-sync push --username VF_Kibana_EMEA --password Vfkibana****** --url https://vpc-vf-sysint-emea-es-ir-jiqfwlmrnmrkm7ydovqpsvqueu.eu-west-1.es.amazonaws.com
+./odfe-kibana-sync push --username admin --password admin --url https://localhost:9200
 ```
 
-
 ## Open distro
 
 You can run the following command to install opendistro
@@ -68,3 +65,20 @@ More details on getting started with ELK Open distro can be found [here](https:/
 
 
 
+```
+├── config
+│   ├── dashboard
+│   │   ├── dashboard:4b85e090-f4be-11ea-8342-bf90f7b9d26e.json
+│   │   ├── dashboard:722b74f0-b882-11e8-a6d9-e546fe2bba5f.json
+│   │   └── dashboard:edf84fe0-e1a0-11e7-b6d5-4dc382ef7f5b.json
+│   ├── destination
+│   │   └── XILEfXcBZWXOV7PGGr3r.json
+│   ├── email_account
+│   ├── email_group
+│   ├── monitor
+│   │   └── 13otoHcBbX-aeATowSlk.json
+│   └── search
+│       ├── search:3ba638e0-b894-11e8-a6d9-e546fe2bba5f.json
+│       └── search:571aaf70-4c88-11e8-b3d7-01146121b73d.json
+```
+

cmd/create.go

@@ -19,7 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	es "kib-sync/client"
+	es "odfe-kibana-sync/client"
 
 	"log"
 	"net/http"
@@ -31,14 +31,15 @@ import (
 // creates cobra command that represents the push command
 var createCmd = &cobra.Command{
 	Use:   "create",
-	Short: "create monitors in the given kiban cluster",
-	Long:  `create monitors in the given kiban cluster`,
-	Run:   createMonitorsConfig(newCreateHandler()),
+	Short: "create all kiban objects (monitors, dashbaor, etc) present in the config folder",
+	Long:  `create monitors, destinations, dashboards, email accoutns, group emails, search in the given kibana cluster`,
+	Run:   createKibanaConfig(newCreateHandler()),
 }
 
-type createHandler func(url string, body []byte) (*http.Response, error)
+// CreateHandler function implementation of the Create Kibana config
+type CreateHandler func(url string, body []byte) (*http.Response, error)
 
-func newCreateHandler() createHandler {
+func newCreateHandler() CreateHandler {
 	return func(path string, body []byte) (*http.Response, error) {
 		// create http client
 		client := es.NewClient(getValue(URL), getValue(UserName), getValue(Password))
@@ -48,17 +49,18 @@ func newCreateHandler() createHandler {
 	}
 }
 
-// PushMonitorsConfig pushes the monitor config to kibana cluster
-func createMonitorsConfig(handler createHandler) func(cmd *cobra.Command, args []string) {
+func createKibanaConfig(handler CreateHandler) func(cmd *cobra.Command, args []string) {
 	return func(cmd *cobra.Command, args []string) {
+
+		// crete parent config directory
 		log.Println("invoking create kibana config")
-		configs, err := ioutil.ReadDir("config")
+		configs, err := ioutil.ReadDir(getValue(WorkDir))
 		if err != nil {
 			log.Fatal(err)
 		}
 
 		for _, config := range configs {
-			fileInfos, err := ioutil.ReadDir("config/" + config.Name())
+			fileInfos, err := ioutil.ReadDir(fmt.Sprintf("%s/%s", getValue(WorkDir), config.Name()))
 			if err != nil {
 				ErrorLog.Printf("failed to read dir: %s", config.Name())
 				continue
@@ -68,7 +70,7 @@ func createMonitorsConfig(handler createHandler) func(cmd *cobra.Command, args [
 			for _, fileInfo := range fileInfos {
 
 				log.Printf("creating kibana %s: %s to kiban cluster", config.Name(), fileInfo.Name())
-				filename := fmt.Sprintf("config/%s/%s", config.Name(), fileInfo.Name())
+				filename := fmt.Sprintf("%s/%s/%s", getValue(WorkDir), config.Name(), fileInfo.Name())
 				bytes, err := ioutil.ReadFile(filename)
 				if err != nil {
 					ErrorLog.Printf("failed to read the content of the file %s", filename)
@@ -91,13 +93,13 @@ func createMonitorsConfig(handler createHandler) func(cmd *cobra.Command, args [
 					ErrorLog.Printf("failed to decode the request body for config: %s", filename)
 					continue
 				}
+				// create kibana object
 				res, err := handler(path(config.Name(), index, documentID), body)
 
 				if err != nil {
 					ErrorLog.Printf("Error getting response: %s", err.Error())
 					continue
 				}
-
 				defer res.Body.Close()
 
 				if res.StatusCode >= 300 {
@@ -109,7 +111,7 @@ func createMonitorsConfig(handler createHandler) func(cmd *cobra.Command, args [
 						ErrorLog.Printf("Error parsing the response body: %s", err)
 					} else {
 						// Print the response status and indexed document version.
-						InfoLog.Printf("[%d] %s; version=%d; monitor=%s", res.StatusCode, r["result"], int(r["_version"].(float64)), strings.SplitAfter(fileInfo.Name(), ".")[0])
+						InfoLog.Printf("[%d] %s; version=%d; config=%s", res.StatusCode, r["result"], int(r["_version"].(float64)), strings.SplitAfter(fileInfo.Name(), ".")[0])
 					}
 				}
 			}

cmd/create_test.go

@@ -0,0 +1,61 @@
+/*
+Copyright © 2020 NAME HERE <EMAIL ADDRESS>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/spf13/cobra"
+)
+
+func Test_createKibanaConfig(t *testing.T) {
+
+	// createTest dir
+	createDir("config/dashboard")
+
+	// 1. Create insntace of the crate command
+	cmd := &cobra.Command{
+		Use:   "create",
+		Short: "create all kiban objects (monitors, dashbaor, etc) present in the config folder",
+		Long:  `create monitors, destinations, dashboards, email accoutns, group emails, search in the given kibana cluster`,
+		Run:   createKibanaConfig(testCreateHandler()),
+	}
+	// 2. Execute comand
+	if err := cmd.Execute(); err != nil {
+		t.Errorf("failed to run create command: %v", err)
+	}
+
+	os.RemoveAll("config")
+
+}
+
+func testCreateHandler() CreateHandler {
+	return func(url string, body []byte) (*http.Response, error) {
+
+		bytes, err := ioutil.ReadFile("../data/dashboard_create_res.json")
+		if err != nil {
+			log.Fatal(err)
+		}
+		r := ioutil.NopCloser(strings.NewReader(string(bytes)))
+		res := http.Response{StatusCode: 201, Body: r}
+		return &res, nil
+	}
+}

cmd/root.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+
 	"github.com/spf13/cobra"
 )
 
@@ -30,6 +31,8 @@ const (
 	Password = "password"
 	// URL the elk url
 	URL = "url"
+	// WorkDir the working dir where the configuration will be stored
+	WorkDir = "workdir"
 )
 
 var (
@@ -42,10 +45,10 @@ var (
 )
 
 var rootCmd = &cobra.Command{
-	Use:   "kibana-sync",
-	Short: "Kibsync is a tool that fetches configured monitors",
+	Use:   "odfe-kibana-sync",
+	Short: "Kibsync is a tool that fetches configured objects in kibana cluster",
 	Long: `This tool performs the followings:
-		1. Fetches configured monitors for the given kibana cluster and store them locally as json files.
+		1. Fetches configured monitors, dashboards, alert destinations for the given kibana cluster and store them locally as json files.
 		2. Pushes the changes done to the monitor's config to Kiban cluster`,
 	Run: func(cmd *cobra.Command, args []string) {
 		log.Println("running kibana synchronizer")
@@ -71,6 +74,7 @@ func init() {
 	rootCmd.PersistentFlags().String("username", "", "The kibana cluster username. This is required argument to connect to the ELK cluster")
 	rootCmd.PersistentFlags().String("password", "", "The kibana cluster password. This is a required argument to connect to the ELK cluster")
 	rootCmd.PersistentFlags().String("url", "", "The kibana cluster url. This is required argument to connect to the ELk cluster")
+	rootCmd.PersistentFlags().String("workdir", "config", "The working directory where the kibana configuration files will be stored")
 
 	// add the command
 	rootCmd.AddCommand(syncCmd)
@@ -87,3 +91,14 @@ func getValue(flag string) string {
 	}
 	return value
 }
+
+func createDir(name string) {
+	_, err := os.Stat(name)
+	if os.IsNotExist(err) {
+		errDir := os.MkdirAll(name, 0755)
+		if errDir != nil {
+			log.Fatal(err)
+		}
+
+	}
+}