Include remote configuration retrival

Improve authorization error handling

Signed-off-by: Raul Sevilla <rsevilla@redhat.com>

Makefile

@@ -12,7 +12,7 @@ SOURCES := $(shell find . -type f -name "*.go")
 BUILD_DATE = $(shell date '+%Y-%m-%d-%H:%M:%S')
 KUBE_BURNER_PACKAGE = github.com/cloud-bulldozer/kube-burner
 
-ifeq (, $(shell which docker))
+ifeq (, $(shell command -v docker))
   ENGINE := podman
 else
   ENGINE := docker

README.md

@@ -82,7 +82,7 @@ Use "kube-burner [command] --help" for more information about a command.
 ```
 
 - The **init** option supports the following flags:
-  - config: Path to a valid configuration file.
+  - config: Path or URL to a valid configuration file.
   - log-level: Logging level. Default `info`
   - prometheus-url: Prometheus full URL. i.e. `https://prometheus-k8s-openshift-monitoring.apps.rsevilla.stress.mycluster.example.com`
   - metrics-profile: Path to a valid metrics profile file. Default `metrics.yaml`
@@ -101,6 +101,12 @@ With the above, triggering kube-burner would be as simple as:
 $ kube-burner init -c cfg.yml -u https://prometheus-k8s-openshift-monitoring.apps.rsevilla.stress.mycluster.example.com -t ${token} --uuid 67f9ec6d-6a9e-46b6-a3bb-065cde988790`
 ```
 
+or
+```console
+$ kube-burner init -c http://web.domain.com:8080/cfg.yml -u https://prometheus-k8s-openshift-monitoring.apps.rsevilla.stress.mycluster.example.com -t ${token} --uuid 67f9ec6d-6a9e-46b6-a3bb-065cde988790`
+```
+
+
 If you have no interest in collecting prometheus metrics, kube-burner can also be launched w/o any prometheus endpoint.
 
 ```console
@@ -161,11 +167,11 @@ A valid example of a configuration file can be found at [./examples/cfg.yml](./e
 The objects created by `kube-burner` are rendered using the default golang's [template library](https://golang.org/pkg/text/template/).
 Each object element supports the following parameters:
 
-| Option               | Description                                                       | Type    | Example        | Default |
-|----------------------|-------------------------------------------------------------------|---------|----------------|---------|
-| objectTemplate       | Object template file                                              | String  | deployment.yml | ""      |
-| replicas             | How replicas of this object to create per job iteration           | Integer | 10             | -       |
-| inputVars            | Map of arbitrary input variables to inject to the object template | Object  | -              | -       |
+| Option               | Description                                                       | Type    | Example                                             | Default |
+|----------------------|-------------------------------------------------------------------|---------|-----------------------------------------------------|---------|
+| objectTemplate       | Object template file or URL                                       | String  | deployment.yml or https://domain.com/deployment.yml | ""      |
+| replicas             | How replicas of this object to create per job iteration           | Integer | 10                                                  | -       |
+| inputVars            | Map of arbitrary input variables to inject to the object template | Object  | -                                                   | -       |
 
 It's important to note that all objects created by kube-burner are labeled with. `kube-burner-uuid=<UUID>,kube-burner-job=<jobName>,kube-burner-index=<objectIndex>`
 
@@ -231,7 +237,7 @@ All object templates are injected a series of variables by default:
 - JobName: Job name.
 - UUID: Benchmark UUID.
 
-In addition, you can also inject arbitrary variables with the option inputVars from the objectTemplate object:
+In addition, you can also inject arbitrary variables with the option **inputVars** from the objectTemplate object:
 
 ```yaml
     - objectTemplate: service.yml
@@ -277,7 +283,7 @@ spec:
 
 ### Metrics profile
 
-The metrics-profile flag points to a YAML file containing a list of the prometheus queries kube-burner will collect for each job.
+The metrics-profile flag points to a YAML or URL of a file containing a list of the prometheus queries kube-burner will collect for each job.
 As soon one of job finishes, `kube-burner` makes a range query for each query described in this file, and indexes it in the index configured by the parameter `defaultIndex`.
 We can use the parameter `indexName` in a metrics-profile file to make `kube-burner` to index the resulting metrics to a different index.
 An example of a valid metrics profile file is shown below:

cmd/kube-burner.go

@@ -64,13 +64,13 @@ func initCmd() *cobra.Command {
 			log.Infof("🔥 Starting kube-burner with UUID %s", uuid)
 			err := config.Parse(configFile, true)
 			if err != nil {
-				log.Fatalf("Error parsing configuration: %s", err)
+				log.Fatal(err)
 			}
 			var p *prometheus.Prometheus
 			if url != "" {
 				p, err = prometheus.NewPrometheusClient(url, token, username, password, metricsProfile, uuid, skipTLSVerify, prometheusStep)
 				if err != nil {
-					log.Fatalf("Error setting up Prometheus client: %s", err)
+					log.Fatal(err)
 				}
 			}
 			steps(uuid, p, prometheusStep)
@@ -85,9 +85,7 @@ func initCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&skipTLSVerify, "skip-tls-verify", true, "Verify prometheus TLS certificate")
 	cmd.Flags().DurationVarP(&prometheusStep, "step", "s", 30*time.Second, "Prometheus step size")
 	cmd.Flags().StringVarP(&configFile, "config", "c", "", "Config file path")
-	cmd.MarkFlagFilename("config")
 	cmd.MarkFlagRequired("config")
-	cmd.MarkFlagFilename("metrics-profile")
 	return cmd
 }
 
@@ -101,7 +99,7 @@ func destroyCmd() *cobra.Command {
 			if configFile != "" {
 				err := config.Parse(configFile, false)
 				if err != nil {
-					log.Fatalf("Error parsing configuration: %s", err)
+					log.Fatal(err)
 				}
 			}
 			selector := util.NewSelector()
@@ -131,7 +129,7 @@ func indexCmd() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			err := config.Parse(configFile, false)
 			if err != nil {
-				log.Fatalf("Error parsing configuration: %s", err)
+				log.Fatal(err)
 			}
 			var indexer *indexers.Indexer
 			if config.ConfigSpec.GlobalConfig.IndexerConfig.Enabled {
@@ -141,7 +139,7 @@ func indexCmd() *cobra.Command {
 			}
 			p, err := prometheus.NewPrometheusClient(url, token, username, password, metricsProfile, uuid, skipTLSVerify, prometheusStep)
 			if err != nil {
-				log.Fatalf("Error setting up Prometheus client: %s", err)
+				log.Fatal(err)
 			}
 			startTime := time.Unix(start, 0)
 			endTime := time.Unix(end, 0)
@@ -162,9 +160,7 @@ func indexCmd() *cobra.Command {
 	cmd.Flags().Int64VarP(&start, "start", "", time.Now().Unix()-3600, "Epoch start time")
 	cmd.Flags().Int64VarP(&end, "end", "", time.Now().Unix(), "Epoch end time")
 	cmd.Flags().StringVarP(&configFile, "config", "c", "", "Config file path")
-	cmd.MarkFlagFilename("metrics-profile")
 	cmd.MarkFlagRequired("prometheus-url")
-	cmd.MarkFlagFilename("config")
 	cmd.MarkFlagRequired("config")
 	return cmd
 }

pkg/burner/create.go

@@ -17,6 +17,7 @@ package burner
 import (
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"strconv"
@@ -42,6 +43,8 @@ const (
 )
 
 func setupCreateJob(jobConfig config.Job) Executor {
+	var f io.Reader
+	var err error
 	log.Infof("Preparing create job: %s", jobConfig.Name)
 	var empty interface{}
 	selector := util.NewSelector()
@@ -55,9 +58,13 @@ func setupCreateJob(jobConfig config.Job) Executor {
 			continue
 		}
 		log.Debugf("Processing template: %s", o.ObjectTemplate)
-		f, err := os.Open(o.ObjectTemplate)
+		f, err = os.Open(o.ObjectTemplate)
+		// If the template file does not exist we try to read it from an URL
+		if os.IsNotExist(err) {
+			f, err = util.ReadURL(o.ObjectTemplate)
+		}
 		if err != nil {
-			log.Fatalf("Error getting gvr: %s", err)
+			log.Fatalf("Error reading template %s: %s", o.ObjectTemplate, err)
 		}
 		t, err := ioutil.ReadAll(f)
 		if err != nil {
@@ -169,6 +176,9 @@ func (ex *Executor) replicaHandler(objectIndex int, obj object, ns string, itera
 			defer wg.Done()
 			ex.limiter.Wait(context.TODO())
 			_, err := dynamicClient.Resource(obj.gvr).Namespace(ns).Create(context.TODO(), newObject, metav1.CreateOptions{})
+			if errors.IsForbidden(err) {
+				log.Fatalf("Authorization error creating object %s: %s", newObject.GetName(), err)
+			}
 			if errors.IsAlreadyExists(err) {
 				log.Errorf("Object %s in namespace %s already exists", newObject.GetName(), ns)
 			} else if err != nil {

pkg/burner/namespaces.go

@@ -33,6 +33,9 @@ func createNamespace(clientset *kubernetes.Clientset, namespaceName string, nsLa
 	}
 	log.Infof("Creating namespace %s", ns.Name)
 	_, err := clientset.CoreV1().Namespaces().Create(context.TODO(), &ns, metav1.CreateOptions{})
+	if errors.IsForbidden(err) {
+		log.Fatalf("Authorization error creating namespace %s: %s", ns.Name, err)
+	}
 	if errors.IsAlreadyExists(err) {
 		log.Warnf("Namespace %s already exists", ns.Name)
 	} else if err != nil {

pkg/config/config.go

@@ -16,11 +16,13 @@ package config
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/cloud-bulldozer/kube-burner/log"
+	"github.com/cloud-bulldozer/kube-burner/pkg/util"
 	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/rest"
@@ -64,9 +66,14 @@ func (j *Job) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 // Parse parses configuration file
 func Parse(c string, jobsRequired bool) error {
+	var f io.Reader
 	f, err := os.Open(c)
+	// If the config file does not exist we try to read it from an URL
+	if os.IsNotExist(err) {
+		f, err = util.ReadURL(c)
+	}
 	if err != nil {
-		return err
+		return fmt.Errorf("Error reading configuration file %s: %s", c, err)
 	}
 	yamlDec := yaml.NewDecoder(f)
 	yamlDec.KnownFields(true)

pkg/prometheus/prometheus.go

@@ -19,6 +19,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"io"
 	"math"
 	"net/http"
 	"os"
@@ -29,6 +30,7 @@ import (
 	"github.com/cloud-bulldozer/kube-burner/log"
 	"github.com/cloud-bulldozer/kube-burner/pkg/config"
 	"github.com/cloud-bulldozer/kube-burner/pkg/indexers"
+	"github.com/cloud-bulldozer/kube-burner/pkg/util"
 	api "github.com/prometheus/client_golang/api"
 	apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
@@ -125,15 +127,20 @@ func (p *Prometheus) verifyConnection() error {
 	return nil
 }
 
-func (p *Prometheus) readProfile(metricsFile string) error {
-	f, err := os.Open(metricsFile)
+func (p *Prometheus) readProfile(metricsProfile string) error {
+	var f io.Reader
+	f, err := os.Open(metricsProfile)
+	// If the metricsProfile file does not exist we try to read it from an URL
+	if os.IsNotExist(err) {
+		f, err = util.ReadURL(metricsProfile)
+	}
 	if err != nil {
-		log.Fatalf("Error reading metrics profile %s: %s", metricsFile, err)
+		log.Fatalf("Error reading metrics profile %s: %s", metricsProfile, err)
 	}
 	yamlDec := yaml.NewDecoder(f)
 	yamlDec.KnownFields(true)
 	if err = yamlDec.Decode(&p.metricsProfile); err != nil {
-		return fmt.Errorf("Error decoding metrics profile %s: %s", metricsFile, err)
+		return fmt.Errorf("Error decoding metrics profile %s: %s", metricsProfile, err)
 	}
 	return nil
 }

pkg/util/url_reader.go

@@ -0,0 +1,43 @@
+// Copyright 2020 The Kube-burner Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+// ReadURL reads an URL and returns a reader
+func ReadURL(stringURL string) (io.Reader, error) {
+	var body io.Reader
+	u, err := url.Parse(stringURL)
+	if err != nil {
+		return body, err
+	}
+	if !u.IsAbs() {
+		return body, fmt.Errorf("%s is not a valid URL", u)
+	}
+	r, err := http.Get(stringURL)
+	if err != nil {
+		return body, err
+	}
+	if r.StatusCode != http.StatusOK {
+		return body, fmt.Errorf("Error requesting %s: %d", u, r.StatusCode)
+	}
+	body = r.Body
+	return body, nil
+}