Merge pull request #567 from lpabon/v3.1

Update Release 3 to Release 3.1

Makefile

@@ -29,7 +29,7 @@ GO=go
 # Sources and Targets
 EXECUTABLES :=$(APP_NAME)
 # Build Binaries setting main.version and main.build vars
-LDFLAGS :=-ldflags "-X main.HEKETI_VERSION=$(VERSION)"
+LDFLAGS :=-ldflags "-X main.HEKETI_VERSION=$(VERSION) -extldflags '-z relro -z now'"
 # Package target
 PACKAGE :=$(DIR)/dist/$(APP_NAME)-$(VERSION).$(GOOS).$(ARCH).tar.gz
 CLIENT_PACKAGE :=$(DIR)/dist/$(APP_NAME)-client-$(VERSION).$(GOOS).$(ARCH).tar.gz
@@ -84,9 +84,12 @@ $(PACKAGE): all
 $(CLIENT_PACKAGE): all
 	@echo Packaging client Binaries...
 	@mkdir -p tmp/$(CLIENT_PKG_NAME)/bin
-	@mkdir -p tmp/$(CLIENT_PKG_NAME)/share/heketi/templates
+	@mkdir -p tmp/$(CLIENT_PKG_NAME)/share/heketi/openshift/templates
+	@mkdir -p tmp/$(CLIENT_PKG_NAME)/share/heketi/kubernetes
+	@cp client/cli/go/topology-sample.json tmp/$(CLIENT_PKG_NAME)/share/heketi
 	@cp client/cli/go/heketi-cli tmp/$(CLIENT_PKG_NAME)/bin
-	@cp extras/openshift/templates/* tmp/$(CLIENT_PKG_NAME)/share/heketi/templates
+	@cp extras/openshift/templates/* tmp/$(CLIENT_PKG_NAME)/share/heketi/openshift/templates
+	@cp extras/kubernetes/* tmp/$(CLIENT_PKG_NAME)/share/heketi/kubernetes
 	@mkdir -p $(DIR)/dist/
 	tar -czf $@ -C tmp $(CLIENT_PKG_NAME);
 	@rm -rf tmp

apps/glusterfs/app.go

@@ -346,8 +346,10 @@ func (a *App) SetRoutes(router *mux.Router) error {
 
 	}
 
-	return nil
+	// Set default error handler
+	router.NotFoundHandler = http.HandlerFunc(a.NotFoundHandler)
 
+	return nil
 }
 
 func (a *App) Close() {
@@ -389,3 +391,8 @@ func (a *App) Backup(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}
 }
+
+func (a *App) NotFoundHandler(w http.ResponseWriter, r *http.Request) {
+	logger.Warning("Invalid path or request %v", r.URL.Path)
+	http.Error(w, "Invalid path or request", http.StatusNotFound)
+}

apps/glusterfs/app_test.go

@@ -18,10 +18,14 @@ package glusterfs
 
 import (
 	"bytes"
+	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/boltdb/bolt"
+	"github.com/gorilla/mux"
+	client "github.com/heketi/heketi/client/api/go-client"
 	"github.com/heketi/heketi/pkg/utils"
 	"github.com/heketi/tests"
 )
@@ -193,3 +197,31 @@ func TestAppReadOnlyDb(t *testing.T) {
 	tests.Assert(t, app != nil)
 	tests.Assert(t, app.dbReadOnly == true)
 }
+
+func TestAppPathNotFound(t *testing.T) {
+	dbfile := tests.Tempfile()
+	defer os.Remove(dbfile)
+
+	app := NewTestApp(dbfile)
+	tests.Assert(t, app != nil)
+	defer app.Close()
+	router := mux.NewRouter()
+	app.SetRoutes(router)
+
+	// Setup the server
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	// Setup a new client
+	c := client.NewClientNoAuth(ts.URL)
+
+	// Test paths which do not match the hexadecimal id
+	_, err := c.ClusterInfo("xxx")
+	tests.Assert(t, strings.Contains(err.Error(), "Invalid path or request"))
+
+	_, err = c.NodeInfo("xxx")
+	tests.Assert(t, strings.Contains(err.Error(), "Invalid path or request"))
+
+	_, err = c.VolumeInfo("xxx")
+	tests.Assert(t, strings.Contains(err.Error(), "Invalid path or request"))
+}

client/api/python/setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1

client/api/python/setup.py

@@ -27,12 +27,14 @@
     test_suite='nose.collector',
     install_requires=['pyjwt', 'requests'],
     classifiers=[
-        'Development Status :: 5 - Production/Stable'
-        'Intended Audience :: Information Technology'
-        'Intended Audience :: System Administrators'
-        'License :: OSI Approved :: Apache Software License'
-        'Operating System :: POSIX :: Linux'
-        'Programming Language :: Python'
-        'Programming Language :: Python :: 2.7'
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2.7',
+        'Topic :: System :: Filesystems',
+        'Topic :: System :: Distributed Computing',
     ],
 )

client/cli/go/Makefile

@@ -29,7 +29,7 @@ TEST="go test"
 # Sources and Targets
 EXECUTABLES :=$(APP_NAME)
 # Build Binaries setting main.version and main.build vars
-LDFLAGS :=-ldflags "-X main.HEKETI_CLI_VERSION=$(VERSION)"
+LDFLAGS :=-ldflags "-X main.HEKETI_CLI_VERSION=$(VERSION) -extldflags '-z relro -z now'"
 # Package target
 PACKAGE :=$(DIR)/dist/$(APP_NAME)-$(VERSION).$(GOOS).$(ARCH).tar.gz
 

executors/kubeexec/config.go

@@ -27,10 +27,13 @@ type KubeConfig struct {
 	Insecure  bool   `json:"insecure"`
 	User      string `json:"user"`
 	Password  string `json:"password"`
+	Token     string `json:"token"`
 	Namespace string `json:"namespace"`
 	// Use Secrets to get the Bearerkey
-	UseSecrets bool   `json:"use_secrets"`
-	TokenFile  string `json:"token"`
+	UseSecrets bool `json:"use_secrets"`
+
+	TokenFile     string `json:"token_file"`
+	NamespaceFile string `json:"namespace_file"`
 
 	// Use POD name instead of using label
 	// to access POD

executors/kubeexec/kubeexec.go

@@ -39,15 +39,6 @@ import (
 	"github.com/heketi/heketi/pkg/utils"
 )
 
-type KubernetesClient interface {
-}
-
-type KubernetesRemoteCommand interface {
-}
-
-type KubernetesRemoteCommandStream interface {
-}
-
 const (
 	KubeGlusterFSPodLabelKey = "glusterfs-node"
 )
@@ -139,6 +130,11 @@ func setWithEnvVariables(config *KubeConfig) {
 		config.TokenFile = env
 	}
 
+	env = os.Getenv("HEKETI_KUBE_NAMESPACEFILE")
+	if "" != env {
+		config.NamespaceFile = env
+	}
+
 	// Use POD names
 	env = os.Getenv("HEKETI_KUBE_USE_POD_NAMES")
 	if "" != env {
@@ -168,6 +164,13 @@ func NewKubeExecutor(config *KubeConfig) (*KubeExecutor, error) {
 	}
 
 	// Check required values
+	if k.config.NamespaceFile != "" {
+		var err error
+		k.config.Namespace, err = k.readAllLinesFromFile(k.config.NamespaceFile)
+		if err != nil {
+			return nil, err
+		}
+	}
 	if k.config.Namespace == "" {
 		return nil, fmt.Errorf("Namespace must be provided in configuration")
 	}
@@ -193,13 +196,12 @@ func (k *KubeExecutor) RemoteCommandExecute(host string,
 
 	// Execute
 	return k.ConnectAndExec(host,
-		k.config.Namespace,
 		"pods",
 		commands,
 		timeoutMinutes)
 }
 
-func (k *KubeExecutor) ConnectAndExec(host, namespace, resource string,
+func (k *KubeExecutor) ConnectAndExec(host, resource string,
 	commands []string,
 	timeoutMinutes int) ([]string, error) {
 
@@ -213,7 +215,10 @@ func (k *KubeExecutor) ConnectAndExec(host, namespace, resource string,
 	clientConfig.Insecure = k.config.Insecure
 
 	// Login
-	if k.config.User != "" && k.config.Password != "" {
+	if k.config.UseSecrets == false &&
+		k.config.User != "" &&
+		k.config.Password != "" {
+
 		token, err := tokenCreator(clientConfig,
 			nil,
 			k.config.User,
@@ -224,12 +229,11 @@ func (k *KubeExecutor) ConnectAndExec(host, namespace, resource string,
 		}
 		clientConfig.BearerToken = token
 	} else if k.config.UseSecrets {
-		tokenBytes, err := ioutil.ReadFile(k.config.TokenFile)
+		var err error
+		clientConfig.BearerToken, err = k.readAllLinesFromFile(k.config.TokenFile)
 		if err != nil {
-			logger.Err(err)
-			return nil, logger.LogError("Secret token not found in %v", k.config.TokenFile)
+			return nil, err
 		}
-		clientConfig.BearerToken = string(tokenBytes)
 	}
 
 	// Get a client
@@ -254,7 +258,7 @@ func (k *KubeExecutor) ConnectAndExec(host, namespace, resource string,
 		}
 
 		// Get a list of pods
-		pods, err := conn.Pods(namespace).List(api.ListOptions{
+		pods, err := conn.Pods(k.config.Namespace).List(api.ListOptions{
 			LabelSelector: selector,
 			FieldSelector: fields.Everything(),
 		})
@@ -294,7 +298,7 @@ func (k *KubeExecutor) ConnectAndExec(host, namespace, resource string,
 		req := conn.RESTClient.Post().
 			Resource(resource).
 			Name(podName).
-			Namespace(namespace).
+			Namespace(k.config.Namespace).
 			SubResource("exec")
 		req.VersionedParams(&api.PodExecOptions{
 			Command: []string{"/bin/bash", "-c", command},
@@ -339,3 +343,11 @@ func (k *KubeExecutor) RebalanceOnExpansion() bool {
 func (k *KubeExecutor) SnapShotLimit() int {
 	return k.config.SnapShotLimit
 }
+
+func (k *KubeExecutor) readAllLinesFromFile(filename string) (string, error) {
+	fileBytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return "", logger.LogError("Error reading %v file: %v", filename, err.Error())
+	}
+	return string(fileBytes), nil
+}

extras/docker/unstable/README.md

@@ -9,7 +9,7 @@ First you will need to download the latest development container:
 
     # docker pull heketi/heketi:dev
 
-> NOTE: Must likely you will always need to do a new pull before staring your tests since the container changes so often.
+> NOTE: Most likely you will always need to do a new pull before staring your tests since the container changes so often.
 
 ## Server Setup
 You will need to create a directory which has a directory containing configuraiton and any private key if necessary, and an empty directory used for storing the database.  Directory and files must be read/write by user with id 1000 and if an ssh private key is used, it must also have a mod of 0600.

extras/kubernetes/README.md

@@ -0,0 +1,50 @@
+# Overview
+Kubernetes templates for Heketi and Gluster. The following documentation is setup
+to deploy the containers in Kubernetes.  It is not a full setup.  For full
+documentation, please visit the Heketi wiki page.
+
+# Usage
+
+## Deploy Gluster
+
+* Get node name by running:
+
+```
+$ kubectl get nodes
+```
+
+* Deploy gluster container onto specified node:
+
+```
+$ sed -e \
+   's#<GLUSTERFS_NODE>#..type your node name here..#' \
+   glusterfs-deployment.json | kubectl create -f -
+```
+
+Repeat as needed.
+
+## Deploy Heketi
+
+* Create a service account for Heketi
+
+```
+$ kubectl create -f heketi-service-account.json
+```
+
+* Note the secret for the service account 
+
+```
+$ heketi_secret=$(kubectl get sa heketi-service-account -o="go-template" --template="{{(index .secrets 0).name}}")
+```
+
+* Deploy deploy-heketi.  Before deploying you will need to determine the Kubernetes API endpoint and namespace.
+
+In this example, we will use `https://1.1.1.1:443` as our Kubernetes API endpoint
+
+```
+$ sed -e "s#<HEKETI_KUBE_SECRETNAME>#\"$heketi_secret\"#" \
+      -e "s#<HEKETI_KUBE_APIHOST>#\"http://1.1.1.1:443\"#" deploy-heketi-deployment.json | kubectl create -f -
+```
+
+Please refer to the wiki Kubernetes Deployment page for more information
+