Add flag for go template to format host names (#130)

When using an alternate DNS like external-dns, your host names will be
different than the internal core-dns format.  This adds a flag so that
the user can specify the name format using a go template.

README.md

@@ -9,6 +9,7 @@ MinIO-Operator brings native MinIO, [MCS](https://github.com/minio/mcs), and [KE
 | Create and delete highly available distributed MinIO clusters  | [Create a MinIO Instance](https://github.com/minio/minio-operator#create-a-minio-instance). |
 | Automatic TLS for MinIO                                        | [Automatic TLS for MinIO Instance](https://github.com/minio/minio-operator/blob/master/docs/tls.md#automatic-csr-generation). |
 | Expand an existing MinIO cluster                               | [Expand a MinIO Cluster](https://github.com/minio/minio-operator/blob/master/docs/adding-zones.md). |
+| Use a custom template for hostname discovery                   | [Custom Hostname Discovery](https://github.com/minio/minio-operator/blob/master/docs/custom-name-templates.md). |
 | Deploy MCS with MinIO cluster  | [Deploy MinIO Instance with MCS](https://github.com/minio/minio-operator/blob/master/docs/mcs.md). |
 | Deploy KES with MinIO cluster  | [Deploy MinIO Instance with KES](https://github.com/minio/minio-operator/blob/master/docs/kes.md). |
 

docs/custom-name-templates.md

@@ -0,0 +1,29 @@
+# Custom Hostname Discovery
+
+[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
+[![Docker Pulls](https://img.shields.io/docker/pulls/minio/k8s-operator.svg?maxAge=604800)](https://hub.docker.com/r/minio/k8s-operator)
+
+This document explains how to control the names used for host discovery.  This allows us to discover hosts using external name services, which is useful for serving with trusted certificates.
+
+## Getting Started
+
+Assuming you have a MinIO cluster with single zone, `zone-0` with 4 drives (as shown in [examples](https://github.com/minio/minio-operator/tree/master/examples)). You can dd a new zone `zone-1` with 4 drives using `kubectl patch` command.
+
+The example cluster is named minio, so the four servers will be called `minio-0`, `minio-1`, `minio-2`, and `minio-3`.  If all of your hosts are available at the domain `example.com` then you can use the `--hosts-template` flag to update discovery:
+
+```
+  containers:
+  - command:
+    - /minio-operator
+    - --hosts-template
+    - '{{.StatefulSet}}-{{.Ellipsis}}.example.com'
+```
+
+This will generate the discovery string `minio-{0...3}.example.com`.  The following fields are available
+| Field                 | Description |
+|-----------------------|-------------|
+| StatefulSet | The name of the instance StatefulSet (e.g. `minio`). |
+| CIService | The name of the service provided in `spec.serviceName`. |
+| HLService | The name of the headless service that is generated (e.g. `minio-hl-service`) |
+| Ellipsis | `{0...N-1}` the per-zone host numbers. |
+| Domain | The cluster domain, either `cluster.local` or the contents of the `CLUSTER_DOMAIN` environment variable. |

main.go

@@ -44,9 +44,10 @@ import (
 var Version = "DEVELOPMENT.GOGET"
 
 var (
-	masterURL    string
-	kubeconfig   string
-	checkVersion bool
+	masterURL     string
+	kubeconfig    string
+	hostsTemplate string
+	checkVersion  bool
 
 	onlyOneSignalHandler = make(chan struct{})
 	shutdownSignals      = []os.Signal{os.Interrupt, syscall.SIGTERM}
@@ -55,6 +56,7 @@ var (
 func init() {
 	flag.StringVar(&kubeconfig, "kubeconfig", "", "path to a kubeconfig. Only required if out-of-cluster")
 	flag.StringVar(&masterURL, "master", "", "the address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster")
+	flag.StringVar(&hostsTemplate, "hosts-template", "", "the go template to use for hostname formatting of name fields (StatefulSet, CIService, HLService, Ellipsis, Domain)")
 	flag.BoolVar(&checkVersion, "version", false, "print version")
 }
 
@@ -113,7 +115,8 @@ func main() {
 		kubeInformerFactory.Apps().V1().Deployments(),
 		kubeInformerFactory.Batch().V1().Jobs(),
 		minioInformerFactory.Operator().V1().MinIOInstances(),
-		kubeInformerFactory.Core().V1().Services())
+		kubeInformerFactory.Core().V1().Services(),
+		hostsTemplate)
 
 	go kubeInformerFactory.Start(stopCh)
 	go minioInformerFactory.Start(stopCh)

pkg/apis/operator.min.io/v1/helper.go

@@ -18,6 +18,7 @@
 package v1
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"errors"
@@ -26,6 +27,7 @@ import (
 	"net/http"
 	"path"
 	"strconv"
+	"text/template"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -39,6 +41,19 @@ import (
 	"github.com/minio/minio/pkg/madmin"
 )
 
+type hostsTemplateValues struct {
+	StatefulSet string
+	CIService   string
+	HLService   string
+	Ellipsis    string
+	Domain      string
+}
+
+// ellipsis returns the host range string
+func ellipsis(start, end int) string {
+	return "{" + strconv.Itoa(start) + "..." + strconv.Itoa(end) + "}"
+}
+
 // HasCredsSecret returns true if the user has provided a secret
 // for a MinIOInstance else false
 func (mi *MinIOInstance) HasCredsSecret() bool {
@@ -77,7 +92,7 @@ func (mi *MinIOInstance) VolumePath() string {
 	if mi.Spec.VolumesPerServer == 1 {
 		return path.Join(mi.Spec.Mountpath, mi.Spec.Subpath)
 	}
-	return path.Join(mi.Spec.Mountpath+"{0..."+strconv.Itoa((mi.Spec.VolumesPerServer)-1)+"}", mi.Spec.Subpath)
+	return path.Join(mi.Spec.Mountpath+ellipsis(0, mi.Spec.VolumesPerServer-1), mi.Spec.Subpath)
 }
 
 // MinIOReplicas returns the number of total replicas
@@ -203,7 +218,37 @@ func (mi *MinIOInstance) MinIOHosts() []string {
 	// Create the ellipses style URL
 	for _, z := range mi.Spec.Zones {
 		max = max + z.Servers
-		hosts = append(hosts, fmt.Sprintf("%s-{"+strconv.Itoa(int(index))+"..."+strconv.Itoa(int(max)-1)+"}.%s.%s.svc.%s", mi.MinIOStatefulSetName(), mi.MinIOHLServiceName(), mi.Namespace, ClusterDomain))
+		hosts = append(hosts, fmt.Sprintf("%s-%s.%s.%s.svc.%s", ellipsis(int(index), int(max)-1), mi.MinIOStatefulSetName(), mi.MinIOHLServiceName(), mi.Namespace, ClusterDomain))
+		index = max
+	}
+	return hosts
+}
+
+// TemplatedMinIOHosts returns the domain names in ellipses format created for current MinIOInstance without the service part
+func (mi *MinIOInstance) TemplatedMinIOHosts(hostsTemplate string) []string {
+	hosts := make([]string, 0)
+	tmpl, err := template.New("hosts").Parse(hostsTemplate)
+	if err != nil {
+		msg := "Invalid go template for hosts"
+		klog.V(2).Infof(msg)
+		return hosts
+	}
+	var max, index int32
+	// Create the ellipses style URL
+	for _, z := range mi.Spec.Zones {
+		max = max + z.Servers
+		data := hostsTemplateValues{
+			StatefulSet: mi.MinIOStatefulSetName(),
+			CIService:   mi.MinIOCIServiceName(),
+			HLService:   mi.MinIOHLServiceName(),
+			Ellipsis:    ellipsis(int(index), int(max)-1),
+			Domain:      ClusterDomain,
+		}
+		output := new(bytes.Buffer)
+		if err = tmpl.Execute(output, data); err != nil {
+			continue
+		}
+		hosts = append(hosts, output.String())
 		index = max
 	}
 	return hosts

pkg/apis/operator.min.io/v1/helper_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestEnsureDefaults(t *testing.T) {
@@ -42,3 +43,39 @@ func TestEnsureDefaults(t *testing.T) {
 		assert.Equal(t, newImage, mi.Spec.Image)
 	})
 }
+
+func TestTemplateVariables(t *testing.T) {
+	servers := 2
+	mi := MinIOInstance{
+		ObjectMeta: metav1.ObjectMeta{Name: "test"},
+		Spec: MinIOInstanceSpec{
+			Zones: []Zone{{"single", int32(servers)}},
+		},
+	}
+	mi.EnsureDefaults()
+
+	t.Run("StatefulSet", func(t *testing.T) {
+		hosts := mi.TemplatedMinIOHosts("{{.StatefulSet}}")
+		assert.Contains(t, hosts, mi.MinIOStatefulSetName())
+	})
+
+	t.Run("CIService", func(t *testing.T) {
+		hosts := mi.TemplatedMinIOHosts("{{.CIService}}")
+		assert.Contains(t, hosts, mi.MinIOCIServiceName())
+	})
+
+	t.Run("HLService", func(t *testing.T) {
+		hosts := mi.TemplatedMinIOHosts("{{.HLService}}")
+		assert.Contains(t, hosts, mi.MinIOHLServiceName())
+	})
+
+	t.Run("Ellipsis", func(t *testing.T) {
+		hosts := mi.TemplatedMinIOHosts("{{.Ellipsis}}")
+		assert.Contains(t, hosts, ellipsis(0, servers-1))
+	})
+
+	t.Run("Domain", func(t *testing.T) {
+		hosts := mi.TemplatedMinIOHosts("{{.Domain}}")
+		assert.Contains(t, hosts, ClusterDomain)
+	})
+}

pkg/controller/cluster/csr.go

@@ -67,7 +67,7 @@ func isEqual(a, b []string) bool {
 	return true
 }
 
-func generateCryptoData(mi *miniov1.MinIOInstance) ([]byte, []byte, error) {
+func generateCryptoData(mi *miniov1.MinIOInstance, hostsTemplate string) ([]byte, []byte, error) {
 	var dnsNames []string
 	klog.V(0).Infof("Generating private key")
 	privateKey, err := newPrivateKey(miniov1.DefaultEllipticCurve)
@@ -85,6 +85,9 @@ func generateCryptoData(mi *miniov1.MinIOInstance) ([]byte, []byte, error) {
 	klog.V(0).Infof("Generating CSR with CN=%s", mi.Spec.CertConfig.CommonName)
 
 	hosts := mi.AllMinIOHosts()
+	if hostsTemplate != "" {
+		hosts = mi.TemplatedMinIOHosts(hostsTemplate)
+	}
 
 	if isEqual(mi.Spec.CertConfig.DNSNames, hosts) {
 		dnsNames = mi.Spec.CertConfig.DNSNames
@@ -113,7 +116,7 @@ func generateCryptoData(mi *miniov1.MinIOInstance) ([]byte, []byte, error) {
 // finally creating a secret that MinIO statefulset will use to mount private key and certificate for TLS
 // This Method Blocks till the CSR Request is approved via kubectl approve
 func (c *Controller) createCSR(ctx context.Context, mi *miniov1.MinIOInstance) error {
-	privKeysBytes, csrBytes, err := generateCryptoData(mi)
+	privKeysBytes, csrBytes, err := generateCryptoData(mi, c.hostsTemplate)
 	if err != nil {
 		klog.Errorf("Private Key and CSR generation failed with error: %v", err)
 		return err

pkg/controller/cluster/kes-csr.go

@@ -101,7 +101,7 @@ func (c *Controller) createKESTLSCSR(ctx context.Context, mi *miniov1.MinIOInsta
 // createMinIOClientTLSCSR handles all the steps required to create the CSR: from creation of keys, submitting CSR and
 // finally creating a secret that KES Statefulset will use for MinIO Client Auth
 func (c *Controller) createMinIOClientTLSCSR(ctx context.Context, mi *miniov1.MinIOInstance) error {
-	privKeysBytes, csrBytes, err := generateCryptoData(mi)
+	privKeysBytes, csrBytes, err := generateCryptoData(mi, c.hostsTemplate)
 	if err != nil {
 		klog.Errorf("Private Key and CSR generation failed with error: %v", err)
 		return err

pkg/controller/cluster/main-controller.go

@@ -125,6 +125,9 @@ type Controller struct {
 	// recorder is an event recorder for recording Event resources to the
 	// Kubernetes API.
 	recorder record.EventRecorder
+
+	// Use a go template to render the hosts string
+	hostsTemplate string
 }
 
 // NewController returns a new sample controller
@@ -136,7 +139,8 @@ func NewController(
 	deploymentInformer appsinformers.DeploymentInformer,
 	jobInformer batchinformers.JobInformer,
 	minioInstanceInformer informers.MinIOInstanceInformer,
-	serviceInformer coreinformers.ServiceInformer) *Controller {
+	serviceInformer coreinformers.ServiceInformer,
+	hostsTemplate string) *Controller {
 
 	// Create event broadcaster
 	// Add minio-controller types to the default Kubernetes Scheme so Events can be
@@ -164,6 +168,7 @@ func NewController(
 		serviceListerSynced:     serviceInformer.Informer().HasSynced,
 		workqueue:               queue.NewNamedRateLimitingQueue(queue.DefaultControllerRateLimiter(), "MinIOInstances"),
 		recorder:                recorder,
+		hostsTemplate:           hostsTemplate,
 	}
 
 	klog.Info("Setting up event handlers")
@@ -411,7 +416,7 @@ func (c *Controller) syncHandler(key string) error {
 			if err != nil {
 				return err
 			}
-			ss = statefulsets.NewForMinIO(mi, hlSvc.Name)
+			ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate)
 			ss, err = c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Create(ctx, ss, cOpts)
 			if err != nil {
 				return err
@@ -450,7 +455,7 @@ func (c *Controller) syncHandler(key string) error {
 				}
 			}
 
-			ss = statefulsets.NewForMinIO(mi, hlSvc.Name)
+			ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate)
 			klog.V(2).Infof("Removing the existing StatefulSet %s with replicas: %d", name, *ss.Spec.Replicas)
 			if err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Delete(ctx, ss.Name, metav1.DeleteOptions{}); err != nil {
 				return err
@@ -476,7 +481,7 @@ func (c *Controller) syncHandler(key string) error {
 				return err
 			}
 			klog.V(4).Infof("Updating MinIOInstance %s MinIO server version %s, to: %s", name, mi.Spec.Image, ss.Spec.Template.Spec.Containers[0].Image)
-			ss = statefulsets.NewForMinIO(mi, hlSvc.Name)
+			ss = statefulsets.NewForMinIO(mi, hlSvc.Name, c.hostsTemplate)
 			if _, err := c.kubeClientSet.AppsV1().StatefulSets(mi.Namespace).Update(ctx, ss, uOpts); err != nil {
 				return err
 			}

pkg/resources/statefulsets/minio-statefulset.go

@@ -183,15 +183,19 @@ func probes(mi *miniov1.MinIOInstance) (readiness, liveness *corev1.Probe) {
 }
 
 // Builds the MinIO container for a MinIOInstance.
-func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string) corev1.Container {
+func minioServerContainer(mi *miniov1.MinIOInstance, serviceName string, hostsTemplate string) corev1.Container {
 	args := []string{"server", "--certs-dir", "/tmp/certs"}
 
 	if mi.Spec.Zones[0].Servers == 1 {
 		// to run in standalone mode we must pass the path
 		args = append(args, mi.VolumePath())
 	} else {
 		// append all the MinIOInstance replica URLs
-		for _, h := range mi.MinIOHosts() {
+		hosts := mi.MinIOHosts()
+		if hostsTemplate != "" {
+			hosts = mi.TemplatedMinIOHosts(hostsTemplate)
+		}
+		for _, h := range hosts {
 			args = append(args, fmt.Sprintf("%s://"+h+"%s", miniov1.Scheme, mi.VolumePath()))
 		}
 	}
@@ -245,7 +249,7 @@ func getVolumesForContainer(mi *miniov1.MinIOInstance) []corev1.Volume {
 }
 
 // NewForMinIO creates a new StatefulSet for the given Cluster.
-func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string) *appsv1.StatefulSet {
+func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string, hostsTemplate string) *appsv1.StatefulSet {
 	// If a PV isn't specified just use a EmptyDir volume
 	var podVolumes = getVolumesForContainer(mi)
 	var replicas = mi.MinIOReplicas()
@@ -327,7 +331,7 @@ func NewForMinIO(mi *miniov1.MinIOInstance, serviceName string) *appsv1.Stateful
 		})
 	}
 
-	containers := []corev1.Container{minioServerContainer(mi, serviceName)}
+	containers := []corev1.Container{minioServerContainer(mi, serviceName, hostsTemplate)}
 
 	ss := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{