registry.go

package registry

import (
	cryptorand "crypto/rand"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"strconv"
	"strings"

	"github.com/spf13/cobra"
	kapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/client/restclient"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/intstr"

	authapi "github.com/openshift/origin/pkg/authorization/api"
	cmdutil "github.com/openshift/origin/pkg/cmd/util"
	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
	"github.com/openshift/origin/pkg/cmd/util/variable"
	configcmd "github.com/openshift/origin/pkg/config/cmd"
	deployapi "github.com/openshift/origin/pkg/deploy/api"
	"github.com/openshift/origin/pkg/generate/app"
)

const (
	registryLong = `
Install or configure an integrated Docker registry

This command sets up a Docker registry integrated with your cluster to provide notifications when
images are pushed. With no arguments, the command will check for the existing registry service
called 'docker-registry' and try to create it. If you want to test whether the registry has
been created add the --dry-run flag and the command will exit with 1 if the registry does not
exist.

To run a highly available registry, you should be using a remote storage mechanism like an
object store (several are supported by the Docker registry). The default Docker registry image
is configured to accept configuration as environment variables - refer to the configuration file in
that image for more on setting up alternative storage. Once you've made those changes, you can
pass --replicas=2 or higher to ensure you have failover protection. The default registry setup
uses a local volume and the data will be lost if you delete the running pod.

If multiple ports are specified using the option --ports, the first specified port will be
chosen for use as the REGISTRY_HTTP_ADDR and will be passed to Docker registry.

NOTE: This command is intended to simplify the tasks of setting up a Docker registry in a new
  installation. Some configuration beyond this command is still required to make
  your registry persist data.`

	registryExample = `  # Check if default Docker registry ("docker-registry") has been created
  %[1]s %[2]s --dry-run

  # See what the registry will look like if created
  %[1]s %[2]s -o yaml

  # Create a registry with two replicas if it does not exist
  %[1]s %[2]s --replicas=2

  # Use a different registry image
  %[1]s %[2]s --images=myrepo/docker-registry:mytag

  # Enforce quota and limits on images
  %[1]s %[2]s --enforce-quota`
)

// RegistryOptions contains the configuration for the registry as well as any other
// helpers required to run the command.
type RegistryOptions struct {
	Config *RegistryConfig

	// helpers required for Run.
	factory       *clientcmd.Factory
	cmd           *cobra.Command
	out           io.Writer
	label         map[string]string
	nodeSelector  map[string]string
	ports         []kapi.ContainerPort
	namespace     string
	serviceClient kclient.ServicesNamespacer
	image         string
}

// RegistryConfig contains configuration for the registry that will be created.
type RegistryConfig struct {
	Action configcmd.BulkAction

	Name           string
	Type           string
	ImageTemplate  variable.ImageTemplate
	Ports          string
	Replicas       int32
	Labels         string
	Volume         string
	HostMount      string
	DryRun         bool
	Credentials    string
	Selector       string
	ServiceAccount string
	DaemonSet      bool
	EnforceQuota   bool

	ServingCertPath string
	ServingKeyPath  string

	// TODO: accept environment values.
}

// randomSecretSize is the number of random bytes to generate.
const randomSecretSize = 32

const (
	defaultLabel = "docker-registry=default"
	defaultPort  = 5000
	/* TODO: `/healthz` has been deprecated by `/`; keep it temporarily for backwards compatibility until
	 * a next major release with a strict requirement on newer registry image
	 * NOTE that `/` is supported since ose `v3.1.1.0`
	 * To make the transition safe, we could change `HTTPGetAction` to an `ExecAction` which would first curl
	 * `/` and then fallback to `/healthz` if unreachable. Reachable endpoint could be cached on tmpfs inside
	 * a container and be used on subsequent checks. */
	healthzRoute               = "/healthz"
	healthzRouteTimeoutSeconds = 5
	// this is the official private certificate path on Red Hat distros, and is at least structurally more
	// correct than ubuntu based distributions which don't distinguish between public and private certs.
	// Since Origin is CentOS based this is more likely to work.  Ubuntu images should symlink this directory
	// into /etc/ssl/certs to be compatible.
	defaultCertificateDir = "/etc/pki/tls/private"
)

// NewCmdRegistry implements the OpenShift cli registry command
func NewCmdRegistry(f *clientcmd.Factory, parentName, name string, out io.Writer) *cobra.Command {
	cfg := &RegistryConfig{
		ImageTemplate:  variable.NewDefaultImageTemplate(),
		Name:           "registry",
		Labels:         defaultLabel,
		Ports:          strconv.Itoa(defaultPort),
		Volume:         "/registry",
		ServiceAccount: "registry",
		Replicas:       1,
		EnforceQuota:   false,
	}

	cmd := &cobra.Command{
		Use:     name,
		Short:   "Install the integrated Docker registry",
		Long:    registryLong,
		Example: fmt.Sprintf(registryExample, parentName, name),
		Run: func(cmd *cobra.Command, args []string) {
			opts := &RegistryOptions{
				Config: cfg,
			}
			kcmdutil.CheckErr(opts.Complete(f, cmd, out, args))
			kcmdutil.CheckErr(opts.RunCmdRegistry())
		},
	}

	cmd.Flags().StringVar(&cfg.Type, "type", "docker-registry", "The registry image to use - if you specify --images this flag may be ignored.")
	cmd.Flags().StringVar(&cfg.ImageTemplate.Format, "images", cfg.ImageTemplate.Format, "The image to base this registry on - ${component} will be replaced with --type")
	cmd.Flags().BoolVar(&cfg.ImageTemplate.Latest, "latest-images", cfg.ImageTemplate.Latest, "If true, attempt to use the latest image for the registry instead of the latest release.")
	cmd.Flags().StringVar(&cfg.Ports, "ports", cfg.Ports, fmt.Sprintf("A comma delimited list of ports or port pairs to expose on the registry pod. The default is set for %d.", defaultPort))
	cmd.Flags().Int32Var(&cfg.Replicas, "replicas", cfg.Replicas, "The replication factor of the registry; commonly 2 when high availability is desired.")
	cmd.Flags().StringVar(&cfg.Labels, "labels", cfg.Labels, "A set of labels to uniquely identify the registry and its components.")
	cmd.Flags().StringVar(&cfg.Volume, "volume", cfg.Volume, "The volume path to use for registry storage; defaults to /registry which is the default for origin-docker-registry.")
	cmd.Flags().StringVar(&cfg.HostMount, "mount-host", cfg.HostMount, "If set, the registry volume will be created as a host-mount at this path.")
	cmd.Flags().Bool("create", false, "deprecated; this is now the default behavior")
	cmd.Flags().StringVar(&cfg.Credentials, "credentials", "", "Path to a .kubeconfig file that will contain the credentials the registry should use to contact the master.")
	cmd.Flags().StringVar(&cfg.ServiceAccount, "service-account", cfg.ServiceAccount, "Name of the service account to use to run the registry pod.")
	cmd.Flags().StringVar(&cfg.Selector, "selector", cfg.Selector, "Selector used to filter nodes on deployment. Used to run registries on a specific set of nodes.")
	cmd.Flags().StringVar(&cfg.ServingCertPath, "tls-certificate", cfg.ServingCertPath, "An optional path to a PEM encoded certificate (which may contain the private key) for serving over TLS")
	cmd.Flags().StringVar(&cfg.ServingKeyPath, "tls-key", cfg.ServingKeyPath, "An optional path to a PEM encoded private key for serving over TLS")
	cmd.Flags().BoolVar(&cfg.DaemonSet, "daemonset", cfg.DaemonSet, "Use a daemonset instead of a deployment config.")
	cmd.Flags().BoolVar(&cfg.EnforceQuota, "enforce-quota", cfg.EnforceQuota, "If set, the registry will refuse to write blobs if they exceed quota limits")

	// autocompletion hints
	cmd.MarkFlagFilename("credentials", "kubeconfig")

	// Deprecate credentials
	cmd.Flags().MarkDeprecated("credentials", "use --service-account to specify the service account the registry will use to make API calls")

	cfg.Action.BindForOutput(cmd.Flags())
	cmd.Flags().String("output-version", "", "The preferred API versions of the output objects")

	return cmd
}

// Complete completes any options that are required by validate or run steps.
func (opts *RegistryOptions) Complete(f *clientcmd.Factory, cmd *cobra.Command, out io.Writer, args []string) error {
	if len(args) > 0 {
		return kcmdutil.UsageError(cmd, "No arguments are allowed to this command")
	}

	opts.image = opts.Config.ImageTemplate.ExpandOrDie(opts.Config.Type)

	opts.label = map[string]string{
		"docker-registry": "default",
	}
	if opts.Config.Labels != defaultLabel {
		valid, remove, err := app.LabelsFromSpec(strings.Split(opts.Config.Labels, ","))
		if err != nil {
			return err
		}
		if len(remove) > 0 {
			return kcmdutil.UsageError(cmd, "You may not pass negative labels in %q", opts.Config.Labels)
		}
		opts.label = valid
	}

	opts.nodeSelector = map[string]string{}
	if len(opts.Config.Selector) > 0 {
		valid, remove, err := app.LabelsFromSpec(strings.Split(opts.Config.Selector, ","))
		if err != nil {
			return err
		}
		if len(remove) > 0 {
			return kcmdutil.UsageError(cmd, "You may not pass negative labels in selector %q", opts.Config.Selector)
		}
		opts.nodeSelector = valid
	}

	var portsErr error
	if opts.ports, portsErr = app.ContainerPortsFromString(opts.Config.Ports); portsErr != nil {
		return portsErr
	}

	var nsErr error
	if opts.namespace, _, nsErr = f.OpenShiftClientConfig.Namespace(); nsErr != nil {
		return fmt.Errorf("error getting namespace: %v", nsErr)
	}

	var kClientErr error
	if _, opts.serviceClient, kClientErr = f.Clients(); kClientErr != nil {
		return fmt.Errorf("error getting client: %v", kClientErr)
	}

	opts.Config.Action.Bulk.Mapper = clientcmd.ResourceMapper(f)
	opts.Config.Action.Out, opts.Config.Action.ErrOut = out, cmd.OutOrStderr()
	opts.Config.Action.Bulk.Op = configcmd.Create
	opts.out = out
	opts.cmd = cmd
	opts.factory = f

	return nil
}

// RunCmdRegistry contains all the necessary functionality for the OpenShift cli registry command
func (opts *RegistryOptions) RunCmdRegistry() error {
	name := "docker-registry"

	var clusterIP string

	output := opts.Config.Action.ShouldPrint()
	generate := output
	service, err := opts.serviceClient.Services(opts.namespace).Get(name)
	if err != nil {
		if !generate {
			if !errors.IsNotFound(err) {
				return fmt.Errorf("can't check for existing docker-registry %q: %v", name, err)
			}
			if opts.Config.Action.DryRun {
				return fmt.Errorf("Docker registry %q service does not exist", name)
			}
			generate = true
		}
	} else {
		clusterIP = service.Spec.ClusterIP
	}

	if !generate {
		fmt.Fprintf(opts.out, "Docker registry %q service exists\n", name)
		return nil
	}

	// create new registry
	secretEnv := app.Environment{}
	switch {
	case len(opts.Config.ServiceAccount) == 0 && len(opts.Config.Credentials) == 0:
		return fmt.Errorf("registry could not be created; a service account or the path to a .kubeconfig file must be provided")
	case len(opts.Config.Credentials) > 0:
		clientConfigLoadingRules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: opts.Config.Credentials}
		credentials, err := clientConfigLoadingRules.Load()
		if err != nil {
			return fmt.Errorf("registry does not exist; the provided credentials %q could not be loaded: %v", opts.Config.Credentials, err)
		}
		config, err := kclientcmd.NewDefaultClientConfig(*credentials, &kclientcmd.ConfigOverrides{}).ClientConfig()
		if err != nil {
			return fmt.Errorf("registry does not exist; the provided credentials %q could not be used: %v", opts.Config.Credentials, err)
		}
		if err := restclient.LoadTLSFiles(config); err != nil {
			return fmt.Errorf("registry does not exist; the provided credentials %q could not load certificate info: %v", opts.Config.Credentials, err)
		}
		insecure := "false"
		if config.Insecure {
			insecure = "true"
		} else {
			if len(config.KeyData) == 0 || len(config.CertData) == 0 {
				return fmt.Errorf("registry does not exist; the provided credentials %q are missing the client certificate and/or key", opts.Config.Credentials)
			}
		}

		secretEnv = app.Environment{
			"OPENSHIFT_MASTER":    config.Host,
			"OPENSHIFT_CA_DATA":   string(config.CAData),
			"OPENSHIFT_KEY_DATA":  string(config.KeyData),
			"OPENSHIFT_CERT_DATA": string(config.CertData),
			"OPENSHIFT_INSECURE":  insecure,
		}
	}

	needServiceAccountRole := len(opts.Config.ServiceAccount) > 0 && len(opts.Config.Credentials) == 0

	var servingCert, servingKey []byte
	if len(opts.Config.ServingCertPath) > 0 {
		data, err := ioutil.ReadFile(opts.Config.ServingCertPath)
		if err != nil {
			return fmt.Errorf("registry does not exist; could not load TLS certificate file %q: %v", opts.Config.ServingCertPath, err)
		}
		servingCert = data
	}
	if len(opts.Config.ServingKeyPath) > 0 {
		data, err := ioutil.ReadFile(opts.Config.ServingKeyPath)
		if err != nil {
			return fmt.Errorf("registry does not exist; could not load TLS private key file %q: %v", opts.Config.ServingKeyPath, err)
		}
		servingKey = data
	}

	env := app.Environment{}
	env.Add(secretEnv)

	env["REGISTRY_MIDDLEWARE_REPOSITORY_OPENSHIFT_ENFORCEQUOTA"] = fmt.Sprintf("%t", opts.Config.EnforceQuota)
	healthzPort := defaultPort
	if len(opts.ports) > 0 {
		healthzPort = int(opts.ports[0].ContainerPort)
		env["REGISTRY_HTTP_ADDR"] = fmt.Sprintf(":%d", healthzPort)
		env["REGISTRY_HTTP_NET"] = "tcp"
	}
	secrets, volumes, mounts, extraEnv, tls, err := generateSecretsConfig(opts.Config, opts.namespace, servingCert, servingKey)
	if err != nil {
		return err
	}
	env.Add(extraEnv)

	livenessProbe := generateLivenessProbeConfig(healthzPort, tls)
	readinessProbe := generateReadinessProbeConfig(healthzPort, tls)

	mountHost := len(opts.Config.HostMount) > 0
	podTemplate := &kapi.PodTemplateSpec{
		ObjectMeta: kapi.ObjectMeta{Labels: opts.label},
		Spec: kapi.PodSpec{
			NodeSelector: opts.nodeSelector,
			Containers: []kapi.Container{
				{
					Name:  "registry",
					Image: opts.image,
					Ports: opts.ports,
					Env:   env.List(),
					VolumeMounts: append(mounts, kapi.VolumeMount{
						Name:      "registry-storage",
						MountPath: opts.Config.Volume,
					}),
					SecurityContext: &kapi.SecurityContext{
						Privileged: &mountHost,
					},
					LivenessProbe:  livenessProbe,
					ReadinessProbe: readinessProbe,
					Resources: kapi.ResourceRequirements{
						Requests: kapi.ResourceList{
							kapi.ResourceCPU:    resource.MustParse("100m"),
							kapi.ResourceMemory: resource.MustParse("256Mi"),
						},
					},
				},
			},
			Volumes: append(volumes, kapi.Volume{
				Name:         "registry-storage",
				VolumeSource: kapi.VolumeSource{},
			}),
			ServiceAccountName: opts.Config.ServiceAccount,
		},
	}
	if mountHost {
		podTemplate.Spec.Volumes[len(podTemplate.Spec.Volumes)-1].HostPath = &kapi.HostPathVolumeSource{Path: opts.Config.HostMount}
	} else {
		podTemplate.Spec.Volumes[len(podTemplate.Spec.Volumes)-1].EmptyDir = &kapi.EmptyDirVolumeSource{}
	}

	objects := []runtime.Object{}
	for _, s := range secrets {
		objects = append(objects, s)
	}
	if needServiceAccountRole {
		objects = append(objects,
			&kapi.ServiceAccount{ObjectMeta: kapi.ObjectMeta{Name: opts.Config.ServiceAccount}},
			&authapi.ClusterRoleBinding{
				ObjectMeta: kapi.ObjectMeta{Name: fmt.Sprintf("registry-%s-role", opts.Config.Name)},
				Subjects: []kapi.ObjectReference{
					{
						Kind:      "ServiceAccount",
						Name:      opts.Config.ServiceAccount,
						Namespace: opts.namespace,
					},
				},
				RoleRef: kapi.ObjectReference{
					Kind: "ClusterRole",
					Name: "system:registry",
				},
			},
		)
	}

	if opts.Config.DaemonSet {
		objects = append(objects, &extensions.DaemonSet{
			ObjectMeta: kapi.ObjectMeta{
				Name:   name,
				Labels: opts.label,
			},
			Spec: extensions.DaemonSetSpec{
				Template: kapi.PodTemplateSpec{
					ObjectMeta: podTemplate.ObjectMeta,
					Spec:       podTemplate.Spec,
				},
			},
		})
	} else {
		objects = append(objects, &deployapi.DeploymentConfig{
			ObjectMeta: kapi.ObjectMeta{
				Name:   name,
				Labels: opts.label,
			},
			Spec: deployapi.DeploymentConfigSpec{
				Replicas: opts.Config.Replicas,
				Selector: opts.label,
				Triggers: []deployapi.DeploymentTriggerPolicy{
					{Type: deployapi.DeploymentTriggerOnConfigChange},
				},
				Template: podTemplate,
			},
		})
	}

	objects = app.AddServices(objects, true)

	// Set registry service's sessionAffinity to ClientIP to prevent push
	// failures due to a use of poorly consistent storage shared by
	// multiple replicas. Also reuse the cluster IP if provided to avoid
	// changing the internal value.
	for _, obj := range objects {
		switch t := obj.(type) {
		case *kapi.Service:
			t.Spec.SessionAffinity = kapi.ServiceAffinityClientIP
			t.Spec.ClusterIP = clusterIP
		}
	}

	// TODO: label all created objects with the same label
	list := &kapi.List{Items: objects}

	if opts.Config.Action.ShouldPrint() {
		mapper, _ := opts.factory.Object(false)
		fn := cmdutil.VersionedPrintObject(opts.factory.PrintObject, opts.cmd, mapper, opts.out)
		if err := fn(list); err != nil {
			return fmt.Errorf("unable to print object: %v", err)
		}
		return nil
	}

	if errs := opts.Config.Action.WithMessage(fmt.Sprintf("Creating registry %s", opts.Config.Name), "created").Run(list, opts.namespace); len(errs) > 0 {
		return cmdutil.ErrExit
	}
	return nil
}

func generateLivenessProbeConfig(port int, https bool) *kapi.Probe {
	var scheme kapi.URIScheme
	if https {
		scheme = kapi.URISchemeHTTPS
	}
	return &kapi.Probe{
		InitialDelaySeconds: 10,
		TimeoutSeconds:      healthzRouteTimeoutSeconds,
		Handler: kapi.Handler{
			HTTPGet: &kapi.HTTPGetAction{
				Scheme: scheme,
				Path:   healthzRoute,
				Port:   intstr.FromInt(port),
			},
		},
	}
}

func generateReadinessProbeConfig(port int, https bool) *kapi.Probe {
	var scheme kapi.URIScheme
	if https {
		scheme = kapi.URISchemeHTTPS
	}
	return &kapi.Probe{
		TimeoutSeconds: healthzRouteTimeoutSeconds,
		Handler: kapi.Handler{
			HTTPGet: &kapi.HTTPGetAction{
				Scheme: scheme,
				Path:   healthzRoute,
				Port:   intstr.FromInt(port),
			},
		},
	}
}

// generateSecretsConfig generates any Secret and Volume objects, such
// as the TLS serving cert that are necessary for the registry container.
// Runs true if the registry should be served over TLS.
func generateSecretsConfig(
	cfg *RegistryConfig, namespace string, defaultCrt, defaultKey []byte,
) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount, app.Environment, bool, error) {
	var secrets []*kapi.Secret
	var volumes []kapi.Volume
	var mounts []kapi.VolumeMount
	extraEnv := app.Environment{}

	if len(defaultCrt) > 0 && len(defaultKey) == 0 {
		keys, err := cmdutil.PrivateKeysFromPEM(defaultCrt)
		if err != nil {
			return nil, nil, nil, nil, false, err
		}
		if len(keys) == 0 {
			return nil, nil, nil, nil, false, fmt.Errorf("the default cert must contain a private key")
		}
		defaultKey = keys
	}

	if len(defaultCrt) > 0 {
		secret := &kapi.Secret{
			ObjectMeta: kapi.ObjectMeta{
				Name: fmt.Sprintf("%s-certs", cfg.Name),
			},
			Type: kapi.SecretTypeTLS,
			Data: map[string][]byte{
				kapi.TLSCertKey:       defaultCrt,
				kapi.TLSPrivateKeyKey: defaultKey,
			},
		}
		secrets = append(secrets, secret)
		volume := kapi.Volume{
			Name: "server-certificate",
			VolumeSource: kapi.VolumeSource{
				Secret: &kapi.SecretVolumeSource{
					SecretName: secret.Name,
				},
			},
		}
		volumes = append(volumes, volume)

		mount := kapi.VolumeMount{
			Name:      volume.Name,
			ReadOnly:  true,
			MountPath: defaultCertificateDir,
		}
		mounts = append(mounts, mount)

		extraEnv.Add(app.Environment{
			"REGISTRY_HTTP_TLS_CERTIFICATE": path.Join(defaultCertificateDir, kapi.TLSCertKey),
			"REGISTRY_HTTP_TLS_KEY":         path.Join(defaultCertificateDir, kapi.TLSPrivateKeyKey),
		})
	}

	secretBytes := make([]byte, randomSecretSize)
	if _, err := cryptorand.Read(secretBytes); err != nil {
		return nil, nil, nil, nil, false, fmt.Errorf("registry does not exist; could not generate random bytes for HTTP secret: %v", err)
	}
	httpSecretString := base64.StdEncoding.EncodeToString(secretBytes)
	extraEnv["REGISTRY_HTTP_SECRET"] = httpSecretString

	return secrets, volumes, mounts, extraEnv, len(defaultCrt) > 0, nil
}