Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use HTTPS in gitops run S3 bucket server #3106

Merged
merged 1 commit into from Dec 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
use HTTPS in gitops run S3 bucket server
When installing the dev bucket server on the cluster, it is now
configured with a self-signed certificate generated on demand. The
certificate is valid for 3 days which should be enough for a typical
`gitops run` use case.

Instead of using HTTP in the forwarding from the host machine into the
bucket server pod, the CLI now uses HTTPS so that traffic can't be
intercepted on its way between the developer's machine and the
cluster.

The certificate, after being generated, is put into a Kubernetes
Secret which is in turn mounted into the bucket server pod.
  • Loading branch information
makkes committed Dec 6, 2022
commit ce2bbff0a3609c33396050ed544a5a21f8d0797f
4 changes: 2 additions & 2 deletions Makefile
Expand Up @@ -9,7 +9,7 @@ GIT_COMMIT?=$(shell which git > /dev/null && git log -n1 --pretty='%h')
VERSION?=$(shell which git > /dev/null && git describe --always --match "v*")
FLUX_VERSION=0.37.0
CHART_VERSION=$(shell which yq > /dev/null && yq e '.version' charts/gitops-server/Chart.yaml)
DEV_BUCKET_CONTAINER_IMAGE=ghcr.io/weaveworks/gitops-bucket-server@sha256:8fbb7534e772e14ea598d287a4b54a3f556416cac6621095ce45f78346fda78a
DEV_BUCKET_CONTAINER_IMAGE?=ghcr.io/weaveworks/gitops-bucket-server@sha256:157fa617e893e3ab0239547d8f1e820664b10c849fbd652c7f8738920b842f13
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This lets devs override the image by issuing

DEV_BUCKET_CONTAINER_IMAGE=something make gitops

TIER=oss

# Go build args
Expand All @@ -31,7 +31,7 @@ LDFLAGS?=-X github.com/weaveworks/weave-gitops/cmd/gitops/version.Branch=$(BRANC
# Docker args
# LDFLAGS is passed so we don't have to copy the entire .git directory into the image
# just to get, e.g. the commit hash
DOCKERARGS:=--build-arg FLUX_VERSION=$(FLUX_VERSION) --build-arg LDFLAGS="$(LDFLAGS)" --build-arg GIT_COMMIT=$(GIT_COMMIT)
DOCKERARGS+=--build-arg FLUX_VERSION=$(FLUX_VERSION) --build-arg LDFLAGS="$(LDFLAGS)" --build-arg GIT_COMMIT=$(GIT_COMMIT)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is necessary so devs can provide flags to Docker's build process, e.g. --load or --push.

# We want to be able to reference this in builds & pushes
DEFAULT_DOCKER_REPO=localhost:5001
DOCKER_REGISTRY?=$(DEFAULT_DOCKER_REPO)
Expand Down
39 changes: 15 additions & 24 deletions cmd/gitops/beta/run/cmd.go
Expand Up @@ -22,8 +22,6 @@ import (
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fsnotify/fsnotify"
"github.com/manifoldco/promptui"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/spf13/cobra"
"github.com/weaveworks/weave-gitops/cmd/gitops/cmderrors"
"github.com/weaveworks/weave-gitops/cmd/gitops/config"
Expand All @@ -35,6 +33,7 @@ import (
"github.com/weaveworks/weave-gitops/pkg/run/bootstrap"
"github.com/weaveworks/weave-gitops/pkg/run/install"
"github.com/weaveworks/weave-gitops/pkg/run/watch"
"github.com/weaveworks/weave-gitops/pkg/s3"
"github.com/weaveworks/weave-gitops/pkg/validate"
"github.com/weaveworks/weave-gitops/pkg/version"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -615,21 +614,22 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {

// ====================== Dev-bucket ======================
// Install dev-bucket server before everything, so that we can also forward logs to it
unusedPorts, err := run.GetUnusedPorts(1)
unusedPorts, err := run.GetUnusedPorts(2)
if err != nil {
cancel()
return err
}

devBucketPort := unusedPorts[0]
cancelDevBucketPortForwarding, err := watch.InstallDevBucketServer(ctx, log0, kubeClient, cfg, devBucketPort)
devBucketHTTPPort := unusedPorts[0]
pjbgf marked this conversation as resolved.
Show resolved Hide resolved
devBucketHTTPSPort := unusedPorts[1]

cancelDevBucketPortForwarding, cert, err := watch.InstallDevBucketServer(ctx, log0, kubeClient, cfg, devBucketHTTPPort, devBucketHTTPSPort)
if err != nil {
cancel()
return err
return fmt.Errorf("unable to install S3 bucket server: %w", err)
}

log, err := logger.NewS3LogWriter(sessionName, fmt.Sprintf("localhost:%d", devBucketPort), log0)
log, err := logger.NewS3LogWriter(sessionName, fmt.Sprintf("localhost:%d", devBucketHTTPSPort), cert, log0)
if err != nil {
cancel()
return err
Expand Down Expand Up @@ -666,7 +666,7 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
Namespace: flags.Namespace,
Path: paths.TargetDir,
Timeout: flags.Timeout,
DevBucketPort: devBucketPort,
DevBucketPort: devBucketHTTPPort,
SessionName: sessionName,
Username: username,
}
Expand All @@ -683,16 +683,7 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
}
}

ignorer := watch.CreateIgnorer(paths.RootDir)
minioClient, err := minio.New(
"localhost:"+strconv.Itoa(int(devBucketPort)),
&minio.Options{
Creds: credentials.NewStaticV4("user", "doesn't matter", ""),
Secure: false,
BucketLookup: minio.BucketLookupPath,
},
)

minioClient, err := s3.NewMinioClient("localhost:"+strconv.Itoa(int(devBucketHTTPSPort)), cert)
if err != nil {
cancel()
return err
Expand All @@ -705,6 +696,8 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
return err
}

ignorer := watch.CreateIgnorer(paths.RootDir)

err = filepath.Walk(paths.RootDir, watch.WatchDirsForFileWalker(watcher, ignorer))
if err != nil {
cancel()
Expand Down Expand Up @@ -1043,12 +1036,10 @@ func runBootstrap(ctx context.Context, log logger.Logger, paths *run.Paths, mani

workloadKustomizationContentStr := string(workloadKustomizationContent)

commitFiles := []gitprovider.CommitFile{
gitprovider.CommitFile{
Path: &workloadKustomizationPath,
Content: &workloadKustomizationContentStr,
},
}
commitFiles := []gitprovider.CommitFile{{
Path: &workloadKustomizationPath,
Content: &workloadKustomizationContentStr,
}}

if len(manifests) > 0 {
strManifests := string(manifests)
Expand Down
14 changes: 3 additions & 11 deletions pkg/logger/s3_log_writer.go
Expand Up @@ -8,7 +8,7 @@ import (

"github.com/go-logr/logr"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/weaveworks/weave-gitops/pkg/s3"
)

type S3LogWriter struct {
Expand All @@ -23,16 +23,8 @@ func (l *S3LogWriter) L() logr.Logger {
return l.log0.L()
}

func NewS3LogWriter(id string, endpoint string, log0 Logger) (Logger, error) {
minioClient, err := minio.New(
endpoint,
&minio.Options{
Creds: credentials.NewStaticV4("user", "doesn't matter", ""),
Secure: false,
BucketLookup: minio.BucketLookupPath,
},
)

func NewS3LogWriter(id, endpoint string, caCert []byte, log0 Logger) (Logger, error) {
minioClient, err := s3.NewMinioClient(endpoint, caCert)
if err != nil {
return nil, err
}
Expand Down
83 changes: 67 additions & 16 deletions pkg/run/watch/install_dev_bucket_server.go
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/weaveworks/weave-gitops/pkg/logger"
"github.com/weaveworks/weave-gitops/pkg/run"
"github.com/weaveworks/weave-gitops/pkg/tls"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -29,11 +30,11 @@ var (
// The variables below are to be set by flags passed to `go build`.
// Examples: -X run.DevBucketContainerImage=xxxxx

DevBucketContainerImage = "ghcr.io/weaveworks/gitops-bucket-server@sha256:8fbb7534e772e14ea598d287a4b54a3f556416cac6621095ce45f78346fda78a"
DevBucketContainerImage = "ghcr.io/weaveworks/gitops-bucket-server:1670322194"
)

// InstallDevBucketServer installs the dev bucket server, open port forwarding, and returns a function that can be used to the port forwarding.
func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient client.Client, config *rest.Config, devBucketPort int32) (func(), error) {
func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient client.Client, config *rest.Config, httpPort, httpsPort int32) (func(), []byte, error) {
var (
err error
devBucketAppLabels = map[string]string{
Expand All @@ -57,7 +58,7 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
if err != nil && apierrors.IsNotFound(err) {
if err := kubeClient.Create(ctx, &devBucketNamespace); err != nil {
log.Failuref("Error creating namespace %s: %v", GitOpsRunNamespace, err.Error())
return nil, err
return nil, nil, err
} else {
log.Successf("Created namespace %s", GitOpsRunNamespace)
}
Expand All @@ -76,8 +77,12 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{
{
Name: RunDevBucketName,
Port: devBucketPort,
Name: fmt.Sprintf("%s-http", RunDevBucketName),
Port: httpPort,
},
{
Name: fmt.Sprintf("%s-https", RunDevBucketName),
Port: httpsPort,
},
},
Selector: devBucketAppLabels,
Expand All @@ -93,14 +98,38 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
if err != nil && apierrors.IsNotFound(err) {
if err := kubeClient.Create(ctx, &devBucketService); err != nil {
log.Failuref("Error creating service %s/%s: %v", GitOpsRunNamespace, RunDevBucketName, err.Error())
return nil, err
return nil, nil, err
} else {
log.Successf("Created service %s/%s", GitOpsRunNamespace, RunDevBucketName)
}
} else if err == nil {
log.Successf("Service %s/%s already existed", GitOpsRunNamespace, RunDevBucketName)
}

cert, err := tls.GenerateSelfSignedCertificate("localhost", fmt.Sprintf("%s.%s.svc.cluster.local", devBucketService.Name, devBucketService.Namespace))
if err != nil {
err = fmt.Errorf("failed generating self-signed certificate for dev bucket server: %w", err)
log.Failuref(err.Error())

return nil, nil, err
}

certsSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "dev-bucket-server-certs",
Namespace: GitOpsRunNamespace,
Labels: devBucketAppLabels,
},
Data: map[string][]byte{
"cert.pem": cert.Cert,
"cert.key": cert.Key,
},
}
if err := kubeClient.Create(ctx, certsSecret); err != nil {
log.Failuref("Error creating Secret %s/%s: %v", certsSecret.Namespace, certsSecret.Name, err.Error())
return nil, nil, err
}

// create deployment
replicas := int32(1)
devBucketDeployment := appsv1.Deployment{
Expand All @@ -119,21 +148,43 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
Labels: devBucketAppLabels,
},
Spec: corev1.PodSpec{
Volumes: []corev1.Volume{{
Name: "certs",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "dev-bucket-server-certs",
},
},
}},
Containers: []corev1.Container{
{
Name: RunDevBucketName,
Image: DevBucketContainerImage,
Name: RunDevBucketName,
Image: DevBucketContainerImage,
ImagePullPolicy: corev1.PullIfNotPresent,
Env: []corev1.EnvVar{
{Name: "MINIO_ROOT_USER", Value: "user"},
{Name: "MINIO_ROOT_PASSWORD", Value: "doesn't matter"},
},
Ports: []corev1.ContainerPort{
{
ContainerPort: devBucketPort,
HostPort: devBucketPort,
ContainerPort: httpPort,
HostPort: httpPort,
},
{
ContainerPort: httpsPort,
HostPort: httpsPort,
},
},
Args: []string{strconv.Itoa(int(devBucketPort))},
Args: []string{
fmt.Sprintf("--http-port=%d", httpPort),
fmt.Sprintf("--https-port=%d", httpsPort),
"--cert-file=/tmp/certs/cert.pem",
"--key-file=/tmp/certs/cert.key",
},
VolumeMounts: []corev1.VolumeMount{{
Name: "certs",
MountPath: "/tmp/certs",
}},
},
},
RestartPolicy: corev1.RestartPolicyAlways,
Expand All @@ -151,7 +202,7 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
if err != nil && apierrors.IsNotFound(err) {
if err := kubeClient.Create(ctx, &devBucketDeployment); err != nil {
log.Failuref("Error creating deployment %s/%s: %v", GitOpsRunNamespace, RunDevBucketName, err.Error())
return nil, err
return nil, nil, err
} else {
log.Successf("Created deployment %s/%s", GitOpsRunNamespace, RunDevBucketName)
}
Expand Down Expand Up @@ -189,8 +240,8 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
Name: RunDevBucketName,
Namespace: GitOpsRunNamespace,
Kind: "service",
HostPort: strconv.Itoa(int(devBucketPort)),
ContainerPort: strconv.Itoa(int(devBucketPort)),
HostPort: strconv.Itoa(int(httpsPort)),
ContainerPort: strconv.Itoa(int(httpsPort)),
}
// get pod from specMap
namespacedName := types.NamespacedName{Namespace: specMap.Namespace, Name: specMap.Name}
Expand Down Expand Up @@ -218,10 +269,10 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c

log.Successf("Port forwarding for %s is ready.", RunDevBucketName)

return cancelPortFwd, nil
return cancelPortFwd, cert.Cert, nil
}

return nil, fmt.Errorf("pod not found")
return nil, nil, fmt.Errorf("pod not found")
}

// UninstallDevBucketServer deletes the dev-bucket namespace.
Expand Down
58 changes: 58 additions & 0 deletions pkg/s3/minio.go
@@ -0,0 +1,58 @@
package s3

import (
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"

"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)

func NewMinioClient(endpoint string, caCert []byte) (*minio.Client, error) {
tr, err := NewTLSRoundTripper(caCert)
if err != nil {
return nil, fmt.Errorf("failed creating transport: %w", err)
}

return minio.New(
endpoint,
&minio.Options{
Creds: credentials.NewStaticV4("user", "doesn't matter", ""),
Secure: true,
BucketLookup: minio.BucketLookupPath,
Transport: tr,
},
)
}

func NewTLSRoundTripper(caCert []byte) (http.RoundTripper, error) {
tr, err := minio.DefaultTransport(true)
if err != nil {
return nil, fmt.Errorf("failed creating default transport: %w", err)
}

tr.TLSClientConfig = &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
}
rootCAs := mustGetSystemCertPool()

rootCAs.AppendCertsFromPEM(caCert)
tr.TLSClientConfig.RootCAs = rootCAs

return tr, nil
}

// mustGetSystemCertPool - return system CAs or empty pool in case of error (or windows)
func mustGetSystemCertPool() *x509.CertPool {
pool, err := x509.SystemCertPool()
if err != nil {
return x509.NewCertPool()
}

return pool
}