Add support for running cortex through docker image without root uid

cmd/cortex/Dockerfile

@@ -1,7 +1,22 @@
 FROM       alpine:3.13
-RUN        apk add --no-cache ca-certificates
+RUN        apk add --no-cache ca-certificates libcap
+
+# create cortex user to run cortex as non-root
+RUN addgroup -g 10000 -S cortex && \
+    adduser  -u 10000 -S cortex -G cortex
+
+RUN mkdir /data && \
+    chown cortex:cortex /data
+
+VOLUME /data
+WORKDIR /data
+
 COPY       migrations /migrations/
 COPY       cortex /bin/cortex
+
+# allow cortex to bind to port 80 as non-root
+RUN setcap 'cap_net_bind_service=+ep' /bin/cortex
+
 EXPOSE     80
 ENTRYPOINT [ "/bin/cortex" ]
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/aws/aws-sdk-go v1.38.35
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
 	github.com/cespare/xxhash v1.1.0
+	github.com/coreos/go-semver v0.3.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/felixge/fgprof v0.9.1

integration/backward_compatibility_test.go

@@ -4,9 +4,11 @@ package integration
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -33,6 +35,27 @@ var (
 	}
 )
 
+// Cortex before 1.10 does not support running docker images as non-root. This
+// will make sure that we run cortex version before 1.10 with a root user.
+func useRootUserForOlderImages(image string, svc *e2ecortex.CortexService) {
+
+	// find versioned image tag
+	needle := ":v"
+	pos := strings.Index(image, needle)
+
+	// return early if needle not found (most likely :latest) image, which we want to run as non-root user
+	if pos == -1 {
+		return
+	}
+
+	// if version is before 1.10 set an explicict root user
+	ver := semver.New(image[pos+len(needle):])
+	v1_10 := semver.New("1.10.0")
+	if ver.LessThan(*v1_10) {
+		svc.SetUser("root:root")
+	}
+}
+
 func preCortex14Flags(flags map[string]string) map[string]string {
 	return e2e.MergeFlagsWithoutRemovingEmpty(flags, map[string]string{
 		// Blocks storage CLI flags removed the "experimental" prefix in 1.4.
@@ -101,6 +124,7 @@ func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage s
 
 	// Start other Cortex components (ingester running on previous version).
 	ingester1 := e2ecortex.NewIngester("ingester-1", consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage)
+	useRootUserForOlderImages(previousImage, ingester1)
 	distributor := e2ecortex.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), ChunksStorageFlags(), "")
 	require.NoError(t, s.StartAndWaitReady(distributor, ingester1))
 
@@ -169,6 +193,9 @@ func runNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T, previo
 	ingester1 := e2ecortex.NewIngester("ingester-1", consul.NetworkHTTPEndpoint(), flagsForPreviousImage, previousImage)
 	ingester2 := e2ecortex.NewIngester("ingester-2", consul.NetworkHTTPEndpoint(), flagsForPreviousImage, previousImage)
 	ingester3 := e2ecortex.NewIngester("ingester-3", consul.NetworkHTTPEndpoint(), flagsForPreviousImage, previousImage)
+	useRootUserForOlderImages(previousImage, ingester1)
+	useRootUserForOlderImages(previousImage, ingester2)
+	useRootUserForOlderImages(previousImage, ingester3)
 	distributor := e2ecortex.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), flagsForNewImage, "")
 	require.NoError(t, s.StartAndWaitReady(distributor, ingester1, ingester2, ingester3))
 
@@ -231,6 +258,7 @@ func checkQueries(
 		t.Run(name, func(t *testing.T) {
 			// Start query-frontend.
 			queryFrontend := e2ecortex.NewQueryFrontend("query-frontend", c.queryFrontendFlags, c.queryFrontendImage)
+			useRootUserForOlderImages(c.queryFrontendImage, queryFrontend)
 			require.NoError(t, s.Start(queryFrontend))
 			defer func() {
 				require.NoError(t, s.Stop(queryFrontend))
@@ -240,6 +268,7 @@ func checkQueries(
 			querier := e2ecortex.NewQuerier("querier", consul.NetworkHTTPEndpoint(), e2e.MergeFlagsWithoutRemovingEmpty(c.querierFlags, map[string]string{
 				"-querier.frontend-address": queryFrontend.NetworkGRPCEndpoint(),
 			}), c.querierImage)
+			useRootUserForOlderImages(c.querierImage, querier)
 
 			require.NoError(t, s.Start(querier))
 			defer func() {

integration/ca/ca.go

@@ -79,7 +79,9 @@ func (ca *CA) WriteCertificate(template *x509.Certificate, certPath string, keyP
 		return err
 	}
 
-	if err := writeExclusivePEMFile(keyPath, "PRIVATE KEY", 0600, keyBytes); err != nil {
+	// TODO: private keys should not be worldreadable. This is required when
+	// the container is run as non-root user
+	if err := writeExclusivePEMFile(keyPath, "PRIVATE KEY", 0644, keyBytes); err != nil {
 		return err
 	}
 

integration/e2e/util.go

@@ -167,5 +167,11 @@ func GetTempDirectory() (string, error) {
 		return "", err
 	}
 
+	// change mode of temporary directory to support non-root containers.
+	if err := os.Chmod(absDir, 0777); err != nil {
+		_ = os.RemoveAll(tmpDir)
+		return "", err
+	}
+
 	return absDir, nil
 }

integration/e2ecortex/service.go

@@ -18,10 +18,13 @@ func NewCortexService(
 	grpcPort int,
 	otherPorts ...int,
 ) *CortexService {
-	return &CortexService{
+	svc := &CortexService{
 		HTTPService: e2e.NewHTTPService(name, image, command, readiness, httpPort, append(otherPorts, grpcPort)...),
 		grpcPort:    grpcPort,
 	}
+	// run all tests as non-root user
+	svc.SetUser("cortex:cortex")
+	return svc
 }
 
 func (s *CortexService) GRPCEndpoint() string {

integration/e2ecortex/services.go

@@ -384,6 +384,7 @@ func NewRuler(name string, consulAddress string, flags map[string]string, image
 			// Configure the ingesters ring backend
 			"-ring.store":      "consul",
 			"-consul.hostname": consulAddress,
+			"-ruler.rule-path": "/data/rules",
 		}, flags))...),
 		e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299),
 		httpPort,