Add RPM and deb packaging for cortex binary (#2838)

* Use docker containers to build RPM and deb packages for cortex

Signed-off-by: Jack Baldry <jack.baldry@grafana.com>

* Add DISABLE_CLEANUP variable to prevent the removal of testing containers

Signed-off-by: Jack Baldry <jack.baldry@grafana.com>

* Updated RELEASE doc

Signed-off-by: Marco Pracucci <marco@pracucci.com>

Co-authored-by: Marco Pracucci <marco@pracucci.com>

Makefile

@@ -190,7 +190,7 @@ endif
 
 clean:
 	$(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true
-	rm -rf $(UPTODATE_FILES) $(EXES) .cache
+	rm -rf -- $(UPTODATE_FILES) $(EXES) .cache $(PACKAGES) dist/*
 	go clean ./...
 
 clean-protos:
@@ -232,7 +232,7 @@ web-serve:
 	cd website && hugo --config config.toml -v server
 
 # Generate binaries for a Cortex release
-dist:
+dist dist/cortex-linux-amd64 dist/cortex-darwin-amd64 dist/query-tee-linux-amd64 dist/query-tee-darwin-amd64 dist/cortex-linux-amd64-sha-256 dist/cortex-darwin-amd64-sha-256 dist/query-tee-linux-amd64-sha-256 dist/query-tee-darwin-amd64-sha-256:
 	rm -fr ./dist
 	mkdir -p ./dist
 	GOOS="linux"  GOARCH="amd64" CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/cortex-linux-amd64   ./cmd/cortex
@@ -243,3 +243,50 @@ dist:
 	shasum -a 256 ./dist/cortex-linux-amd64  | cut -d ' ' -f 1 > ./dist/cortex-linux-amd64-sha-256
 	shasum -a 256 ./dist/query-tee-darwin-amd64 | cut -d ' ' -f 1 > ./dist/query-tee-darwin-amd64-sha-256
 	shasum -a 256 ./dist/query-tee-linux-amd64  | cut -d ' ' -f 1 > ./dist/query-tee-linux-amd64-sha-256
+
+# Generate packages for a Cortex release.
+FPM_OPTS := fpm -s dir -v $(VERSION) -n cortex -f \
+	--license "Apache 2.0" \
+	--url "https://github.com/cortexproject/cortex"
+
+FPM_ARGS := dist/cortex-linux-amd64=/usr/local/bin/cortex \
+	docs/configuration/single-process-config.yaml=/etc/cortex/single-process-config.yaml\
+
+PACKAGES := dist/cortex-$(VERSION).rpm dist/cortex-$(VERSION).deb
+PACKAGE_IN_CONTAINER := true
+PACKAGE_IMAGE ?= $(IMAGE_PREFIX)fpm
+ifeq ($(PACKAGE_IN_CONTAINER), true)
+
+.PHONY: packages
+packages: dist/cortex-linux-amd64 packaging/fpm/$(UPTODATE)
+	@mkdir -p $(shell pwd)/.pkg
+	@mkdir -p $(shell pwd)/.cache
+	@echo ">>>> Entering build container: $@"
+	@$(SUDO) time docker run $(RM) $(TTY) \
+		-v  $(shell pwd):/src/github.com/cortexproject/cortex:delegated \
+		-i $(PACKAGE_IMAGE) $@;
+
+else
+
+packages: $(PACKAGES)
+
+dist/%.deb: dist/cortex-linux-amd64 $(wildcard packaging/deb/**)
+	$(FPM_OPTS) -t deb \
+		--after-install packaging/deb/control/postinst \
+		--before-remove packaging/deb/control/prerm \
+		--package $@ $(FPM_ARGS) \
+		packaging/deb/default/cortex=/etc/default/cortex \
+		packaging/deb/systemd/cortex.service=/etc/systemd/system/cortex.service
+
+dist/%.rpm: dist/cortex-linux-amd64 $(wildcard packaging/rpm/**)
+	$(FPM_OPTS) -t rpm  \
+		--after-install packaging/rpm/control/post \
+		--before-remove packaging/rpm/control/preun \
+		--package $@ $(FPM_ARGS) \
+		packaging/rpm/sysconfig/cortex=/etc/sysconfig/cortex \
+		packaging/rpm/systemd/cortex.service=/etc/systemd/system/cortex.service
+endif
+
+.PHONY: test-packages
+test-packages: packages packaging/rpm/centos-systemd/$(UPTODATE) packaging/deb/debian-systemd/$(UPTODATE)
+	./tools/packaging/test-packages $(IMAGE_PREFIX) $(VERSION)

RELEASE.md

@@ -89,6 +89,7 @@ To publish a release candidate:
 3. Create a pre-release in GitHub
    - Write the release notes (including a copy-paste of the changelog)
    - Build binaries with `make dist` and attach them to the release
+   - Build packages with `make packages`, test them with `make test-packages` and attach them to the release
 
 ### Publish a stable release
 
@@ -101,6 +102,7 @@ To publish a stable release:
 5. Create a release in GitHub
    - Write the release notes (including a copy-paste of the changelog)
    - Build binaries with `make dist` and attach them to the release
+   - Build packages with `make packages`, test them with `make test-packages` and attach them to the release
 6. Merge the release branch `release-x.y` to `master`
    - Merge to `master` in the local checkout
    - Fix any conflict and `git commit -s`

packaging/deb/control/postinst

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+
+[ -f /etc/sysconfig/cortex ] && . /etc/default/cortex
+
+# Initial installation: $1 == configure
+# Upgrade: $1 == 2, and configured to restart on upgrade
+case "$1" in
+  configure)
+     [ -z "$CORTEX_USER" ] && CORTEX_USER="cortex"
+     [ -z "$CORTEX_GROUP" ] && CORTEX_GROUP="cortex"
+     if ! getent group "$CORTEX_GROUP" > /dev/null 2>&1 ; then
+	 groupadd -r "$CORTEX_GROUP"
+     fi
+     if ! getent passwd "$CORTEX_USER" > /dev/null 2>&1 ; then
+	 useradd -m -r -g cortex -d /var/lib/cortex -s /sbin/nologin -c "cortex user" cortex
+     fi
+
+     chmod 640 /etc/cortex/single-process-config.yaml
+     chown root:$CORTEX_GROUP /etc/cortex/single-process-config.yaml
+
+     if [ -z ${2+x} ] &&  [ "$RESTART_ON_UPGRADE" == "true" ]; then
+	 if command -v systemctl 2>/dev/null; then
+	     systemctl daemon-reload
+	 fi
+     fi
+esac

packaging/deb/control/prerm

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+[ -f /etc/default/cortex ] && . /etc/default/cortex
+
+# Final uninstallation $1=0
+# If other copies of this RPM are installed, then $1>0
+if [ $1 -eq 0 ] ; then
+    if command -v systemctl 2>/dev/null; then
+	systemctl stop cortex.service > /dev/null 2>&1 || :
+    fi
+fi

packaging/deb/debian-systemd/Dockerfile

@@ -0,0 +1,18 @@
+FROM debian:10
+ENV container docker
+ENV LC_ALL C
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update \
+        && apt-get install -y systemd \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN rm -f /lib/systemd/system/multi-user.target.wants/* \
+        /etc/systemd/system/*.wants/* \
+        /lib/systemd/system/local-fs.target.wants/* \
+        /lib/systemd/system/sockets.target.wants/*udev* \
+        /lib/systemd/system/sockets.target.wants/*initctl* \
+        /lib/systemd/system/sysinit.target.wants/systemd-tmpfiles-setup* \
+        /lib/systemd/system/systemd-update-utmp*
+
+VOLUME [ "/sys/fs/cgroup" ]
+CMD ["/lib/systemd/systemd"]

packaging/deb/default/cortex

@@ -0,0 +1,11 @@
+# Log level. Valid levels: [debug, info, warn, error]. Default: "info"
+LOG_LEVEL="info"
+
+# Path to Cortex YAML configuration file.
+CONFIG_FILE="/etc/cortex/single-process-config.yaml"
+
+# Custom user defined arguments.
+CUSTOM_ARGS=""
+
+# Restart on system upgrade. Default: "true".
+RESTART_ON_UPGRADE="true"

packaging/deb/systemd/cortex.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Horizontally scalable, highly available, multi-tenant, long term Prometheus.
+Documentation=https://cortexmetrics.io/docs
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Restart=always
+User=cortex
+EnvironmentFile=/etc/default/cortex
+ExecStart=/usr/local/bin/cortex --config.file $CONFIG_FILE --log.level $LOG_LEVEL $CUSTOM_ARGS
+ExecReload=/bin/kill -HUP $MAINPID
+TimeoutStopSec=20s
+SendSIGKILL=no
+WorkingDirectory=/var/lib/cortex
+
+[Install]
+WantedBy=multi-user.target

packaging/fpm/Dockerfile

@@ -0,0 +1,23 @@
+FROM alpine:3.8
+
+RUN apk add --no-cache \
+        ruby \
+        ruby-dev \
+        ruby-etc \
+        gcc \
+        git \
+        libc-dev \
+        libffi-dev \
+        make \
+        rpm \
+        tar \
+        && gem install --no-ri --no-rdoc fpm
+
+COPY package.sh /
+ENTRYPOINT ["/package.sh"]
+
+ARG revision
+LABEL org.opencontainers.image.title="fpm" \
+        # TODO: should this label point to the fpm source code?
+        org.opencontainers.image.source="https://github.com/cortexproject/cortex/tree/master/packaging/fpm" \
+        org.opencontainers.image.revision="${revision}"

packaging/fpm/package.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -eu
+
+SRC_PATH=/src/github.com/cortexproject/cortex
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat -c "%u" $SRC_PATH)
+gid=$(stat -c "%g" $SRC_PATH)
+echo "cortex:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
+echo "cortex:*:::::::" >>/etc/shadow
+echo "cortex	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su cortex -c "PATH=$PATH make -C $SRC_PATH PACKAGE_IN_CONTAINER=false $*"

packaging/rpm/centos-systemd/Dockerfile

@@ -0,0 +1,14 @@
+FROM centos:8
+ENV container docker
+RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
+        systemd-tmpfiles-setup.service ] || rm -f $i; done); \
+        rm -f /lib/systemd/system/multi-user.target.wants/*; \
+        rm -f /etc/systemd/system/*.wants/*; \
+        rm -f /lib/systemd/system/local-fs.target.wants/*; \
+        rm -f /lib/systemd/system/sockets.target.wants/*udev*; \
+        rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \
+        rm -f /lib/systemd/system/basic.target.wants/*; \
+        rm -f /lib/systemd/system/anaconda.target.wants/*;
+
+VOLUME [ "/sys/fs/cgroup"]
+CMD ["/usr/sbin/init"]

packaging/rpm/control/post

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+
+[ -f /etc/sysconfig/cortex ] && . /etc/sysconfig/cortex
+
+ # Initial installation: $1 == 1
+ # Upgrade: $1 == 2, and configured to restart on upgrade
+ if [ $1 -eq 1 ] ; then
+     [ -z "$CORTEX_USER" ] && CORTEX_USER="cortex"
+     [ -z "$CORTEX_GROUP" ] && CORTEX_GROUP="cortex"
+     if ! getent group "$CORTEX_GROUP" > /dev/null 2>&1 ; then
+	 groupadd -r "$CORTEX_GROUP"
+     fi
+     if ! getent passwd "$CORTEX_USER" > /dev/null 2>&1 ; then
+	 useradd -m -r -g cortex -d /var/lib/cortex -s /sbin/nologin -c "cortex user" cortex
+     fi
+
+     chmod 640 /etc/cortex/single-process-config.yaml
+     chown root:$CORTEX_GROUP /etc/cortex/single-process-config.yaml
+
+ elif [ $1 -ge 2 ] ; then
+     if [ "$RESTART_ON_UPGRADE" == "true" ]; then
+	 if command -v systemctl 2>/dev/null; then
+	     systemctl daemon-reload
+	 fi
+     fi
+ fi

packaging/rpm/control/preun

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+[ -f /etc/sysconfig/cortex ] && . /etc/sysconfig/cortex
+
+# Final uninstallation $1=0
+# If other copies of this RPM are installed, then $1>0
+if [ $1 -eq 0 ] ; then
+    if command -v systemctl 2>/dev/null; then
+	systemctl stop cortex.service > /dev/null 2>&1 || :
+    fi
+fi

packaging/rpm/sysconfig/cortex

@@ -0,0 +1,11 @@
+# Log level. Valid levels: [debug, info, warn, error]. Default: "info"
+LOG_LEVEL="info"
+
+# Path to Cortex YAML configuration file.
+CONFIG_FILE="/etc/cortex/single-process-config.yaml"
+
+# Custom user defined arguments.
+CUSTOM_ARGS=""
+
+# Restart on system upgrade. Default: "true".
+RESTART_ON_UPGRADE="true"

packaging/rpm/systemd/cortex.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Horizontally scalable, highly available, multi-tenant, long term Prometheus.
+Documentation=https://cortexmetrics.io/docs
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Restart=always
+User=cortex
+EnvironmentFile=/etc/sysconfig/cortex
+ExecStart=/usr/local/bin/cortex --config.file $CONFIG_FILE --log.level $LOG_LEVEL $CUSTOM_ARGS
+ExecReload=/bin/kill -HUP $MAINPID
+TimeoutStopSec=20s
+SendSIGKILL=no
+WorkingDirectory=/var/lib/cortex
+
+[Install]
+WantedBy=multi-user.target

tools/packaging/test-packages

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -euf -o pipefail
+
+readonly IMAGE_PREFIX=$1
+readonly VERSION=$2
+readonly DISABLE_CLEANUP=${DISABLE_CLEANUP:-0}
+
+declare -a CONTAINERS=()
+
+function error() {
+    echo "$@"; exit 1
+}
+
+function cleanup() {
+    if [[ "${DISABLE_CLEANUP}" -ne 1 ]]; then
+        docker rm --force "${CONTAINERS[@]}" &>/dev/null
+    fi
+}
+
+function ready() {
+    local -ri max_attempts=$1
+    local -ri sleep_interval=$2
+    local -ri port=$3
+    local -i attempt=1
+
+    sleep "${sleep_interval}"
+    until curl -s localhost:"${port}"/ready | grep -q ready; do
+        if [[ ${attempt} -eq ${max_attempts} ]]; then
+            echo "Cortex not ready in ${max_attempts} attempts"
+            return 1
+        else
+            (( attempt++ ))
+        fi
+        sleep "${sleep_interval}"
+    done
+}
+
+trap cleanup EXIT
+
+function test_with_systemd() {
+    local -r image=$1
+    local -r install_command=$2
+    local container
+
+    container=$(docker run --tmpfs /run --tmpfs /run/lock -v /sys/fs/cgroup:/sys/fs/cgroup:ro -itd -v "$(pwd)"/dist:/opt/cortex -p 9009 "${image}")
+    CONTAINERS+=("${container}")
+
+    port=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "9009/tcp") 0).HostPort}}' "${container}")
+
+    docker exec -it "${container}" /bin/bash -c "${install_command}; systemctl start cortex.service; systemctl enable cortex.service"
+
+    ready 10 1 "${port}" || error "Testing image: ${image} with command: '${install_command}' failed"
+}
+
+test_with_systemd "${IMAGE_PREFIX}"debian-systemd "dpkg -i /opt/cortex/cortex-${VERSION}.deb"
+test_with_systemd "${IMAGE_PREFIX}"centos-systemd "rpm -i /opt/cortex/cortex-${VERSION}.rpm"