From c710bd0d8bd9d02d6a810daaaa1826c5f19a204e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20R=C3=BCegg?= Date: Mon, 18 May 2020 14:22:24 +0200 Subject: [PATCH 1/3] Create docker-compose setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify the user and SSH setup to make local usage easier. Signed-off-by: Simon Rüegg --- .dockerignore | 2 +- CHANGELOG.md | 5 +++++ Dockerfile | 44 ++++++++++++++++++++++--------------------- README.md | 46 +++++++++++++-------------------------------- docker-compose.yaml | 12 ++++++++++++ tools/entrypoint.sh | 22 ++++++++++++++++++++++ tools/ssh | 31 ------------------------------ 7 files changed, 76 insertions(+), 86 deletions(-) create mode 100644 docker-compose.yaml create mode 100755 tools/entrypoint.sh delete mode 100755 tools/ssh diff --git a/.dockerignore b/.dockerignore index 4fa11fad7..95a73659f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,7 +5,7 @@ compiled/ dependencies/ inventory/ tools/ -!tools/ssh +!tools/entrypoint.sh README.md Dockerfile _antora diff --git a/CHANGELOG.md b/CHANGELOG.md index fc76e1d4b..ddca389a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ Please document all notable changes to this project in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## Unreleased + +### Changed +* Dockerfile to support local docker-compose setup + ## [v0.1.4] ### Fixed diff --git a/Dockerfile b/Dockerfile index 69c0c888a..7f4b0df5e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,20 @@ FROM docker.io/python:3.8.3-slim-buster AS base -FROM base AS builder - WORKDIR /app -RUN apt-get update && apt-get install -y make build-essential && apt-get clean +ENV HOME=/app \ + PIPENV_VENV_IN_PROJECT=1 + RUN pip install pipenv -ENV HOME=/app \ - PIPENV_VENV_IN_PROJECT=1 \ - VIRTUALENV_SEEDER=pip +FROM base AS builder + +RUN apt-get update && apt-get install -y \ + build-essential \ + make \ + && rm -rf /var/lib/apt/lists/* + +ENV VIRTUALENV_SEEDER=pip COPY Pipfile Pipfile.lock ./ @@ -20,7 +25,9 @@ RUN pipenv install FROM docker.io/golang:1.14-stretch AS helm_binding_builder -RUN apt-get update && apt-get install -y python3-cffi && apt-get clean +RUN apt-get update && apt-get install -y \ + python3-cffi \ + && rm -rf /var/lib/apt/lists/* WORKDIR /virtualenv COPY --from=builder /app/.venv/lib/python3.8/site-packages/kapitan ./kapitan @@ -28,13 +35,10 @@ RUN ./kapitan/inputs/helm/build.sh FROM base -WORKDIR /app -RUN apt-get update && apt-get install -y git libnss-wrapper && apt-get clean - -RUN pip install pipenv - -ENV HOME=/app \ - PIPENV_VENV_IN_PROJECT=1 +RUN apt-get update && apt-get install -y \ + git \ + libnss-wrapper \ + && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/.venv/ ./.venv/ COPY --from=helm_binding_builder \ @@ -44,15 +48,13 @@ COPY --from=helm_binding_builder \ COPY . ./ -RUN ssh-keyscan -t rsa git.vshn.net > /app/.known_hosts - -ENV GIT_SSH=/app/tools/ssh - -ARG BINARY_VERSION +ARG BINARY_VERSION=unreleased RUN sed -ie "s/^__version__ = 'Unreleased'$/__version__ = '$BINARY_VERSION'/" ./commodore/__init__.py -RUN chown 1001 /app +RUN chgrp 0 /app/ \ + && chmod g+rwX /app/ + USER 1001 -ENTRYPOINT ["pipenv", "run", "commodore"] +ENTRYPOINT [ "/app/tools/entrypoint.sh", "pipenv", "run", "commodore" ] diff --git a/README.md b/README.md index 244932b67..982f52c96 100644 --- a/README.md +++ b/README.md @@ -60,17 +60,17 @@ using `helm template`. pipenv run sh -c '${VIRTUAL_ENV}/lib/python3.*/site-packages/kapitan/inputs/helm/build.sh' ``` -1. Setup a `.env` file to configure Commodore (or provide command line flags): +1. Setup a `.env` file to configure Commodore (don't use quotes): ```shell # URL of Lieutenant API - COMMODORE_API_URL="https://lieutenant-api.example.com/" + COMMODORE_API_URL=https://lieutenant-api.example.com/ # Lieutenant API token - COMMODORE_API_TOKEN="..." + COMMODORE_API_TOKEN= # Base URL for global Git repositories - COMMODORE_GLOBAL_GIT_BASE="ssh://git@github.com/projectsyn/" - # Base URL for customer Git repositories - COMMODORE_CUSTOMER_GIT_BASE="ssh://git@git.example.com/syn/customers/" + COMMODORE_GLOBAL_GIT_BASE=ssh://git@github.com/projectsyn/ + # Your local user ID to be used in the container (optional, defaults to root) + USER_ID= ``` For Commodore to work, you need to run an instance of the @@ -129,40 +129,21 @@ using `helm template`. ## Run Commodore in Docker +A docker-compose setup enables running Commodore in a container. +The environment variables are picked up from the local `.env` file. +By default your `~/.ssh/` directory is mounted into the container and an `ssh-agent` is started. +You can skip starting an agent by setting the `SSH_AUTH_SOCK` env variable and mounting the socket into the container. + 1. Build the Docker image inside of the cloned Commodore repository: ```console -docker build -t commodore . +docker-compose build ``` 1. Run the built image: ```console -docker run -it --rm \ - -e COMMODORE_API_URL="https://lieutenant-api.example.com/" \ - -e COMMODORE_API_TOKEN="..." \ - -e COMMODORE_GLOBAL_GIT_BASE="ssh://git@github.com/projectsyn/" \ - -e COMMODORE_CUSTOMER_GIT_BASE="ssh://git@git.example.com/syn/customers/" \ - -e SSH_PRIVATE_KEY="$(cat ~/.ssh/id_ed25519)" \ - -v $(pwd)/catalog:/app/catalog/ \ - -v $(pwd)/dependencies:/app/dependencies/ \ - -v $(pwd)/inventory:/app/inventory/ \ - --entrypoint bash \ - commodore -``` - -1. Set up ssh-agent in the running Docker container for the access to Git repositories: - -```console -tools/ssh -eval $(ssh-agent) -ssh-add .identityfile -``` - -1. Run Commodore inside of the running Docker container: - -```console -pipenv run commodore +docker-compose run commodore compile $CLUSTER_ID ``` ## Documentation @@ -170,4 +151,3 @@ pipenv run commodore Run the `make docs` command in the `docs` subfolder to generate the Antora documentation website locally. The website will be available at the `_antora/index.html` file. After writing the documentation, please use the `make check` command and correct any warnings raised by the tool. - diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 000000000..3ddf98cc8 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,12 @@ +version: "3.0" +services: + commodore: + build: . + image: docker.io/projectsyn/commodore:latest + env_file: .env + user: "${USER_ID:-0}" + volumes: + - ./catalog/:/app/catalog/ + - ./dependencies/:/app/dependencies/ + - ./inventory/:/app/inventory/ + - ~/.ssh/:/app/.ssh/:ro diff --git a/tools/entrypoint.sh b/tools/entrypoint.sh new file mode 100755 index 000000000..b18c5aadb --- /dev/null +++ b/tools/entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +# Make sure that if we are using an arbitrary UID that it appears in /etc/passwd, +# otherwise this will cause issues with things like cloning with git+ssh +# reference: https://access.redhat.com/documentation/en-us/openshift_container_platform/3.11/html/creating_images/creating-images-guidelines#use-uid + +export LD_PRELOAD=/usr/lib/libnss_wrapper.so +export NSS_WRAPPER_PASSWD=/tmp/passwd +export NSS_WRAPPER_GROUP=/etc/group + +if ! whoami &> /dev/null; then + echo "commodore:x:$(id -u):0:commodore user:${HOME}:/sbin/nologin" > "${NSS_WRAPPER_PASSWD}" +fi + +if [ -z "${SSH_AUTH_SOCK}" ]; then + eval "$(ssh-agent)" + ssh-add $(grep -rlE 'BEGIN .+ PRIVATE KEY' /app/.ssh) || echo "No SSH keys were added" +fi + +exec "$@" diff --git a/tools/ssh b/tools/ssh deleted file mode 100755 index 4bbabd8a0..000000000 --- a/tools/ssh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# run a ssh as (non-existent) user, using libnss-wrapper - -U=$(id -u) -G=$(id -g) - -HOME_DIR=/tmp/user -PASSWD=/var/tmp/passwd -GROUP=/var/tmp/group - -if [ ! -d "$HOME_DIR" ]; then - mkdir "$HOME_DIR" -fi -if [ ! -f "$PASSWD" ]; then - echo "user::$U:$G::$HOME_DIR:" > "$PASSWD" -fi -if [ ! -f "$GROUP" ]; then - echo "user::$G:" > "$GROUP" -fi - -echo "${SSH_PRIVATE_KEY}" > /app/.identityfile -chmod 600 /app/.identityfile - -LD_PRELOAD=libnss_wrapper.so \ -NSS_WRAPPER_PASSWD="$PASSWD" \ -NSS_WRAPPER_GROUP="$GROUP" \ -ssh -i /app/.identityfile \ - -oGlobalKnownHostsfile=/app/.known_hosts \ - "$@" - -# vim: set et sw=4 ts=4 : From 75e47ceba012d8b1ac1c648f15c307ebc8130781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20R=C3=BCegg?= Date: Mon, 18 May 2020 14:30:04 +0200 Subject: [PATCH 2/3] Remove customer git base fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The customer repo URL is required from the API, remove the fallback to the env var. Signed-off-by: Simon Rüegg --- CHANGELOG.md | 7 +++++-- commodore/cli.py | 6 ++---- commodore/compile.py | 5 ++--- commodore/config.py | 3 +-- tests/test_compile.py | 22 ++++++++++++++++++++++ 5 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 tests/test_compile.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ddca389a3..6b5ba8fe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,11 @@ Please document all notable changes to this project in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## Unreleased +## [v0.1.5] ### Changed -* Dockerfile to support local docker-compose setup +* Dockerfile to support local docker-compose setup ([#99]) +* Remove the customer git base fallback and make the value required from the API ([#99]) ## [v0.1.4] @@ -55,6 +56,7 @@ Initial implementation [v0.1.2]: https://github.com/projectsyn/commodore/releases/tag/v0.1.2 [v0.1.3]: https://github.com/projectsyn/commodore/releases/tag/v0.1.3 [v0.1.4]: https://github.com/projectsyn/commodore/releases/tag/v0.1.4 +[v0.1.5]: https://github.com/projectsyn/commodore/releases/tag/v0.1.5 [#53]: https://github.com/projectsyn/commodore/pull/53 [#58]: https://github.com/projectsyn/commodore/pull/58 [#81]: https://github.com/projectsyn/commodore/pull/81 @@ -62,3 +64,4 @@ Initial implementation [#91]: https://github.com/projectsyn/commodore/pull/91 [#94]: https://github.com/projectsyn/commodore/pull/94 [#95]: https://github.com/projectsyn/commodore/pull/95 +[#99]: https://github.com/projectsyn/commodore/pull/99 diff --git a/commodore/cli.py b/commodore/cli.py index 54237765c..3cd982cb7 100644 --- a/commodore/cli.py +++ b/commodore/cli.py @@ -18,14 +18,12 @@ @click.option('--api-token', metavar='TOKEN', help='Lieutenant API token') @click.option('--global-git-base', metavar='URL', help='Base directory for global Git config repositories') -@click.option('--customer-git-base', metavar='URL', - help='Base directory for customer Git config repositories') @verbosity @click.version_option(__version__, prog_name='commodore') @click.pass_context # pylint: disable=too-many-arguments -def commodore(ctx, api_url, api_token, global_git_base, customer_git_base, verbose): - ctx.obj = Config(api_url, api_token, global_git_base, customer_git_base, verbose) +def commodore(ctx, api_url, api_token, global_git_base, verbose): + ctx.obj = Config(api_url, api_token, global_git_base, verbose) @commodore.command(short_help='Delete generated files') diff --git a/commodore/compile.py b/commodore/compile.py index 0f6e1f761..17595cf19 100644 --- a/commodore/compile.py +++ b/commodore/compile.py @@ -45,9 +45,8 @@ def _fetch_customer_config(cfg, customer_id): raise click.ClickException("Customer id mismatch") repopath = customer.get('gitRepo', {}).get('url', None) if repopath is None: - repopath = f"{cfg.customer_git_base}/{customer_id}.git" - click.echo(" > API did not return a repository URL for customer " + - f"'{customer_id}', using '{repopath}'") + raise click.ClickException( + f" > API did not return a repository URL for customer '{customer_id}'") click.echo(f"Cloning {repopath}") repo = git.clone_repository(repopath, P('inventory/classes') / customer_id) cfg.register_config('customer', repo) diff --git a/commodore/config.py b/commodore/config.py index 6080d3c48..c53bdfd98 100644 --- a/commodore/config.py +++ b/commodore/config.py @@ -6,7 +6,7 @@ class Config: # pylint: disable=too-many-arguments - def __init__(self, api_url, api_token, global_git, customer_git, verbose): + def __init__(self, api_url, api_token, global_git, verbose): self.api_url = api_url self.api_token = None if api_token is not None: @@ -20,7 +20,6 @@ def __init__(self, api_url, api_token, global_git, customer_git, verbose): pass self.api_token = api_token.strip() self.global_git_base = global_git - self.customer_git_base = customer_git self._components = {} self._config_repos = {} self._verbose = verbose diff --git a/tests/test_compile.py b/tests/test_compile.py new file mode 100644 index 000000000..65113689a --- /dev/null +++ b/tests/test_compile.py @@ -0,0 +1,22 @@ +""" +Unit-tests for catalog compilation +""" + +import click +from unittest.mock import patch +import pytest + +from commodore import compile +from commodore.config import Config + + +@patch('commodore.compile.lieutenant_query') +def test_no_customer_repo(test_patch): + customer_id = "t-wild-fire-234" + config = Config("https://syn.example.com", "token", "ssh://git@git.example.com", False) + test_patch.return_value = { + 'id': customer_id + } + with pytest.raises(click.ClickException) as excinfo: + compile._fetch_customer_config(config, customer_id) + assert customer_id in str(excinfo.value) From 2b86d43a8820e9859f3a381f56c85d89ef2b6e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20R=C3=BCegg?= Date: Tue, 19 May 2020 08:19:05 +0200 Subject: [PATCH 3/3] Update GH actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validate and compile docs. Signed-off-by: Simon Rüegg --- .github/workflows/test.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6213d90b5..56471ed80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Test +name: Pull Request on: pull_request: branches: @@ -24,11 +24,16 @@ jobs: - name: Install Pipenv run: | pip install pipenv tox - - name: Flake8 + - name: Run ${{ matrix.command }} run: make ${{ matrix.command }} - - - name: Run tests on Python 3.8 - run: make test_py38 + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Check Docs + run: make check + - name: Compile Docs + run: make docs build: needs: - tests