Create docker-compose and remove customer git fallback

.dockerignore

@@ -5,7 +5,7 @@ compiled/
 dependencies/
 inventory/
 tools/
-!tools/ssh
+!tools/entrypoint.sh
 README.md
 Dockerfile
 _antora

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Test
+name: Pull Request
 on:
   pull_request:
     branches:
@@ -24,11 +24,16 @@ jobs:
     - name: Install Pipenv
       run: |
         pip install pipenv tox
-    - name: Flake8
+    - name: Run ${{ matrix.command }}
       run: make ${{ matrix.command }}
-
-    - name: Run tests on Python 3.8
-      run: make test_py38
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Check Docs
+      run: make check
+    - name: Compile Docs
+      run: make docs
   build:
     needs:
     - tests

CHANGELOG.md

@@ -5,6 +5,12 @@ Please document all notable changes to this project in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v0.1.5]
+
+### Changed
+* Dockerfile to support local docker-compose setup ([#99])
+* Remove the customer git base fallback and make the value required from the API ([#99])
+
 ## [v0.1.4]
 
 ### Fixed
@@ -50,10 +56,12 @@ Initial implementation
 [v0.1.2]: https://github.com/projectsyn/commodore/releases/tag/v0.1.2
 [v0.1.3]: https://github.com/projectsyn/commodore/releases/tag/v0.1.3
 [v0.1.4]: https://github.com/projectsyn/commodore/releases/tag/v0.1.4
+[v0.1.5]: https://github.com/projectsyn/commodore/releases/tag/v0.1.5
 [#53]: https://github.com/projectsyn/commodore/pull/53
 [#58]: https://github.com/projectsyn/commodore/pull/58
 [#81]: https://github.com/projectsyn/commodore/pull/81
 [#88]: https://github.com/projectsyn/commodore/pull/88
 [#91]: https://github.com/projectsyn/commodore/pull/91
 [#94]: https://github.com/projectsyn/commodore/pull/94
 [#95]: https://github.com/projectsyn/commodore/pull/95
+[#99]: https://github.com/projectsyn/commodore/pull/99

Dockerfile

@@ -1,15 +1,20 @@
 FROM docker.io/python:3.8.3-slim-buster AS base
 
-FROM base AS builder
-
 WORKDIR /app
 
-RUN apt-get update && apt-get install -y make build-essential && apt-get clean
+ENV HOME=/app \
+    PIPENV_VENV_IN_PROJECT=1
+
 RUN pip install pipenv
 
-ENV HOME=/app \
-    PIPENV_VENV_IN_PROJECT=1 \
-    VIRTUALENV_SEEDER=pip
+FROM base AS builder
+
+RUN apt-get update && apt-get install -y \
+      build-essential \
+      make \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV VIRTUALENV_SEEDER=pip
 
 COPY Pipfile Pipfile.lock ./
 
@@ -20,21 +25,20 @@ RUN pipenv install
 
 FROM docker.io/golang:1.14-stretch AS helm_binding_builder
 
-RUN apt-get update && apt-get install -y python3-cffi && apt-get clean
+RUN apt-get update && apt-get install -y \
+      python3-cffi \
+ && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /virtualenv
 COPY --from=builder /app/.venv/lib/python3.8/site-packages/kapitan ./kapitan
 RUN ./kapitan/inputs/helm/build.sh
 
 FROM base
 
-WORKDIR /app
-RUN apt-get update && apt-get install -y git libnss-wrapper && apt-get clean
-
-RUN pip install pipenv
-
-ENV HOME=/app \
-    PIPENV_VENV_IN_PROJECT=1
+RUN apt-get update && apt-get install -y \
+      git \
+      libnss-wrapper \
+ && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /app/.venv/ ./.venv/
 COPY --from=helm_binding_builder \
@@ -44,15 +48,13 @@ COPY --from=helm_binding_builder \
 
 COPY . ./
 
-RUN ssh-keyscan -t rsa git.vshn.net > /app/.known_hosts
-
-ENV GIT_SSH=/app/tools/ssh
-
-ARG BINARY_VERSION
+ARG BINARY_VERSION=unreleased
 
 RUN sed -ie "s/^__version__ = 'Unreleased'$/__version__ = '$BINARY_VERSION'/" ./commodore/__init__.py
 
-RUN chown 1001 /app
+RUN chgrp 0 /app/ \
+ && chmod g+rwX /app/
+
 USER 1001
 
-ENTRYPOINT ["pipenv", "run", "commodore"]
+ENTRYPOINT [ "/app/tools/entrypoint.sh", "pipenv", "run", "commodore" ]

README.md

@@ -60,17 +60,17 @@ using `helm template`.
       pipenv run sh -c '${VIRTUAL_ENV}/lib/python3.*/site-packages/kapitan/inputs/helm/build.sh'
       ```
 
-1. Setup a `.env` file to configure Commodore (or provide command line flags):
+1. Setup a `.env` file to configure Commodore (don't use quotes):
 
    ```shell
    # URL of Lieutenant API
-   COMMODORE_API_URL="https://lieutenant-api.example.com/"
+   COMMODORE_API_URL=https://lieutenant-api.example.com/
    # Lieutenant API token
-   COMMODORE_API_TOKEN="..."
+   COMMODORE_API_TOKEN=<my-token>
    # Base URL for global Git repositories
-   COMMODORE_GLOBAL_GIT_BASE="ssh://git@github.com/projectsyn/"
-   # Base URL for customer Git repositories
-   COMMODORE_CUSTOMER_GIT_BASE="ssh://git@git.example.com/syn/customers/"
+   COMMODORE_GLOBAL_GIT_BASE=ssh://git@github.com/projectsyn/
+   # Your local user ID to be used in the container (optional, defaults to root)
+   USER_ID=<your-user-id>
    ```
 
    For Commodore to work, you need to run an instance of the
@@ -129,45 +129,25 @@ using `helm template`.
 
 ## Run Commodore in Docker
 
+A docker-compose setup enables running Commodore in a container.
+The environment variables are picked up from the local `.env` file.
+By default your `~/.ssh/` directory is mounted into the container and an `ssh-agent` is started.
+You can skip starting an agent by setting the `SSH_AUTH_SOCK` env variable and mounting the socket into the container.
+
 1. Build the Docker image inside of the cloned Commodore repository:
 
 ```console
-docker build -t commodore .
+docker-compose build
 ```
 
 1. Run the built image:
 
 ```console
-docker run -it --rm \
-    -e COMMODORE_API_URL="https://lieutenant-api.example.com/" \
-    -e COMMODORE_API_TOKEN="..." \
-    -e COMMODORE_GLOBAL_GIT_BASE="ssh://git@github.com/projectsyn/" \
-    -e COMMODORE_CUSTOMER_GIT_BASE="ssh://git@git.example.com/syn/customers/" \
-    -e SSH_PRIVATE_KEY="$(cat ~/.ssh/id_ed25519)" \
-    -v $(pwd)/catalog:/app/catalog/ \
-    -v $(pwd)/dependencies:/app/dependencies/ \
-    -v $(pwd)/inventory:/app/inventory/ \
-    --entrypoint bash \
-    commodore
-```
-
-1. Set up ssh-agent in the running Docker container for the access to Git repositories:
-
-```console
-tools/ssh
-eval $(ssh-agent)
-ssh-add .identityfile
-```
-
-1. Run Commodore inside of the running Docker container:
-
-```console
-pipenv run commodore
+docker-compose run commodore compile $CLUSTER_ID
 ```
 
 ## Documentation
 
 Run the `make docs` command in the `docs` subfolder to generate the Antora documentation website locally. The website will be available at the `_antora/index.html` file.
 
 After writing the documentation, please use the `make check` command and correct any warnings raised by the tool.
-

commodore/cli.py

@@ -18,14 +18,12 @@
 @click.option('--api-token', metavar='TOKEN', help='Lieutenant API token')
 @click.option('--global-git-base', metavar='URL',
               help='Base directory for global Git config repositories')
-@click.option('--customer-git-base', metavar='URL',
-              help='Base directory for customer Git config repositories')
 @verbosity
 @click.version_option(__version__, prog_name='commodore')
 @click.pass_context
 # pylint: disable=too-many-arguments
-def commodore(ctx, api_url, api_token, global_git_base, customer_git_base, verbose):
-    ctx.obj = Config(api_url, api_token, global_git_base, customer_git_base, verbose)
+def commodore(ctx, api_url, api_token, global_git_base, verbose):
+    ctx.obj = Config(api_url, api_token, global_git_base, verbose)
 
 
 @commodore.command(short_help='Delete generated files')

commodore/compile.py

@@ -45,9 +45,8 @@ def _fetch_customer_config(cfg, customer_id):
         raise click.ClickException("Customer id mismatch")
     repopath = customer.get('gitRepo', {}).get('url', None)
     if repopath is None:
-        repopath = f"{cfg.customer_git_base}/{customer_id}.git"
-        click.echo(" > API did not return a repository URL for customer " +
-                   f"'{customer_id}', using '{repopath}'")
+        raise click.ClickException(
+            f" > API did not return a repository URL for customer '{customer_id}'")
     click.echo(f"Cloning {repopath}")
     repo = git.clone_repository(repopath, P('inventory/classes') / customer_id)
     cfg.register_config('customer', repo)

commodore/config.py

@@ -6,7 +6,7 @@
 
 class Config:
     # pylint: disable=too-many-arguments
-    def __init__(self, api_url, api_token, global_git, customer_git, verbose):
+    def __init__(self, api_url, api_token, global_git, verbose):
         self.api_url = api_url
         self.api_token = None
         if api_token is not None:
@@ -20,7 +20,6 @@ def __init__(self, api_url, api_token, global_git, customer_git, verbose):
                 pass
             self.api_token = api_token.strip()
         self.global_git_base = global_git
-        self.customer_git_base = customer_git
         self._components = {}
         self._config_repos = {}
         self._verbose = verbose

docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3.0"
+services:
+  commodore:
+    build: .
+    image: docker.io/projectsyn/commodore:latest
+    env_file: .env
+    user: "${USER_ID:-0}"
+    volumes:
+    - ./catalog/:/app/catalog/
+    - ./dependencies/:/app/dependencies/
+    - ./inventory/:/app/inventory/
+    - ~/.ssh/:/app/.ssh/:ro

tests/test_compile.py

@@ -0,0 +1,22 @@
+"""
+Unit-tests for catalog compilation
+"""
+
+import click
+from unittest.mock import patch
+import pytest
+
+from commodore import compile
+from commodore.config import Config
+
+
+@patch('commodore.compile.lieutenant_query')
+def test_no_customer_repo(test_patch):
+    customer_id = "t-wild-fire-234"
+    config = Config("https://syn.example.com", "token", "ssh://git@git.example.com", False)
+    test_patch.return_value = {
+        'id': customer_id
+    }
+    with pytest.raises(click.ClickException) as excinfo:
+        compile._fetch_customer_config(config, customer_id)
+    assert customer_id in str(excinfo.value)

tools/entrypoint.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Make sure that if we are using an arbitrary UID that it appears in /etc/passwd,
+# otherwise this will cause issues with things like cloning with git+ssh
+# reference: https://access.redhat.com/documentation/en-us/openshift_container_platform/3.11/html/creating_images/creating-images-guidelines#use-uid
+
+export LD_PRELOAD=/usr/lib/libnss_wrapper.so
+export NSS_WRAPPER_PASSWD=/tmp/passwd
+export NSS_WRAPPER_GROUP=/etc/group
+
+if ! whoami &> /dev/null; then
+  echo "commodore:x:$(id -u):0:commodore user:${HOME}:/sbin/nologin" > "${NSS_WRAPPER_PASSWD}"
+fi
+
+if [ -z "${SSH_AUTH_SOCK}" ]; then
+  eval "$(ssh-agent)"
+  ssh-add $(grep -rlE 'BEGIN .+ PRIVATE KEY' /app/.ssh) || echo "No SSH keys were added"
+fi
+
+exec "$@"