From 7d9f910ac4869e439d32675b082b389e41cf27a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Sun, 13 Oct 2019 16:00:42 +0200 Subject: [PATCH 1/5] externalize netbox download with wget --- .dockerignore | 1 + .gitignore | 1 + Dockerfile | 77 +++++++++++++++++++++++++--------------------- README.md | 11 +++---- build.sh | 68 +++++++++++++++++++++++++--------------- docker-compose.yml | 5 --- 6 files changed, 92 insertions(+), 71 deletions(-) diff --git a/.dockerignore b/.dockerignore index ce32f24e6..1b2baccd1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ .travis.yml build* *.env +.git diff --git a/.gitignore b/.gitignore index 53a4e8159..cbaffa853 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.sql.gz +.netbox diff --git a/Dockerfile b/Dockerfile index 481613e98..dab072316 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG FROM=python:3.7-alpine -FROM ${FROM} as main +FROM ${FROM} as builder RUN apk add --no-cache \ bash \ @@ -8,51 +8,52 @@ RUN apk add --no-cache \ cyrus-sasl-dev \ graphviz \ jpeg-dev \ + libevent-dev \ libffi-dev \ - libxml2-dev \ libxslt-dev \ openldap-dev \ - postgresql-dev \ - ttf-ubuntu-font-family \ - wget + postgresql-dev + +WORKDIR /install -RUN pip install \ +RUN pip install --install-option="--prefix=/install" \ # gunicorn is used for launching netbox gunicorn \ + greenlet \ + eventlet \ # napalm is used for gathering information from network devices napalm \ # ruamel is used in startup_scripts 'ruamel.yaml>=0.15,<0.16' \ -# pinning django to the version required by netbox -# adding it here, to install the correct version of -# django-rq - 'Django>=2.2,<2.3' \ -# django-rq is used for webhooks - django-rq - -ARG BRANCH=master - -WORKDIR /tmp - -# As the requirements don't change very often, -# and as they take some time to compile, -# we try to cache them very agressively. -ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt -ADD ${REQUIREMENTS_URL} requirements.txt -RUN pip install -r requirements.txt - -# Cache bust when the upstream branch changes: -# ADD will fetch the file and check if it has changed -# If not, Docker will use the existing build cache. -# If yes, Docker will bust the cache and run every build step from here on. -ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH -ADD ${REF_URL} version.json +# django_auth_ldap is required for ldap + django_auth_ldap + +COPY .netbox/netbox/requirements.txt / +RUN pip install --install-option="--prefix=/install" -r /requirements.txt + +### +# Main stage +### + +ARG FROM +FROM ${FROM} as main + +RUN apk add --no-cache \ + bash \ + ca-certificates \ + graphviz \ + libevent \ + libffi \ + libjpeg-turbo \ + libressl \ + libxslt \ + postgresql-libs \ + ttf-ubuntu-font-family WORKDIR /opt -ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz -RUN wget -q -O - "${URL}" | tar xz \ - && mv netbox* netbox +COPY --from=builder /install /usr/local +COPY .netbox/netbox /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py COPY configuration/gunicorn_config.py /etc/netbox/config/ @@ -73,13 +74,19 @@ LABEL SRC_URL="$URL" ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION" +ARG NETBOX_BRANCH=custom_build +LABEL NETBOX_BRANCH="$NETBOX_BRANCH" + ##### -## LDAP specific tasks +## LDAP specific configuration ##### FROM main as ldap -RUN pip install django_auth_ldap +RUN apk add --no-cache \ + libsasl \ + libldap \ + util-linux COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py diff --git a/README.md b/README.md index 6c4953569..aa232757a 100644 --- a/README.md +++ b/README.md @@ -68,18 +68,17 @@ To use this feature, set the environment-variable `VERSION` before launching `do [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub]. ```bash -export VERSION=v2.2.6 +export VERSION=v2.6.6 docker-compose pull netbox docker-compose up -d ``` -You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid -[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github]. -Most commonly you will specify a tag or branch name. +You can also build a specific version of the Netbox Docker image yourself. +`VERSION` can be any valid [git ref][git-ref] in that case. ```bash -export VERSION=develop -docker-compose build --no-cache netbox +export VERSION=v2.6.6 +./build.sh $VERSION docker-compose up -d ``` diff --git a/build.sh b/build.sh index 6ebea6fd4..72428aef5 100755 --- a/build.sh +++ b/build.sh @@ -68,6 +68,16 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then fi fi +### +# Determining the build command to use +### +if [ -z "$DRY_RUN" ]; then + DRY="" +else + echo "⚠️ DRY_RUN MODE ON ⚠️" + DRY="echo" +fi + ### # read the project version from the `VERSION` file and trim it # see https://stackoverflow.com/a/3232433/172132 @@ -82,6 +92,26 @@ SRC_REPO="${SRC_REPO-netbox}" BRANCH="${1}" URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}" +### +# fetching the source +### +if [ "${2}" != "--push-only" ] ; then + echo "🗑️ Preparing" + $DRY rm -rf .netbox + $DRY mkdir .netbox + echo "✅ Done preparing" + + echo "🌐 Downloading netbox from the url '${URL}'" + ( + $DRY cd .netbox + + $DRY wget -qO netbox.tgz "${URL}" && \ + $DRY tar -xzf netbox.tgz && \ + $DRY mv netbox-* netbox + ) + echo "✅ Downloaded netbox" +fi + ### # Determining the value for DOCKERFILE # and checking whether it exists @@ -156,21 +186,12 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do --target "$DOCKER_TARGET" ) - # caching is only ok for version tags, - # but turning the cache off is only required for the - # first build target, usually "main". - case "${TAG}" in - v*) ;; - *) [ "$DOCKER_TARGET" == "${DOCKER_TARGETS[0]}" ] && DOCKER_OPTS+=( --no-cache ) ;; - esac - ### # Composing arguments for `docker build` CLI ### DOCKER_BUILD_ARGS=( --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --build-arg "BRANCH=${BRANCH}" - --build-arg "URL=${URL}" + --build-arg "NETBOX_BRANCH=${BRANCH}" --build-arg "DOCKER_ORG=${DOCKER_ORG}" --build-arg "DOCKER_REPO=${DOCKER_REPO}" ) @@ -190,27 +211,17 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) fi - ### - # Determining the build command to use - ### - if [ -z "$DRY_RUN" ]; then - DOCKER_CMD="docker" - else - echo "⚠️ DRY_RUN MODE ON ⚠️" - DOCKER_CMD="echo docker" - fi - ### # Building the docker images, except if `--push-only` is passed ### if [ "${2}" != "--push-only" ] ; then - echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}' from the url '${URL}'." - $DOCKER_CMD build -t "${TARGET_DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . + echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." + $DRY docker build -t "${TARGET_DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" if [ -n "$DOCKER_SHORT_TAG" ]; then echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." - $DOCKER_CMD tag "${TARGET_DOCKER_TAG}" "${DOCKER_SHORT_TAG}" + $DRY docker tag "${TARGET_DOCKER_TAG}" "${DOCKER_SHORT_TAG}" echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" fi fi @@ -220,13 +231,20 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do ### if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then echo "⏫ Pushing '${TARGET_DOCKER_TAG}" - $DOCKER_CMD push "${TARGET_DOCKER_TAG}" + $DRY docker push "${TARGET_DOCKER_TAG}" echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'." if [ -n "$DOCKER_SHORT_TAG" ]; then echo "⏫ Pushing '${DOCKER_SHORT_TAG}'" - $DOCKER_CMD push "${DOCKER_SHORT_TAG}" + $DRY docker push "${DOCKER_SHORT_TAG}" echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'." fi fi done + +### +# Cleaning up +### +echo "🗑️ Cleaning up" +$DRY rm -rf .netbox +echo "✅ Cleaned up" diff --git a/docker-compose.yml b/docker-compose.yml index f57de211d..f70a93f33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,6 @@ version: '3.4' services: netbox: &netbox - build: - context: . - target: ${DOCKER_TARGET-main} - args: - - BRANCH=${VERSION-master} image: netboxcommunity/netbox:${VERSION-latest} depends_on: - postgres From 795ab1b45220b21c1555641d9dd3101a1ccf78ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 21:54:49 +0200 Subject: [PATCH 2/5] Checkout the repository with git This changes the build process even further. Instead f using `wget` to fetch the current code, `git` is used. This allows for faster switching between branches, because only the differences between them have to be fetched from the server. But the main advantage is that the build cache can finally be used as designed by Docker. Repetitive builds are very fast now. This is also true between branches and tags, as long as the `requirements.txt` file doesn't change. --- Dockerfile | 19 ++-- README.md | 3 - build-all.sh | 3 - build-latest.sh | 42 +-------- build.sh | 244 ++++++++++++++++++++++++++---------------------- 5 files changed, 147 insertions(+), 164 deletions(-) diff --git a/Dockerfile b/Dockerfile index dab072316..c605e9973 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,8 @@ RUN pip install --install-option="--prefix=/install" \ # django_auth_ldap is required for ldap django_auth_ldap -COPY .netbox/netbox/requirements.txt / +ARG NETBOX_PATH +COPY ${NETBOX_PATH}/requirements.txt / RUN pip install --install-option="--prefix=/install" -r /requirements.txt ### @@ -53,7 +54,9 @@ RUN apk add --no-cache \ WORKDIR /opt COPY --from=builder /install /usr/local -COPY .netbox/netbox /opt/netbox + +ARG NETBOX_PATH +COPY ${NETBOX_PATH} /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py COPY configuration/gunicorn_config.py /etc/netbox/config/ @@ -69,13 +72,11 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"] -LABEL SRC_URL="$URL" - -ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot -LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION" - -ARG NETBOX_BRANCH=custom_build -LABEL NETBOX_BRANCH="$NETBOX_BRANCH" +LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \ + NETBOX_BRANCH="custom build" \ + ORIGINAL_DOCKER_TAG="custom build" \ + NETBOX_GIT_COMMIT="not built from git" \ + NETBOX_GIT_URL="not built from git" ##### ## LDAP specific configuration diff --git a/README.md b/README.md index aa232757a..e8fed8585 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,6 @@ export VERSION=v2.6.6 docker-compose up -d ``` -Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary. -This can increase the build speed if you're just adjusting the config, for example. - [git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References [netbox-github]: https://github.com/netbox-community/netbox/releases diff --git a/build-all.sh b/build-all.sh index 0ba18ed07..462a83a9f 100755 --- a/build-all.sh +++ b/build-all.sh @@ -18,9 +18,6 @@ fi ERROR=0 -# Don't build if not on `master` and don't build if on a pull request, -# but build when DEBUG is not empty - for BUILD in "${BUILDS[@]}"; do echo "🛠 Building '$BUILD' from '$DOCKERFILE'" case $BUILD in diff --git a/build-latest.sh b/build-latest.sh index ef975694b..f16e211d4 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -72,42 +72,6 @@ if [ "${PRERELEASE}" == "true" ]; then fi fi -### -# Compose DOCKER_TAG to build -### -if [ -z "$DOCKER_TARGET" ] || [ "$DOCKER_TARGET" == "main" ]; then - DOCKER_TAG="${VERSION}" -else - DOCKER_TAG="${VERSION}-${DOCKER_TARGET}" -fi - -### -# Check if the version received is not already available on Docker Hub: -### -ORIGINAL_DOCKERHUB_REPO="${DOCKER_ORG-netboxcommunity}/${DOCKER_REPO-netbox}" -DOCKERHUB_REPO="${DOCKERHUB_REPO-$ORIGINAL_DOCKERHUB_REPO}" - -# Bearer Token -URL_DOCKERHUB_TOKEN="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${DOCKERHUB_REPO}:pull" -BEARER_TOKEN="$($CURL "${URL_DOCKERHUB_TOKEN}" | jq -r .token)" - -# Actual API call -URL_DOCKERHUB_TAG="https://registry.hub.docker.com/v2/${DOCKERHUB_REPO}/tags/list" -AUTHORIZATION_HEADER="Authorization: Bearer ${BEARER_TOKEN}" -ALREADY_BUILT="$($CURL -H "${AUTHORIZATION_HEADER}" "${URL_DOCKERHUB_TAG}" | jq -e ".tags | any(.==\"${DOCKER_TAG}\")")" - -### -# Only build the image if it's not already been built before -### -if [ -n "$DEBUG" ] || [ "$ALREADY_BUILT" == "false" ]; then - if [ -n "$DEBUG" ]; then - echo "⚠️ Would not build, because ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}, but DEBUG is enabled." - fi - - # shellcheck disable=SC2068 - ./build.sh "${VERSION}" $@ - exit $? -else - echo "✅ ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}" - exit 0 -fi +# shellcheck disable=SC2068 +./build.sh "${VERSION}" $@ +exit $? diff --git a/build.sh b/build.sh index 72428aef5..c9ee110a2 100755 --- a/build.sh +++ b/build.sh @@ -8,58 +8,78 @@ set -e if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo "Usage: ${0} [--push|--push-only]" echo " branch The branch or tag to build. Required." - echo " --push Pushes built the Docker image to the registry." - echo " --push-only Does not build. Only pushes the Docker image to the registry." + echo " --push Pushes the built Docker image to the registry." + echo " --push-only Only pushes the Docker image to the registry, but does not build it." echo "" echo "You can use the following ENV variables to customize the build:" - echo " BRANCH The branch to build." - echo " Also used for tagging the image." - echo " TAG The version part of the docker tag." - echo " Default:" - echo " When =master: latest" - echo " When =develop: snapshot" - echo " Else: same as " - echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r//)" - echo " Also used for tagging the image." - echo " Default: netboxcommunity" - echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r//)" - echo " Also used for tagging the image." - echo " Default: netbox" + echo " SRC_ORG Which fork of netbox to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})." + echo " Default: netbox-community" + echo " SRC_REPO The name of the repository to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})." + echo " Default: netbox" + echo " URL Where to fetch the code from." + echo " Must be a git repository. Can be private." + echo " Default: https://github.com/\${SRC_ORG}/\${SRC_REPO}.git" + echo " NETBOX_PATH The path where netbox will be checkout out." + echo " Must not be outside of the netbox-docker repository (because of Docker)!" + echo " Default: .netbox" + echo " SKIP_GIT If defined, git is not invoked and \${NETBOX_PATH} will not be altered." + echo " This may be useful, if you are manually managing the NETBOX_PATH." + echo " Default: undefined" + echo " TAG The version part of the docker tag." + echo " Default:" + echo " When \${BRANCH}=master: latest" + echo " When \${BRANCH}=develop: snapshot" + echo " Else: same as \${BRANCH}" + echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" + echo " Also used for tagging the image." + echo " Default: netboxcommunity" + echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})" + echo " Also used for tagging the image." + echo " Default: netbox" echo " DOCKER_FROM The base image to use." - echo " Default: Whatever is defined as default in the Dockerfile." - echo " DOCKER_TAG The name of the tag which is applied to the image." - echo " Useful for pushing into another registry than hub.docker.com." - echo " Default: /:" + echo " Default: Whatever is defined as default in the Dockerfile." + echo " DOCKER_TAG The name of the tag which is applied to the image." + echo " Useful for pushing into another registry than hub.docker.com." + echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}" echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the" - echo " image. This is used to tag all patch releases to their" - echo " containing version e.g. v2.5.1 -> v2.5" - echo " Default: /:\$MAJOR.\$MINOR" - echo " DOCKERFILE The name of Dockerfile to use." - echo " Default: Dockerfile" + echo " image. This is used to tag all patch releases to their" + echo " containing version e.g. v2.5.1 -> v2.5" + echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:." + echo " DOCKERFILE The name of Dockerfile to use." + echo " Default: Dockerfile" echo " DOCKER_TARGET A specific target to build." - echo " It's currently not possible to pass multiple targets." - echo " Default: main ldap" - echo " SRC_ORG Which fork of netbox to use (i.e. github.com//)." - echo " Default: netbox-community" - echo " SRC_REPO The name of the netbox for to use (i.e. github.com//)." - echo " Default: netbox" - echo " URL Where to fetch the package from." - echo " Must be a tar.gz file of the source code." - echo " Default: https://github.com///archive/\$BRANCH.tar.gz" - echo " HTTP_PROXY The proxy to use for http requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " HTTPS_PROXY The proxy to use for https requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " FTP_PROXY The proxy to use for ftp requests." - echo " Example: http://proxy.domain.tld:3128" - echo " Default: empty" - echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." - echo " Example: .domain1.tld,.domain2.tld" - echo " Default: empty" - echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." - echo " DRY_RUN Prints all build statements instead of running them." + echo " It's currently not possible to pass multiple targets." + echo " Default: main ldap" + echo " HTTP_PROXY The proxy to use for http requests." + echo " Example: http://proxy.domain.tld:3128" + echo " Default: undefined" + echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for." + echo " Example: .domain1.tld,.domain2.tld" + echo " Default: undefined" + echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." + echo " Default: undefined" + echo " DRY_RUN Prints all build statements instead of running them." + echo " Default: undefined" + echo "" + echo "Examples:" + echo " ${0} master" + echo " This will fetch the latest 'master' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:latest'." + echo " ${0} develop" + echo " This will fetch the latest 'develop' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:snapshot'." + echo " ${0} v2.6.6" + echo " This will fetch the 'v2.6.6' tag, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:v2.6.6' and 'netboxcommunity/netbox:v2.6'." + echo " ${0} develop-2.7" + echo " This will fetch the 'develop-2.7' branch, build a Docker Image and tag it" + echo " 'netboxcommunity/netbox:develop-2.7'." + echo " SRC_ORG=cimnine ${0} feature-x" + echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git," + echo " build a Docker Image and tag it 'netboxcommunity/netbox:feature-x'." + echo " SRC_ORG=cimnine DOCKER_ORG=cimnine ${0} feature-x" + echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git," + echo " build a Docker Image and tag it 'cimnine/netbox:feature-x'." if [ "${1}x" == "x" ]; then exit 1 @@ -71,7 +91,7 @@ fi ### # Determining the build command to use ### -if [ -z "$DRY_RUN" ]; then +if [ -z "${DRY_RUN}" ]; then DRY="" else echo "⚠️ DRY_RUN MODE ON ⚠️" @@ -90,26 +110,31 @@ NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:s SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" BRANCH="${1}" -URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}" +URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" ### # fetching the source ### if [ "${2}" != "--push-only" ] ; then - echo "🗑️ Preparing" - $DRY rm -rf .netbox - $DRY mkdir .netbox - echo "✅ Done preparing" + NETBOX_PATH="${NETBOX_PATH-.netbox}" + echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" + if [ ! -d "${NETBOX_PATH}" ]; then + $DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}" + fi - echo "🌐 Downloading netbox from the url '${URL}'" ( - $DRY cd .netbox + $DRY cd "${NETBOX_PATH}" + + if [ -n "${HTTP_PROXY}" ]; then + git config http.proxy "${HTTP_PROXY}" + fi - $DRY wget -qO netbox.tgz "${URL}" && \ - $DRY tar -xzf netbox.tgz && \ - $DRY mv netbox-* netbox + $DRY git remote set-url origin "${URL}" + $DRY git fetch -qpP --depth 10 origin "${BRANCH}" + $DRY git checkout -qf FETCH_HEAD + $DRY git prune ) - echo "✅ Downloaded netbox" + echo "✅ Checked out netbox" fi ### @@ -120,7 +145,7 @@ DOCKERFILE="${DOCKERFILE-Dockerfile}" if [ ! -f "${DOCKERFILE}" ]; then echo "🚨 The Dockerfile ${DOCKERFILE} doesn't exist." - if [ -z "$DEBUG" ]; then + if [ -z "${DEBUG}" ]; then exit 1 else echo "⚠️ Would exit here with code '1', but DEBUG is enabled." @@ -152,13 +177,13 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}" # Build each target ### for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do - echo "🏗 Building the target '$DOCKER_TARGET'" + echo "🏗 Building the target '${DOCKER_TARGET}'" ### # composing the final TARGET_DOCKER_TAG ### TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" - if [ "$DOCKER_TARGET" != "main" ]; then + if [ "${DOCKER_TARGET}" != "main" ]; then TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}" fi @@ -173,57 +198,63 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}" - if [ "$DOCKER_TARGET" != "main" ]; then + if [ "${DOCKER_TARGET}" != "main" ]; then DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}" fi fi ### - # Composing global Docker CLI arguments + # Proceeding to buils stage, except if `--push-only` is passed ### - DOCKER_OPTS=( - --pull - --target "$DOCKER_TARGET" - ) + if [ "${2}" != "--push-only" ] ; then + ### + # Composing all arguments for `docker build` + ### + DOCKER_BUILD_ARGS=( + --pull + --target "${DOCKER_TARGET}" + -f "${DOCKERFILE}" + -t "${TARGET_DOCKER_TAG}" + ) + if [ -n "${DOCKER_SHORT_TAG}" ]; then + DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" ) + fi - ### - # Composing arguments for `docker build` CLI - ### - DOCKER_BUILD_ARGS=( - --build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" - --build-arg "NETBOX_BRANCH=${BRANCH}" - --build-arg "DOCKER_ORG=${DOCKER_ORG}" - --build-arg "DOCKER_REPO=${DOCKER_REPO}" - ) - if [ -n "$DOCKER_FROM" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) - fi - if [ -n "$HTTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) - fi - if [ -n "$HTTPS_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) - fi - if [ -n "$FTP_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" ) - fi - if [ -n "$NO_PROXY" ]; then - DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) - fi + # --label + DOCKER_BUILD_ARGS+=( + --label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}" + --label "NETBOX_BRANCH=${BRANCH}" + --label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}" + ) + if [ -d "${NETBOX_PATH}/.git" ]; then + DOCKER_BUILD_ARGS+=( + --label "NETBOX_GIT_COMMIT=$($DRY cd ${NETBOX_PATH}; $DRY git rev-parse HEAD)" + --label "NETBOX_GIT_URL=$($DRY cd ${NETBOX_PATH}; $DRY git remote get-url origin)" + ) + fi - ### - # Building the docker images, except if `--push-only` is passed - ### - if [ "${2}" != "--push-only" ] ; then + # --build-arg + DOCKER_BUILD_ARGS+=( + --build-arg "NETBOX_PATH=${NETBOX_PATH}" + --build-arg "DOCKER_REPO=${DOCKER_REPO}" + ) + if [ -n "${DOCKER_FROM}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" ) + fi + if [ -n "${HTTP_PROXY}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" ) + DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" ) + fi + if [ -n "${NO_PROXY}" ]; then + DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" ) + fi + + ### + # Building the docker image + ### echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'." - $DRY docker build -t "${TARGET_DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . + $DRY docker build "${DOCKER_BUILD_ARGS[@]}" . echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'" - - if [ -n "$DOCKER_SHORT_TAG" ]; then - echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'." - $DRY docker tag "${TARGET_DOCKER_TAG}" "${DOCKER_SHORT_TAG}" - echo "✅ Tagged image '${DOCKER_SHORT_TAG}'" - fi fi ### @@ -241,10 +272,3 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do fi fi done - -### -# Cleaning up -### -echo "🗑️ Cleaning up" -$DRY rm -rf .netbox -echo "✅ Cleaned up" From cae28fe05e982822d7fd92f3f5434d42db13dca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 22:20:09 +0200 Subject: [PATCH 3/5] Fix build The Docker Hub build system runs on Ubuntu Xenial containers. Xenial's git is 2.7.4, which does not know the `-P` flag yet. --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index c9ee110a2..e14fc17b7 100755 --- a/build.sh +++ b/build.sh @@ -130,7 +130,7 @@ if [ "${2}" != "--push-only" ] ; then fi $DRY git remote set-url origin "${URL}" - $DRY git fetch -qpP --depth 10 origin "${BRANCH}" + $DRY git fetch -qp --depth 10 origin "${BRANCH}" $DRY git checkout -qf FETCH_HEAD $DRY git prune ) From a5baa55e35a8277215aa629ac1e1b961d3aa9586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 23:10:42 +0200 Subject: [PATCH 4/5] Only push on 'main' branch --- hooks/push | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hooks/push b/hooks/push index 0c0b65f09..c1e3ccbd0 100755 --- a/hooks/push +++ b/hooks/push @@ -2,4 +2,13 @@ . hooks/common -run_build --push-only +if [ "${SOURCE_BRANCH}" == "main" ] || [ "${DEBUG}" == "true" ]; then + if [ "${SOURCE_BRANCH}" != "main" ]; then + echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'". + fi + + run_build --push-only +else + echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'" + exit 0 +fi From b11afc709d552c3dfc35cab0a7b5c52dd598b1cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Mon, 14 Oct 2019 23:16:51 +0200 Subject: [PATCH 5/5] Improved Build Job descriptions --- DOCKER_HUB.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md index f37833691..10c415434 100644 --- a/DOCKER_HUB.md +++ b/DOCKER_HUB.md @@ -14,14 +14,23 @@ Build Rules: Source: master Docker Tag: branches Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on - Source Type: Branch Source: master Docker Tag: prerelease Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on - Source Type: Branch Source: master Docker Tag: release Dockerfile location: Dockerfile + Build Context: / + Autobuild: on + Build Caching: on Build Environment Variables: # Create an app on Github and use it's OATH credentials here - Key: GITHUB_OAUTH_CLIENT_ID @@ -30,6 +39,7 @@ Build Environment Variables: Value: Build Triggers: - Name: Cron Trigger + Trigger URL: # Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds ```