From eba0ea81dc3704c87867927ded51d67381b4a0eb Mon Sep 17 00:00:00 2001 From: pepov Date: Wed, 8 Aug 2018 14:03:10 +0200 Subject: [PATCH] NIFI-5487 Move the packages to a version agnostic path and create symlinks in place, add tests to cover, add missing docs on nifi-toolkit --- nifi-docker/dockerhub/Dockerfile | 26 ++++++----- nifi-docker/dockerhub/README.md | 48 +++++++++++++++++---- nifi-docker/dockermaven/Dockerfile | 16 ++++--- nifi-docker/dockermaven/integration-test.sh | 14 ++++-- 4 files changed, 76 insertions(+), 28 deletions(-) diff --git a/nifi-docker/dockerhub/Dockerfile b/nifi-docker/dockerhub/Dockerfile index c3df2f648f5a..7c844ce09824 100644 --- a/nifi-docker/dockerhub/Dockerfile +++ b/nifi-docker/dockerhub/Dockerfile @@ -29,7 +29,8 @@ ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSI ARG NIFI_TOOLKIT_BINARY_PATH=${NIFI_TOOLKIT_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-toolkit-${NIFI_VERSION}-bin.zip} ENV NIFI_BASE_DIR=/opt/nifi -ENV NIFI_HOME=${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} +ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current +ENV NIFI_TOOLKIT_HOME ${NIFI_BASE_DIR}/nifi-toolkit-current ENV NIFI_PID_DIR=${NIFI_HOME}/run ENV NIFI_LOG_DIR=${NIFI_HOME}/logs @@ -39,13 +40,7 @@ ADD sh/ ${NIFI_BASE_DIR}/scripts/ # Setup NiFi user and create necessary directories RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group ${GID} | cut -d: -f1` \ && useradd --shell /bin/bash -u ${UID} -g ${GID} -m nifi \ - && mkdir -p ${NIFI_HOME}/conf \ - && mkdir -p ${NIFI_HOME}/database_repository \ - && mkdir -p ${NIFI_HOME}/flowfile_repository \ - && mkdir -p ${NIFI_HOME}/content_repository \ - && mkdir -p ${NIFI_HOME}/provenance_repository \ - && mkdir -p ${NIFI_HOME}/state \ - && mkdir -p ${NIFI_LOG_DIR} \ + && mkdir -p ${NIFI_BASE_DIR} \ && chown -R nifi:nifi ${NIFI_BASE_DIR} \ && apt-get update \ && apt-get install -y jq xmlstarlet procps @@ -56,13 +51,24 @@ USER nifi RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_TOOLKIT_BINARY_PATH} -o ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip \ && echo "$(curl ${BASE_URL}/${NIFI_TOOLKIT_BINARY_PATH}.sha256) *${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip" | sha256sum -c - \ && unzip ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \ - && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip + && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip \ + && mv ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION} ${NIFI_TOOLKIT_HOME} \ + && ln -s ${NIFI_TOOLKIT_HOME} ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION} # Download, validate, and expand Apache NiFi binary. RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_BINARY_PATH} -o ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip \ && echo "$(curl ${BASE_URL}/${NIFI_BINARY_PATH}.sha256) *${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip" | sha256sum -c - \ && unzip ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \ - && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip + && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip \ + && mv ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} ${NIFI_HOME} \ + && mkdir -p ${NIFI_HOME}/conf \ + && mkdir -p ${NIFI_HOME}/database_repository \ + && mkdir -p ${NIFI_HOME}/flowfile_repository \ + && mkdir -p ${NIFI_HOME}/content_repository \ + && mkdir -p ${NIFI_HOME}/provenance_repository \ + && mkdir -p ${NIFI_HOME}/state \ + && mkdir -p ${NIFI_LOG_DIR} \ + && ln -s ${NIFI_HOME} ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} VOLUME ${NIFI_LOG_DIR} \ ${NIFI_HOME}/conf \ diff --git a/nifi-docker/dockerhub/README.md b/nifi-docker/dockerhub/README.md index 5c523b00660e..dabf7eb6c20c 100644 --- a/nifi-docker/dockerhub/README.md +++ b/nifi-docker/dockerhub/README.md @@ -13,13 +13,33 @@ limitations under the License. --> +## Latest changes + +### 1.8.0 +- The NiFi Toolkit has been added to the image under the path `/opt/nifi/nifi-toolkit-current` also set as the environment variable `NIFI_TOOLKIT_HOME` +- The installation directory and related environment variables are changed to be version-agnostic to `/opt/nifi/nifi-current`: +``` +docker run --rm --entrypoint /bin/bash apache/nifi:1.8.0 -c 'env | grep NIFI' +NIFI_HOME=/opt/nifi/nifi-current +NIFI_LOG_DIR=/opt/nifi/nifi-current/logs +NIFI_TOOLKIT_HOME=/opt/nifi/nifi-toolkit-current +NIFI_PID_DIR=/opt/nifi/nifi-current/run +NIFI_BASE_DIR=/opt/nifi +``` +- A symlink refer to the new path for backward compatibility: +``` +docker run --rm --entrypoint /bin/bash apache/nifi:1.8.0 -c 'readlink /opt/nifi/nifi-1.8.0' /opt/nifi/nifi-current +``` + # Docker Image Quickstart ## Capabilities This image currently supports running in standalone mode either unsecured or with user authentication provided through: * [Two-Way SSL with Client Certificates](http://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#security-configuration) * [Lightweight Directory Access Protocol (LDAP)](http://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#ldap_login_identity_provider) - + +This image also contains the NiFi Toolkit (as of version 1.8.0) preconfigured to use either in secure and unsecure mode. + ## Building The Docker image can be built using the following command: @@ -34,7 +54,7 @@ This build will result in an image tagged apache/nifi:latest **Note**: The default version of NiFi specified by the Dockerfile is typically that of one that is unreleased if working from source. To build an image for a prior released version, one can override the `NIFI_VERSION` build-arg with the following command: - + docker build --build-arg=NIFI_VERSION={Desired NiFi Version} -t apache/nifi:latest . There is, however, no guarantee that older versions will work as properties have changed and evolved with subsequent releases. @@ -49,7 +69,7 @@ The minimum to run a NiFi instance is as follows: -p 8080:8080 \ -d \ apache/nifi:latest - + This will provide a running instance, exposing the instance UI to the host system on at port 8080, viewable at `http://localhost:8080/nifi`. @@ -60,9 +80,9 @@ You can also pass in environment variables to change the NiFi communication port -d \ -e NIFI_WEB_HTTP_PORT='9090' apache/nifi:latest - + For a list of the environment variables recognised in this build, look into the .sh/secure.sh and .sh/start.sh scripts - + ### Standalone Instance, Two-Way SSL In this configuration, the user will need to provide certificates and the associated configuration information. Of particular note, is the `AUTH` environment variable which is set to `tls`. Additionally, the user must provide an @@ -88,8 +108,8 @@ Finally, this command makes use of a volume to provide certificates on the host In this configuration, the user will need to provide certificates and the associated configuration information. Optionally, if the LDAP provider of interest is operating in LDAPS or START_TLS modes, certificates will additionally be needed. Of particular note, is the `AUTH` environment variable which is set to `ldap`. Additionally, the user must provide a -DN as provided by the configured LDAP server in the `INITIAL_ADMIN_IDENTITY` environment variable. This value will be -used to seed the instance with an initial user with administrative privileges. Finally, this command makes use of a +DN as provided by the configured LDAP server in the `INITIAL_ADMIN_IDENTITY` environment variable. This value will be +used to seed the instance with an initial user with administrative privileges. Finally, this command makes use of a volume to provide certificates on the host system to the container instance. #### For a minimal, connection to an LDAP server using SIMPLE authentication: @@ -147,8 +167,20 @@ volume to provide certificates on the host system to the container instance. | Connect String | NIFI_ZK_CONNECT_STRING | | Root Node | NIFI_ZK_ROOT_NODE | + +### Using the Toolkit + +Start the container: + + docker run -d --name nifi apache/nifi + +After NiFi has been started, it is possible to run toolkit commands against the running instance: + + docker exec -ti nifi nifi-toolkit-current/bin/cli.sh nifi current-user + anonymous + ## Configuration Information -The following ports are specified by default in Docker for NiFi operation within the container and +The following ports are specified by default in Docker for NiFi operation within the container and can be published to the host. | Function | Property | Port | diff --git a/nifi-docker/dockermaven/Dockerfile b/nifi-docker/dockermaven/Dockerfile index 796a8fbd6700..25c3ab7ea7a5 100644 --- a/nifi-docker/dockermaven/Dockerfile +++ b/nifi-docker/dockermaven/Dockerfile @@ -24,7 +24,8 @@ ARG NIFI_BINARY ARG NIFI_TOOLKIT_BINARY ENV NIFI_BASE_DIR /opt/nifi -ENV NIFI_HOME $NIFI_BASE_DIR/nifi-$NIFI_VERSION +ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current +ENV NIFI_TOOLKIT_HOME ${NIFI_BASE_DIR}/nifi-toolkit-current ENV NIFI_PID_DIR=${NIFI_HOME}/run ENV NIFI_LOG_DIR=${NIFI_HOME}/logs @@ -32,11 +33,15 @@ ADD sh/ ${NIFI_BASE_DIR}/scripts/ COPY $NIFI_BINARY $NIFI_BASE_DIR RUN unzip ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \ - && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip + && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip \ + && mv ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} ${NIFI_HOME} \ + && ln -s ${NIFI_HOME} ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} COPY $NIFI_TOOLKIT_BINARY $NIFI_BASE_DIR RUN unzip ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \ - && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip + && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip \ + && mv ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION} ${NIFI_TOOLKIT_HOME} \ + && ln -s ${NIFI_TOOLKIT_HOME} ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION} # Create necessary directories RUN mkdir -p ${NIFI_HOME}/conf \ @@ -53,10 +58,9 @@ LABEL maintainer="Apache NiFi " ARG UID=1000 ARG GID=1000 -ARG NIFI_VERSION - ENV NIFI_BASE_DIR /opt/nifi -ENV NIFI_HOME $NIFI_BASE_DIR/nifi-$NIFI_VERSION +ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current +ENV NIFI_TOOLKIT_HOME ${NIFI_BASE_DIR}/nifi-toolkit-current ENV NIFI_PID_DIR=${NIFI_HOME}/run ENV NIFI_LOG_DIR=${NIFI_HOME}/logs diff --git a/nifi-docker/dockermaven/integration-test.sh b/nifi-docker/dockermaven/integration-test.sh index e1eedda2c1cc..4b3b2fbaa629 100755 --- a/nifi-docker/dockermaven/integration-test.sh +++ b/nifi-docker/dockermaven/integration-test.sh @@ -20,15 +20,18 @@ set -exuo pipefail TAG=$1 VERSION=$2 -trap "{ docker rm -f nifi-${TAG}-integration-test; }" EXIT +trap "{ docker ps -qaf Name=nifi-${TAG}-integration-test | xargs docker rm -f; }" EXIT echo "Checking that all files are owned by NiFi" test -z $(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "find /opt/nifi ! -user nifi") echo "Checking environment variables" -test "/opt/nifi/nifi-${VERSION}" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_HOME')" -test "/opt/nifi/nifi-${VERSION}/logs" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_LOG_DIR')" -test "/opt/nifi/nifi-${VERSION}/run" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_PID_DIR')" +test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_HOME')" +test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "readlink \${NIFI_BASE_DIR}/nifi-${VERSION}")" +test "/opt/nifi/nifi-toolkit-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "readlink \${NIFI_BASE_DIR}/nifi-toolkit-${VERSION}")" + +test "/opt/nifi/nifi-current/logs" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_LOG_DIR')" +test "/opt/nifi/nifi-current/run" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_PID_DIR')" test "/opt/nifi" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_BASE_DIR')" echo "Starting NiFi container..." @@ -46,5 +49,8 @@ done echo "Checking system diagnostics" test ${VERSION} = $(docker exec nifi-${TAG}-integration-test bash -c "curl -s $IP:8080/nifi-api/system-diagnostics | jq .systemDiagnostics.aggregateSnapshot.versionInfo.niFiVersion -r") +echo "Checking current user with nifi-toolkit cli" +test "anonymous" = $(docker exec nifi-${TAG}-integration-test bash -c '$NIFI_TOOLKIT_HOME/bin/cli.sh nifi current-user') + echo "Stopping NiFi container" time docker stop nifi-${TAG}-integration-test \ No newline at end of file