From 1576013cf1888e0dc712c46835f859a0d3af1f4c Mon Sep 17 00:00:00 2001 From: Brady Todhunter Date: Wed, 30 Jan 2019 01:30:58 -0800 Subject: [PATCH] functional postgres & registry Signed-off-by: Brady Todhunter --- .dockerignore | 4 + .gitignore | 3 +- Dockerfile | 64 +++++++------ conf/stateless_ci_config.yaml | 139 +++++++++++++++++++++++++++ docker-entrypoint.sh | 171 +++++++++++++++++++++++----------- scripts/anchore_ci_tools.py | 0 6 files changed, 297 insertions(+), 84 deletions(-) create mode 100644 .dockerignore create mode 100644 conf/stateless_ci_config.yaml mode change 100644 => 100755 docker-entrypoint.sh mode change 100644 => 100755 scripts/anchore_ci_tools.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d51d292 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.circleci +.vscode +.gitignore +test/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 39ae917..497b3c8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ logs/ *.DS_Store *.pyc .python-version -.vscode/ \ No newline at end of file +.vscode/ +test/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7680163..3cc20d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,8 @@ -FROM anchore/anchore-engine:v0.3.2 - -ENV PG_MAJOR 9.6 -ENV PGDATA /var/lib/postgresql/data +FROM anchore/anchore-engine:dev RUN apt-get update; \ apt-get upgrade; \ - apt-get install -y ca-certificates + apt-get install -y ca-certificates wget gosu # explicitly set user/group IDs RUN set -ex; \ @@ -15,39 +12,48 @@ RUN set -ex; \ # also create the postgres user's home directory with appropriate permissions # see https://github.com/docker-library/postgres/issues/274 mkdir -p /var/lib/postgresql; \ - chown -R postgres:postgres /var/lib/postgresql - -# grab gosu for easy step-down from root -ENV GOSU_VERSION 1.11 -RUN set -x \ - && apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \ - && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ - && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \ - && export GNUPGHOME="$(mktemp -d)" \ - && gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ - && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ - && { command -v gpgconf > /dev/null && gpgconf --kill all || :; } \ - && rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc \ - && chmod +x /usr/local/bin/gosu \ - && gosu nobody true \ - && apt-get purge -y --auto-remove ca-certificates wget - -RUN mkdir /docker-entrypoint-initdb.d + chown -R postgres:postgres /var/lib/postgresql; \ + mkdir /docker-entrypoint-initdb.d; \ + rm -f /config/config.yaml + +ENV PG_MAJOR 9.6 +ENV PGDATA /var/lib/postgresql/data RUN set -eux; \ - echo 'deb http://apt.postgresql.org/pub/repos/apt/ 18.04-pgdg main' > /etc/apt/sources.list.d/pgdg.list; \ + export DEBIAN_FRONTEND=noninteractive; \ + export DEBCONF_NONINTERACTIVE_SEEN=true; \ + echo 'tzdata tzdata/Areas select Etc' | debconf-set-selections; \ + echo 'tzdata tzdata/Zones/Etc select UTC' | debconf-set-selections; \ + echo 'deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main' > /etc/apt/sources.list.d/pgdg.list; \ curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -; \ apt-get update; \ - apt-get install -y postgresql-common; \ + apt-get install -y --no-install-recommends postgresql-common; \ sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf; \ apt-get install -y "postgresql-${PG_MAJOR}"; \ - apt-get purge -y --auto-remove; + rm -rf /var/lib/apt/lists/*; \ + apt-get purge -y --auto-remove RUN mkdir -p /var/run/postgresql && chown -R postgres:postgres /var/run/postgresql && chmod 2775 /var/run/postgresql RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 700 "$PGDATA" -VOLUME /var/lib/postgresql/data +ENV REGISTRY_VERSION 2.7 + +RUN set -eux; \ + mkdir -p /etc/docker/registry; \ + wget -O /usr/local/bin/registry https://github.com/docker/distribution-library-image/raw/release/${REGISTRY_VERSION}/amd64/registry; \ + chmod +x /usr/local/bin/registry; \ + wget -O /etc/docker/registry/config.yml https://raw.githubusercontent.com/docker/distribution-library-image/release/${REGISTRY_VERSION}/amd64/config-example.yml; \ + apt-get purge -y ca-certificates wget + +COPY anchore-bootstrap.sql.gz /docker-entrypoint-initdb.d/ +COPY conf/stateless_ci_config.yaml /config/config.yaml +COPY scripts/anchore_ci_tools.py /usr/local/bin/ COPY docker-entrypoint.sh /usr/local/bin/ -ENTRYPOINT ["docker-entrypoint.sh"] -EXPOSE 5432 \ No newline at end of file +ENV ANCHORE_CLI_URL "http://anchore-ci:8228/v1" +ENV ANCHORE_HOST_ID anchore-ci +ENV ANCHORE_ENDPOINT_HOSTNAME anchore-ci + +VOLUME ["/var/lib/registry"] +EXPOSE 5432 5000 +ENTRYPOINT ["docker-entrypoint.sh"] \ No newline at end of file diff --git a/conf/stateless_ci_config.yaml b/conf/stateless_ci_config.yaml new file mode 100644 index 0000000..1fb6c77 --- /dev/null +++ b/conf/stateless_ci_config.yaml @@ -0,0 +1,139 @@ +# Anchore Service Configuration File +# + +# General system-wide configuration options, these should not need to +# be altered for basic operation +# + +service_dir: ${ANCHORE_SERVICE_DIR} +tmp_dir: /analysis_scratch +log_level: ${ANCHORE_LOG_LEVEL} +cleanup_images: False +host_id: '${ANCHORE_HOST_ID}' +internal_ssl_verify: False +auto_restart_services: False + +feeds: + # If set to False, instruct anchore-engine to skip (all) feed sync operations + sync_enabled: False + selective_sync: + # If enabled only sync specific feeds instead of all. + enabled: True + feeds: + vulnerabilities: True + # Warning: enabling the packages and nvd sync causes the service to require much + # more memory to do process the significant data volume. We recommend at least 4GB available for the container + packages: False + nvd: True + # Enabling snyk syncs snyk vulnerability data from an on-premise anchore enterprise feeds service. Please contact + # anchore support for finding out more about this service + snyk: False + anonymous_user_username: anon@ancho.re + anonymous_user_password: pbiU2RYZ2XrmYQ + url: 'https://ancho.re/v1/service/feeds' + client_url: 'https://ancho.re/v1/account/users' + token_url: 'https://ancho.re/oauth/token' + connection_timeout_seconds: 3 + read_timeout_seconds: 60 + + +# As of 0.3.0dev0 this section is used instead of the credentials.users section +# Can be omitted and will default to 'foobar' on db initialization +default_admin_password: ${ANCHORE_CLI_PASS} + +# Can be ommitted and will default to 'admin@myanchore' +default_admin_email: ${ANCHORE_ADMIN_EMAIL} + +credentials: + database: + db_connect: 'postgresql+pg8000://${ANCHORE_DB_USER}:${ANCHORE_DB_PASSWORD}@${ANCHORE_DB_HOST}:${ANCHORE_DB_PORT}/${ANCHORE_DB_NAME}' + db_connect_args: + timeout: 120 + ssl: False + db_pool_size: 30 + db_pool_max_overflow: 100 + +services: + apiext: + enabled: True + require_auth: True + endpoint_hostname: '${ANCHORE_ENDPOINT_HOSTNAME}' + listen: '0.0.0.0' + port: 8228 + authorization_handler: native + kubernetes_webhook: + enabled: True + require_auth: False + endpoint_hostname: '${ANCHORE_ENDPOINT_HOSTNAME}' + listen: '0.0.0.0' + port: 8338 + catalog: + enabled: True + require_auth: True + endpoint_hostname: '${ANCHORE_ENDPOINT_HOSTNAME}' + listen: '0.0.0.0' + port: 8082 +# NOTE: use the below external_* parameters to define the port/tls +# setting that will allow other internal services to access this +# service - if left unset services will use the above, +# e.g. http://: +# external_port: 8082 +# external_tls: False + archive: + compression: + enabled: False + min_size_kbytes: 100 + storage_driver: + name: db + config: {} + cycle_timer_seconds: 1 + cycle_timers: + image_watcher: 3600 + policy_eval: 3600 + vulnerability_scan: 14400 + analyzer_queue: 5 + notifications: 30 + service_watcher: 15 + policy_bundle_sync: 300 + repo_watcher: 60 +# Uncomment if you would like to receive notifications for events triggered by asynchronous operations in the system. +# In addition, uncomment the webhooks section and supply the configuration for either a 'general' or an 'event_log' webhook +# event_log: +# notification: +# enabled: True +# # (optional) notify events that match these levels. If this section is commented, notifications for all events are sent +# level: +# - error + simplequeue: + enabled: True + require_auth: True + endpoint_hostname: '${ANCHORE_ENDPOINT_HOSTNAME}' + listen: '0.0.0.0' + port: 8083 +# external_port: 8083 +# external_tls: False + analyzer: + enabled: True + require_auth: True + cycle_timer_seconds: 1 + cycle_timers: + image_analyzer: 5 + max_threads: 1 + analyzer_driver: 'nodocker' + endpoint_hostname: '${ANCHORE_ENDPOINT_HOSTNAME}' + listen: '0.0.0.0' + port: 8084 +# external_port: 8084 +# external_tls: False + policy_engine: + enabled: True + require_auth: True + endpoint_hostname: '${ANCHORE_ENDPOINT_HOSTNAME}' + listen: '0.0.0.0' + port: 8087 +# external_port: 8087 +# external_tls: False + cycle_timer_seconds: 1 + cycle_timers: + feed_sync: 21600 # 6 hours between feed syncs + feed_sync_checker: 3600 # 1 hour between checks to see if there needs to be a task queued \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh old mode 100644 new mode 100755 index 5abcbba..e9528d4 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,58 +1,121 @@ -#!/bin/bash +#!/bin/bash -ex -export POSTGRES_USER=postgres -export POSTGRES_DB=postgres +export POSTGRES_USER="${POSTGRES_USER:-postgres}" +export POSTGRES_DB="${POSTGRES_DB:-postgres}" +export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-mysecretpassword}" -eval 'initdb --username=${POSTGRES_USER} --pwfile=<(echo "$POSTGRES_PASSWORD")' +export ANCHORE_DB_PASSWORD="$POSTGRES_PASSWORD" +export ANCHORE_DB_USER="$POSTGRES_USER" +export ANCHORE_DB_NAME="$POSTGRES_DB" +export ANCHORE_DB_HOST='anchore-ci' -if [ -n "$POSTGRES_PASSWORD" ]; then - authMethod=md5 +export PATH=$PATH:/usr/lib/postgresql/9.6/bin/ +echo "127.0.0.1 anchore-ci" >> /etc/hosts + +init_db () { + # look specifically for PG_VERSION, as it is expected in the DB dir. + # Prevents DB initialization if data exists in $PGDATA + if [ ! -s "$PGDATA/PG_VERSION" ]; then + + gosu postgres bash -c 'initdb --username=postgres --pwfile=<(echo "$POSTGRES_PASSWORD")' + + if [ -n "$POSTGRES_PASSWORD" ]; then + authMethod=md5 + else + authMethod=trust + fi + + { + echo + echo "host all all all $authMethod" + } >> "$PGDATA/pg_hba.conf" + + # internal start of server in order to allow set-up using psql-client + # does not listen on external TCP/IP and waits until start finishes + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + gosu postgres bash -c 'pg_ctl -D "$PGDATA" \ + -o "-c listen_addresses=''" \ + -w start' + + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + psql=( psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password ) + psql+=( --dbname "$POSTGRES_DB" ) + export psql + + echo + gosu postgres bash -c '\ + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + psql=( psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password ) + psql+=( --dbname "$POSTGRES_DB" ) + export psql + for f in /docker-entrypoint-initdb.d/*; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + echo "$0: running $f" + "$f" + else + echo "$0: sourcing $f" + . "$f" + fi + ;; + *.sql) echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;; + *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;; + *) echo "$0: ignoring $f" ;; + esac + echo + done' + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + gosu postgres bash -c 'pg_ctl -D "$PGDATA" -m fast -w stop' + + unset PGPASSWORD + + echo + echo 'PostgreSQL init process complete; ready for start up.' + echo + fi +} + +start_services () { + touch /var/log/postgres.log && chown postgres:postgres /var/log/postgres.log + echo "starting postgresql." + nohup gosu postgres bash -c 'postgres &> /var/log/postgres.log &' + echo "starting docker registry." + nohup registry serve /etc/docker/registry/config.yml &> /var/log/registry.log & + echo "starting anchore engine." + nohup anchore-manager service start --all &> /var/log/anchore.log & +} + +anchore_analysis () { + echo "$1" + image_name="${1%.*}" + echo "$image_name" + skopeo copy --dest-tls-verify=false docker-archive:/anchore-engine/${1} docker://anchore-ci:5000/${image_name}:latest + anchore-cli image add anchore-ci:5000/${image_name}:latest + anchore-cli image wait anchore-ci:5000/${image_name}:latest + anchore-cli --json evaluate check anchore-ci:5000/${image_name}:latest + anchore-cli --json image vuln anchore-ci:5000/${image_name}:latest all + anchore-cli --json image content anchore-ci:5000/${image_name}:latest os +} + +if [ ! $# -eq 0 ]; then + if [ $1 = 'debug' ]; then + init_db + start_services + exec "${@:2}" + elif [ $1 = 'preload' ]; then + init_db + exec "${@:2}" + else + exec "$@" + fi else - authMethod=trust -fi - -{ - echo - echo "host all all all $authMethod" -} >> "$PGDATA/pg_hba.conf" - -# internal start of server in order to allow set-up using psql-client -# does not listen on external TCP/IP and waits until start finishes -PGUSER="${PGUSER:-$POSTGRES_USER}" \ -pg_ctl -D "$PGDATA" \ - -o "-c listen_addresses=''" \ - -w start - -export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" -psql=( psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --no-password ) -psql+=( --dbname "$POSTGRES_DB" ) - -echo -for f in /docker-entrypoint-initdb.d/*; do - case "$f" in - *.sh) - # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 - # https://github.com/docker-library/postgres/pull/452 - if [ -x "$f" ]; then - echo "$0: running $f" - "$f" - else - echo "$0: sourcing $f" - . "$f" - fi - ;; - *.sql) echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;; - *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;; - *) echo "$0: ignoring $f" ;; - esac - echo -done - -PGUSER="${PGUSER:-$POSTGRES_USER}" \ -pg_ctl -D "$PGDATA" -m fast -w stop - -unset PGPASSWORD - -echo -echo 'PostgreSQL init process complete; ready for start up.' -echo \ No newline at end of file + init_db + start_services + anchore-cli system wait + export -f anchore_analysis + find /anchore-engine/ -type f -name "*.tar" -exec bash -c 'anchore_analysis `basename "$0"`' {} \; +fi \ No newline at end of file diff --git a/scripts/anchore_ci_tools.py b/scripts/anchore_ci_tools.py old mode 100644 new mode 100755