From 2ad790bd40d1162dbf061ed479e74ea619262add Mon Sep 17 00:00:00 2001 From: Alexander Mancevice Date: Fri, 2 Aug 2019 11:16:04 -0400 Subject: [PATCH] squish layers --- .dockerignore | 3 ++ .travis.yml | 5 +- Dockerfile | 91 ++++++++++++++++----------------- Makefile | 21 +++++--- {superset => bin}/superset-demo | 0 {superset => bin}/superset-init | 0 docker-compose.yml | 2 +- requirements-db.txt | 18 +++++++ 8 files changed, 80 insertions(+), 60 deletions(-) create mode 100644 .dockerignore rename {superset => bin}/superset-demo (100%) rename {superset => bin}/superset-init (100%) create mode 100644 requirements-db.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..318ffaa24 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +* +!bin/ +!requirements-db.txt diff --git a/.travis.yml b/.travis.yml index 865c3057d..95c4cde13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,11 @@ install: - npm install -g dockerlint script: - dockerlint -- make +- make build=edge before_deploy: - docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} deploy: - provider: script - script: make push + script: docker push amancevice/superset:edge on: - tags: true repo: amancevice/docker-superset diff --git a/Dockerfile b/Dockerfile index 817877dfb..858305188 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,38 @@ -## -# Build assets +# --- Build assets with NodeJS + FROM node:12.7 AS build + +# Superset version to build ARG SUPERSET_VERSION=0.33.0rc1 -WORKDIR /var/lib/superset/ -RUN wget -O superset.tar.gz https://github.com/apache/incubator-superset/archive/${SUPERSET_VERSION}.tar.gz -RUN tar xzf superset.tar.gz -C /var/lib/superset/ --strip-components=1 -WORKDIR /var/lib/superset/superset/assets -RUN npm install -RUN npm run build +ENV SUPERSET_HOME=/var/lib/superset/ + +# Download source +WORKDIR ${SUPERSET_HOME} +RUN wget -O /tmp/superset.tar.gz https://github.com/apache/incubator-superset/archive/${SUPERSET_VERSION}.tar.gz && \ + tar xzf /tmp/superset.tar.gz -C ${SUPERSET_HOME} --strip-components=1 + +# Build assets +WORKDIR ${SUPERSET_HOME}/superset/assets +RUN npm install && \ + npm run build + +# --- Build dist package -## -# Install Superset and dependencies as Python packages -FROM python:3.6 AS install -WORKDIR /var/lib/superset/ -COPY --from=build /var/lib/superset/ . -RUN pip install . -r requirements.txt +FROM python:3.6 AS dist -## -# Build runtime -FROM python:3.6 AS runtime +# Copy prebuilt workspace into stage +ENV SUPERSET_HOME=/var/lib/superset/ +WORKDIR ${SUPERSET_HOME} +COPY --from=build ${SUPERSET_HOME} . +COPY requirements-db.txt . + +# Create package to install +RUN python setup.py sdist && \ + tar czfv /tmp/superset.tar.gz requirements.txt requirements-db.txt dist + +# --- Install dist package and finalize app + +FROM python:3.6 AS final # Configure environment ENV GUNICORN_BIND=0.0.0.0:8088 \ @@ -35,8 +49,10 @@ ENV GUNICORN_BIND=0.0.0.0:8088 \ ENV GUNICORN_CMD_ARGS="--workers ${GUNICORN_WORKERS} --timeout ${GUNICORN_TIMEOUT} --bind ${GUNICORN_BIND} --limit-request-line ${GUNICORN_LIMIT_REQUEST_LINE} --limit-request-field_size ${GUNICORN_LIMIT_REQUEST_FIELD_SIZE}" # Create superset user & install dependencies +WORKDIR /tmp/superset +COPY --from=dist /tmp/superset.tar.gz . RUN useradd -U -m superset && \ - mkdir -p /etc/superset && \ + mkdir -p /etc/superset && \ mkdir -p ${SUPERSET_HOME} && \ chown -R superset:superset /etc/superset && \ chown -R superset:superset ${SUPERSET_HOME} && \ @@ -54,40 +70,19 @@ RUN useradd -U -m superset && \ libsasl2-dev \ libsasl2-modules-gssapi-mit \ libssl1.0 && \ - apt-get clean - -# Copy Superset from build stage & install database helpers -WORKDIR /usr/local/lib/python3.6/site-packages -COPY --from=install /usr/local/lib/python3.6/site-packages . -RUN pip install --no-cache-dir \ - cython==0.29.13 \ - flask-cors==3.0.3 \ - flask-mail==0.9.1 \ - flask-oauth==0.12 \ - flask_oauthlib==0.9.5 \ - gevent==1.2.2 \ - impyla==0.14.0 \ - infi.clickhouse-orm==1.0.2 \ - mysqlclient==1.4.2 \ - psycopg2==2.7.6.1 \ - pyathena==1.5.1 \ - pybigquery==0.4.10 \ - pyhive==0.5.1 \ - pyldap==2.4.28 \ - pymssql==2.1.4 \ - redis==2.10.5 \ - sqlalchemy-clickhouse==0.1.5.post0 \ - sqlalchemy-redshift==0.7.1 \ - werkzeug==0.14.1 + apt-get clean && \ + tar xzf superset.tar.gz && \ + pip install dist/*.tar.gz -r requirements.txt -r requirements-db.txt && \ + rm -rf ./* # Configure Filesystem -COPY superset /usr/local/bin -VOLUME /home/superset \ - /etc/superset \ - /var/lib/superset +COPY bin /usr/local/bin WORKDIR /home/superset +VOLUME /etc/superset \ + /home/superset \ + /var/lib/superset -# Deploy application +# Finalize application EXPOSE 8088 HEALTHCHECK CMD ["curl", "-f", "http://localhost:8088/health"] CMD ["gunicorn", "superset:app"] diff --git a/Makefile b/Makefile index 1e34b8a5a..f71540961 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -stages := build install runtime +stages := build dist final superset_version := 0.33.0rc1 build := $(shell git describe --tags --always) shells := $(foreach stage,$(stages),shell@$(stage)) @@ -10,8 +10,8 @@ all: .docker/$(build) .docker: mkdir -p $@ -.docker/$(build)@install: .docker/$(build)@build -.docker/$(build)@runtime: .docker/$(build)@install +.docker/$(build)@dist: .docker/$(build)@build +.docker/$(build)@final: .docker/$(build)@dist .docker/$(build)@%: | .docker docker build \ --build-arg SUPERSET_VERSION=$(superset_version) \ @@ -19,16 +19,21 @@ all: .docker/$(build) --tag amancevice/superset:$(build)-$* \ --target $* . -.docker/$(build): .docker/$(build)@runtime - docker tag $(shell cat $<) amancevice/superset:$(superset_version) - docker tag $(shell cat $<) amancevice/superset:latest +.docker/$(build): .docker/$(build)@final + docker tag $(shell cat $<) amancevice/superset:$(build) + cp $< $@ clean: -docker image rm -f $(shell awk {print} .docker/*) -rm -rf .docker -push: - docker push amancevice/superset:$(build) amancevice/superset:latest +demo: .docker/$(build) + docker run --detach \ + --name superset-$(build) \ + --publish 8088:8088 \ + $(shell cat $<) + docker exec -it superset-$(build) superset-demo + docker logs -f superset-$(build) $(stages): %: .docker/$(build)@% diff --git a/superset/superset-demo b/bin/superset-demo similarity index 100% rename from superset/superset-demo rename to bin/superset-demo diff --git a/superset/superset-init b/bin/superset-init similarity index 100% rename from superset/superset-init rename to bin/superset-init diff --git a/docker-compose.yml b/docker-compose.yml index 190448783..f4bc50b04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ version: '3' services: superset: - image: amancevice/superset + image: amancevice/superset:edge ports: - 8088:8088 diff --git a/requirements-db.txt b/requirements-db.txt new file mode 100644 index 000000000..0402ebda4 --- /dev/null +++ b/requirements-db.txt @@ -0,0 +1,18 @@ +flask-cors==3.0.3 +flask-mail==0.9.1 +flask-oauth==0.12 +flask_oauthlib==0.9.5 +gevent==1.2.2 +impyla==0.14.0 +infi.clickhouse-orm==1.0.2 +mysqlclient==1.4.2 +psycopg2==2.7.6.1 +pyathena==1.5.1 +pybigquery==0.4.10 +pyhive==0.5.1 +pyldap==2.4.28 +pymssql==2.1.4 +redis==2.10.5 +sqlalchemy-clickhouse==0.1.5.post0 +sqlalchemy-redshift==0.7.1 +werkzeug==0.14.1