diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..96f42fe62bcd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +**/__pycache__/ +**/.mypy_cache +**/.pytest_cache +**/.tox +**/.vscode +**/.idea +**/.coverage +**/.DS_Store +**/.eggs +**/.python-version +**/*.egg-info +**/*.bak +**/*.db +**/*.pyc +**/*.sqllite +**/*.swp + +tests/ +docs/ +install/ +superset/assets/node_modules/ +superset/assets/cypress/ +superset/assets/coverage/ diff --git a/.gitignore b/.gitignore index ae6f75e41b20..c6cc07bf354b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,10 +44,3 @@ yarn-error.log *.iml venv @eaDir/ - -# docker -/Dockerfile -/docker-build.sh -/docker-compose.yml -/docker-entrypoint.sh -/docker-init.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0386248c82ac..e2c6802e31c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -255,11 +255,11 @@ Install third-party dependencies listed in `package.json`: # From the root of the repository cd superset/assets -# Install yarn, a replacement for `npm install` +# If needed, install yarn, a replacement for `npm install` npm install -g yarn # Install dependencies -yarn install +yarn ``` Finally, to compile frontend assets, run any of the following commands. diff --git a/contrib/docker/.env b/contrib/docker/.env new file mode 100644 index 000000000000..0a1d0680ffd5 --- /dev/null +++ b/contrib/docker/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=superset diff --git a/contrib/docker/Dockerfile b/contrib/docker/Dockerfile index 0debe1eddcc7..a59a3edd4379 100644 --- a/contrib/docker/Dockerfile +++ b/contrib/docker/Dockerfile @@ -1,64 +1,63 @@ FROM python:3.6 -MAINTAINER Xiao Hanyu - -# Add a normal user -RUN useradd --user-group --create-home --shell /bin/bash work +RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset # Configure environment ENV LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - HOME=/home/work + LC_ALL=C.UTF-8 RUN apt-get update -y -#Install dependencies to fix `curl https support error` and `elaying package configuration warning` + +# Install dependencies to fix `curl https support error` and `elaying package configuration warning` RUN apt-get install -y apt-transport-https apt-utils -# Install some dependencies -# http://airbnb.io/superset/installation.html#os-dependencies -RUN apt-get update -y && apt-get install -y build-essential libssl-dev \ + +# Install superset dependencies +# https://superset.incubator.apache.org/installation.html#os-dependencies +RUN apt-get install -y build-essential libssl-dev \ libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev +# Install extra useful tool for development RUN apt-get install -y vim less postgresql-client redis-tools # Install nodejs for custom build -# https://github.com/apache/incubator-superset/blob/master/docs/installation.rst#making-your-own-build +# https://superset.incubator.apache.org/installation.html#making-your-own-build # https://nodejs.org/en/download/package-manager/ -RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - -RUN apt-get install -y nodejs -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -; \ - echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list; \ - apt-get update; \ - apt-get install -y yarn +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \ + && apt-get install -y nodejs -RUN mkdir $HOME/incubator-superset +# https://yarnpkg.com/lang/en/docs/install/#debian-stable +RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ + && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \ + && apt-get update \ + && apt-get install -y yarn -WORKDIR $HOME/incubator-superset +WORKDIR /home/superset -COPY ./ ./ +COPY requirements.txt . +COPY requirements-dev.txt . -RUN mkdir -p /home/work/.cache -RUN pip install --upgrade setuptools pip -RUN pip install -r requirements.txt -RUN pip install -r requirements-dev.txt -RUN pip install -e . +RUN pip install --upgrade setuptools pip \ + && pip install -r requirements.txt -r requirements-dev.txt \ + && rm -rf /root/.cache/pip -ENV PATH=/home/work/incubator-superset/superset/bin:$PATH \ - PYTHONPATH=./superset/:$PYTHONPATH +USER superset -COPY docker-entrypoint.sh /usr/local/bin/ -RUN chmod +x /usr/local/bin/docker-entrypoint.sh -RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat +COPY --chown=superset:superset superset superset -COPY ./superset ./superset -RUN chown -R work:work $HOME +ENV PATH=/home/superset/superset/bin:$PATH \ + PYTHONPATH=/home/superset/superset/:$PYTHONPATH -USER work +RUN cd superset/assets \ + && yarn --non-interactive --frozen-lockfile --link-duplicates \ + && yarn run sync-backend \ + && yarn run build \ + && rm -rf node_modules \ + && yarn cache clean -RUN cd superset/assets && yarn -RUN cd superset/assets && npm run build +COPY contrib/docker/docker-init.sh . +COPY contrib/docker/docker-entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] HEALTHCHECK CMD ["curl", "-f", "http://localhost:8088/health"] -ENTRYPOINT ["docker-entrypoint.sh"] - EXPOSE 8088 diff --git a/contrib/docker/docker-build.sh b/contrib/docker/docker-build.sh deleted file mode 100644 index 55f73274d67c..000000000000 --- a/contrib/docker/docker-build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -docker build -t apache/incubator-superset -f Dockerfile . diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 9085793c10fd..49119055fb27 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -2,14 +2,14 @@ version: '3' services: redis: image: redis:3.2 - restart: always + restart: unless-stopped ports: - 6379:6379 volumes: - redis:/data postgres: image: postgres:10 - restart: always + restart: unless-stopped environment: POSTGRES_DB: superset POSTGRES_PASSWORD: superset @@ -19,8 +19,10 @@ services: volumes: - postgres:/var/lib/postgresql/data superset: - image: apache/incubator-superset - restart: always + build: + context: ../../ + dockerfile: contrib/docker/Dockerfile + restart: unless-stopped environment: POSTGRES_DB: superset POSTGRES_USER: superset @@ -29,20 +31,21 @@ services: POSTGRES_PORT: 5432 REDIS_HOST: redis REDIS_PORT: 6379 - SUPERSET_ENV: local + # If using production, comment development volume below + #SUPERSET_ENV: production + SUPERSET_ENV: development ports: - 8088:8088 - command: "tail -f /dev/null" depends_on: - postgres - redis volumes: - - .:/home/work/incubator-superset - - superset-node-modules:/home/work/incubator-superset/superset/assets/node_modules + # this is needed to communicate with the postgres and redis services + - ./superset_config.py:/home/superset/superset/superset_config.py + # this is needed for development, remove with SUPERSET_ENV=production + - ../../superset:/home/superset/superset volumes: postgres: external: false redis: external: false - superset-node-modules: - external: false diff --git a/contrib/docker/docker-entrypoint.sh b/contrib/docker/docker-entrypoint.sh old mode 100644 new mode 100755 index 3e1de5ec8db5..35ae37f38d35 --- a/contrib/docker/docker-entrypoint.sh +++ b/contrib/docker/docker-entrypoint.sh @@ -3,9 +3,14 @@ set -ex if [ "$#" -ne 0 ]; then exec "$@" -elif [ "$SUPERSET_ENV" = "local" ]; then - flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 +elif [ "$SUPERSET_ENV" = "development" ]; then + superset worker & + # needed by superset runserver + (cd superset/assets/ && yarn && yarn run sync-backend) + (cd superset/assets/ && yarn run dev) & + flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 elif [ "$SUPERSET_ENV" = "production" ]; then + superset worker & superset runserver -a 0.0.0.0 -w $((2 * $(getconf _NPROCESSORS_ONLN) + 1)) else superset --help diff --git a/contrib/docker/docker-init.sh b/contrib/docker/docker-init.sh old mode 100644 new mode 100755 index f50046ba937c..f92c0f54ea6a --- a/contrib/docker/docker-init.sh +++ b/contrib/docker/docker-init.sh @@ -8,17 +8,10 @@ fabmanager create-admin --app superset # Initialize the database superset db upgrade -# Load some data to play with -superset load_examples +if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then + # Load some data to play with + superset load_examples +fi # Create default roles and permissions superset init - -# Need to run `npm run build` when enter contains for first time -cd superset/assets && npm run build && cd ../../ - -# Start superset worker for SQL Lab -superset worker & - -# Start the dev web server -flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 diff --git a/docs/installation.rst b/docs/installation.rst index d5c63c4bf1df..295a20d09cd6 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -43,22 +43,29 @@ If you know docker, then you're lucky, we have shortcut road for you to initialize development environment: :: git clone https://github.com/apache/incubator-superset/ - cd incubator-superset - cp contrib/docker/{docker-build.sh,docker-compose.yml,docker-entrypoint.sh,docker-init.sh,Dockerfile} . - cp contrib/docker/superset_config.py superset/ - bash -x docker-build.sh - docker-compose up -d - docker-compose exec superset bash - bash docker-init.sh + cd incubator-superset/contrib/docker + # prefix with SUPERSET_LOAD_EXAMPLES=yes to load examples: + docker-compose run --rm superset ./docker-init.sh + # you can run this command everytime you need to start superset now: + docker-compose up After several minutes for superset initialization to finish, you can open a browser and view `http://localhost:8088` to start your journey. +From there, the container server will reload on modification of the superset python +and javascript source code. +Don't forget to reload the page to take the new frontend into account though. + +See also `CONTRIBUTING.md `_, +for alternative way of serving the frontend. + +It is also possible to run Superset in non-development mode: in the `docker-compose.yml` file remove +the volumes needed for development and change the variable `SUPERSET_ENV` to `production`. + If you are attempting to build on a Mac and it exits with 137 you need to increase your docker resources. OSX instructions: https://docs.docker.com/docker-for-mac/#advanced (Search for memory) -Or if you're curious and want to install superset from bottom up, then go -ahead. +Or if you're curious and want to install superset from bottom up, then go ahead. OS dependencies ---------------