From 16a5bcff2ae179a12363b28ae93648f77b4a02e2 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Thu, 2 Nov 2017 21:07:09 -0700 Subject: [PATCH] Set superset as submodule pointing to 0.20.4 tag (#30) also: * adding initial `superset_config.py` * add dep on mysqlclient * set up some related confidant service entries, but keys don't seem to flow, should the service be called `superset-legacy-iad` or something else ? screen shot 2017-11-02 at 12 11 14 am @hughhhh @amalakar NOT PASSING TESTS --- .dockerignore | 1 - .gitmodules | 3 ++ Dockerfile | 3 +- Makefile | 2 +- bandit_config | 1 + manifest.yaml | 8 +--- ops/config/pillar/env.sls | 1 + ops/config/pillar/superset.sls | 2 +- ops/config/states/superset/dependencies.sls | 16 +++++++ requirements.in | 4 +- requirements.txt | 51 +++++++++++---------- requirements3.txt | 51 +++++++++++---------- superset_config.py | 26 +++++++++++ upstream | 1 + 14 files changed, 107 insertions(+), 63 deletions(-) delete mode 100644 .dockerignore create mode 100644 .gitmodules create mode 100644 bandit_config create mode 100644 superset_config.py create mode 160000 upstream diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 6b8710a711f3..000000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.git diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000000..db4aa1f8c9a8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "superset"] + path = upstream + url = https://github.com/lyft/incubator-superset diff --git a/Dockerfile b/Dockerfile index 288ddb66dab3..1af74fd18f9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM lyft/opsbase:caeebb8308bdb84831261a75530a010fbee2478d ARG IAM_ROLE +COPY . /code/superset-private +RUN cd /code/superset-private && git submodule update --init --recursive COPY ops /code/superset-private/ops COPY requirements.* piptools_requirements.* /code/superset-private/ COPY manifest.yaml /code/superset-private/manifest.yaml RUN SERVICE_NAME=superset CODE_ROOT=/code/superset-private /code/ops/base/build_service.sh -COPY . /code/superset-private diff --git a/Makefile b/Makefile index c43ecd7bfb62..26949e4b10e8 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ else $(error Cannot find base.mk) endif -BANDIT_ENFORCED=true +BANDIT_ENFORCED=false SERVICE_NAME=superset diff --git a/bandit_config b/bandit_config new file mode 100644 index 000000000000..265c1dbdb1bb --- /dev/null +++ b/bandit_config @@ -0,0 +1 @@ +export BANDIT_ENFORCED=false diff --git a/manifest.yaml b/manifest.yaml index 00716b0b79b3..7d73364f33b7 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -1,7 +1,7 @@ name: superset description: internal visualization tool for interactively exploring lyft data (apache/incubator-superset) repository: git@github.com:lyft/superset-private.git -credentials: false # Confidant credentials require a development IAM role +credentials: true containers: - name: legacy command: /code/ops/base/entrypoint.sh @@ -43,9 +43,3 @@ deploy: availability_zone: us-east-1d - name: production legacy: true -python: - mypy: - python_versions: - - '3.6' - exclude: - - 'ops' diff --git a/ops/config/pillar/env.sls b/ops/config/pillar/env.sls index 665d6d502ef2..6a2553487b6a 100644 --- a/ops/config/pillar/env.sls +++ b/ops/config/pillar/env.sls @@ -2,6 +2,7 @@ environment: common: LC_ALL: C.UTF-8 LANG: C.UTF-8 + BANDIT_ENFORCED: false development: PORT: 80 APPLICATION_MONGODB_DBNAME: {{ grains.service_group }} diff --git a/ops/config/pillar/superset.sls b/ops/config/pillar/superset.sls index 5b6da3a53269..89c249b7662e 100644 --- a/ops/config/pillar/superset.sls +++ b/ops/config/pillar/superset.sls @@ -9,7 +9,7 @@ python_executable: python3.6 workers: - web: gunicorn superset:app --workers={{ workers }} --forwarded-allow-ips="*" -k gevent --timeout 120 --worker-connections=1000 -c /etc/gunicorn/gunicorn.conf + web: /usr/local/bin/service_venv gunicorn superset:app --workers={{ workers }} --forwarded-allow-ips="*" -k gevent --timeout 120 --worker-connections=1000 -c /etc/gunicorn/gunicorn.conf envoy_with_gunicorn: True diff --git a/ops/config/states/superset/dependencies.sls b/ops/config/states/superset/dependencies.sls index b5bbe2d778a0..384f768bd6f2 100644 --- a/ops/config/states/superset/dependencies.sls +++ b/ops/config/states/superset/dependencies.sls @@ -18,6 +18,7 @@ Ensure pip dependencies for superset are installed: - libsasl2-dev - libldap2-dev - runit + - libmysqlclient-dev # Start python3.6 dependencies @@ -41,3 +42,18 @@ Ensure virtualenv is up to date: - name: virtualenv >= 15.1.0, <= 16.0.0 # End python3.6 dependencies + +Ensure yarn is installed: + cmd.run: + - name: npm install -g yarn + - cwd: /code/superset-private/upstream/superset/assets + +Fetch npm dependencies: + cmd.run: + - name: yarn + - cwd: /code/superset-private/upstream/superset/assets + +Run javascript build: + cmd.run: + - name: yarn run build + - cwd: /code/superset-private/upstream/superset/assets diff --git a/requirements.in b/requirements.in index 2cb4a3d2d280..f3b78423b87e 100644 --- a/requirements.in +++ b/requirements.in @@ -114,6 +114,7 @@ statsd==3.2.1 typing==3.6.1 # Superset Dependencies +mysqlclient wtforms==2.1 amqp==1.4.9 anyjson==0.3.3 @@ -121,5 +122,4 @@ psycopg2 # Postgres driver sqlalchemy-redshift # Redshift driver pyhive # Presto driver -# Superset github module -superset==0.20.4 +-e upstream/ diff --git a/requirements.txt b/requirements.txt index 673b6db83ff8..c79a9a55d739 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ # control run piptools.compile superset-private # -e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint +-e file:///code/superset-private/upstream alembic==0.9.6 # via flask-migrate amqp==1.4.9 anyjson==0.3.3 @@ -16,15 +17,15 @@ babel==2.5.1 # via flask-babel, flower backports.functools-lru-cache==1.4 billiard==3.3.0.23 # via celery blinker==1.4 -boto3==1.4.4 # via superset +boto3==1.4.4 boto==2.47.0 botocore==1.5.95 # via boto3, s3transfer -celery==3.1.25 # via flower, superset +celery==3.1.25 # via flower certifi==2017.7.27.1 cffi==1.9.1 chardet==3.0.4 # via requests click==6.7 # via flask, flask-appbuilder -colorama==0.3.9 # via flask-appbuilder, superset +colorama==0.3.9 # via flask-appbuilder configparser==3.5.0 coverage==4.4.1 # via pytest-cov cryptography==1.9 @@ -33,23 +34,23 @@ docutils==0.14 # via botocore enum34==1.1.6 flake8-tidy-imports==1.0.6 flake8==3.3.0 -flask-appbuilder==1.9.4 # via superset +flask-appbuilder==1.9.4 flask-babel==0.11.1 # via flask-appbuilder -flask-cache==0.13.1 # via superset +flask-cache==0.13.1 flask-login==0.2.11 # via flask-appbuilder -flask-migrate==2.0.3 # via superset +flask-migrate==2.0.3 flask-openid==1.2.5 # via flask-appbuilder flask-script==2.0.5 -flask-sqlalchemy==2.1 # via flask-appbuilder, flask-migrate, superset -flask-testing==0.6.2 # via superset -flask-wtf==0.14.2 # via flask-appbuilder, superset +flask-sqlalchemy==2.1 # via flask-appbuilder, flask-migrate +flask-testing==0.6.2 +flask-wtf==0.14.2 # via flask-appbuilder flask==0.12.2 -flower==0.9.1 # via superset -future==0.16.0 # via pyhive, superset +flower==0.9.1 +future==0.16.0 # via pyhive gevent==1.2.1 greenlet==0.4.12 gunicorn==19.7.1 -humanize==0.5.1 # via superset +humanize==0.5.1 idna==2.5 ipaddress==1.0.18 isort==4.2.15 @@ -63,14 +64,15 @@ lyft-requests==0.16.0 lyft-stdlib==113.0.0 lyftsettings==0.1.0 # via lyft-requests mako==1.0.7 # via alembic -markdown==2.6.8 # via superset +markdown==2.6.8 markupsafe==1.0 # via jinja2, mako mccabe==0.6.1 # via flake8 +mysqlclient==1.3.12 ndg-httpsclient==0.4.2 numpy==1.13.3 # via pandas packaging==16.8 -pandas==0.20.3 # via superset -parsedatetime==2.0.0 # via superset +pandas==0.20.3 +parsedatetime==2.0.0 pathtools==0.1.2 # via watchdog protobuf==3.2.0 # via lyft-requests psutil==5.4.0 # via rainbow-saddle @@ -79,7 +81,7 @@ py==1.4.34 # via pytest pyasn1==0.2.2 pycodestyle==2.3.1 # via flake8 pycparser==2.17 -pydruid==0.3.1 # via superset +pydruid==0.3.1 pyflakes==1.5.0 # via flake8 pyhive==0.5.0 pyopenssl==16.2.0 @@ -87,25 +89,24 @@ pyparsing==2.1.10 pytest-cov==2.5.1 pytest-mock==1.1 pytest==3.1.1 -python-dateutil==2.6.0 # via alembic, botocore, pandas, superset +python-dateutil==2.6.0 # via alembic, botocore, pandas python-editor==1.0.3 # via alembic python3-openid==3.1.0 # via flask-openid pytz==2017.3 # via babel, celery, flower, pandas pyyaml==3.12 rainbow-saddle==0.4.0 -requests==2.17.3 # via lyft-requests, superset +requests==2.17.3 # via lyft-requests s3transfer==0.1.11 # via boto3 sasl==0.2.1 # via thrift-sasl -simplejson==3.10.0 # via superset +simplejson==3.10.0 six==1.10.0 sqlalchemy-redshift==0.7.0 -sqlalchemy-utils==0.32.16 # via superset -sqlalchemy==1.1.9 # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils, superset -sqlparse==0.2.3 # via superset +sqlalchemy-utils==0.32.16 +sqlalchemy==1.1.9 # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils +sqlparse==0.2.3 statsd==3.2.1 -superset==0.20.4 -thrift-sasl==0.3.0 # via superset -thrift==0.10.0 # via superset, thrift-sasl +thrift-sasl==0.3.0 +thrift==0.10.0 # via thrift-sasl tornado==4.2.0 # via flower typing==3.6.1 ujson==1.35 # via lyft-stdlib diff --git a/requirements3.txt b/requirements3.txt index 673b6db83ff8..c79a9a55d739 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -5,6 +5,7 @@ # control run piptools.compile superset-private # -e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint +-e file:///code/superset-private/upstream alembic==0.9.6 # via flask-migrate amqp==1.4.9 anyjson==0.3.3 @@ -16,15 +17,15 @@ babel==2.5.1 # via flask-babel, flower backports.functools-lru-cache==1.4 billiard==3.3.0.23 # via celery blinker==1.4 -boto3==1.4.4 # via superset +boto3==1.4.4 boto==2.47.0 botocore==1.5.95 # via boto3, s3transfer -celery==3.1.25 # via flower, superset +celery==3.1.25 # via flower certifi==2017.7.27.1 cffi==1.9.1 chardet==3.0.4 # via requests click==6.7 # via flask, flask-appbuilder -colorama==0.3.9 # via flask-appbuilder, superset +colorama==0.3.9 # via flask-appbuilder configparser==3.5.0 coverage==4.4.1 # via pytest-cov cryptography==1.9 @@ -33,23 +34,23 @@ docutils==0.14 # via botocore enum34==1.1.6 flake8-tidy-imports==1.0.6 flake8==3.3.0 -flask-appbuilder==1.9.4 # via superset +flask-appbuilder==1.9.4 flask-babel==0.11.1 # via flask-appbuilder -flask-cache==0.13.1 # via superset +flask-cache==0.13.1 flask-login==0.2.11 # via flask-appbuilder -flask-migrate==2.0.3 # via superset +flask-migrate==2.0.3 flask-openid==1.2.5 # via flask-appbuilder flask-script==2.0.5 -flask-sqlalchemy==2.1 # via flask-appbuilder, flask-migrate, superset -flask-testing==0.6.2 # via superset -flask-wtf==0.14.2 # via flask-appbuilder, superset +flask-sqlalchemy==2.1 # via flask-appbuilder, flask-migrate +flask-testing==0.6.2 +flask-wtf==0.14.2 # via flask-appbuilder flask==0.12.2 -flower==0.9.1 # via superset -future==0.16.0 # via pyhive, superset +flower==0.9.1 +future==0.16.0 # via pyhive gevent==1.2.1 greenlet==0.4.12 gunicorn==19.7.1 -humanize==0.5.1 # via superset +humanize==0.5.1 idna==2.5 ipaddress==1.0.18 isort==4.2.15 @@ -63,14 +64,15 @@ lyft-requests==0.16.0 lyft-stdlib==113.0.0 lyftsettings==0.1.0 # via lyft-requests mako==1.0.7 # via alembic -markdown==2.6.8 # via superset +markdown==2.6.8 markupsafe==1.0 # via jinja2, mako mccabe==0.6.1 # via flake8 +mysqlclient==1.3.12 ndg-httpsclient==0.4.2 numpy==1.13.3 # via pandas packaging==16.8 -pandas==0.20.3 # via superset -parsedatetime==2.0.0 # via superset +pandas==0.20.3 +parsedatetime==2.0.0 pathtools==0.1.2 # via watchdog protobuf==3.2.0 # via lyft-requests psutil==5.4.0 # via rainbow-saddle @@ -79,7 +81,7 @@ py==1.4.34 # via pytest pyasn1==0.2.2 pycodestyle==2.3.1 # via flake8 pycparser==2.17 -pydruid==0.3.1 # via superset +pydruid==0.3.1 pyflakes==1.5.0 # via flake8 pyhive==0.5.0 pyopenssl==16.2.0 @@ -87,25 +89,24 @@ pyparsing==2.1.10 pytest-cov==2.5.1 pytest-mock==1.1 pytest==3.1.1 -python-dateutil==2.6.0 # via alembic, botocore, pandas, superset +python-dateutil==2.6.0 # via alembic, botocore, pandas python-editor==1.0.3 # via alembic python3-openid==3.1.0 # via flask-openid pytz==2017.3 # via babel, celery, flower, pandas pyyaml==3.12 rainbow-saddle==0.4.0 -requests==2.17.3 # via lyft-requests, superset +requests==2.17.3 # via lyft-requests s3transfer==0.1.11 # via boto3 sasl==0.2.1 # via thrift-sasl -simplejson==3.10.0 # via superset +simplejson==3.10.0 six==1.10.0 sqlalchemy-redshift==0.7.0 -sqlalchemy-utils==0.32.16 # via superset -sqlalchemy==1.1.9 # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils, superset -sqlparse==0.2.3 # via superset +sqlalchemy-utils==0.32.16 +sqlalchemy==1.1.9 # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils +sqlparse==0.2.3 statsd==3.2.1 -superset==0.20.4 -thrift-sasl==0.3.0 # via superset -thrift==0.10.0 # via superset, thrift-sasl +thrift-sasl==0.3.0 +thrift==0.10.0 # via thrift-sasl tornado==4.2.0 # via flower typing==3.6.1 ujson==1.35 # via lyft-stdlib diff --git a/superset_config.py b/superset_config.py new file mode 100644 index 000000000000..23bdea06e569 --- /dev/null +++ b/superset_config.py @@ -0,0 +1,26 @@ +import os + +ROW_LIMIT = 5000 +SUPERSET_WORKERS = 4 + +# Your App secret key +SECRET_KEY = os.getenv("CREDENTIALS_SUPERSET_SECRET_KEY") or "NOSECRET!" + +# The SQLAlchemy connection string to your database backend +# This connection defines the path to the database that stores your +# superset metadata (slices, connections, tables, dashboards, ...). +# Note that the connection information to connect to the datasources +# you want to explore are managed directly in the web UI +if os.getenv("APPLICATION_ENV") == 'production': + SQLALCHEMY_DATABASE_URI = 'mysql://{user}:{password}@{host}:5432/{database}'.format( + user=os.getenv("CREDENTIALS_SUPERSET_USERNAME"), + password=os.getenv("CREDENTIALS_SUPERSET_PASSWORD"), + host=os.getenv("CREDENTIALS_INCENTIVES_HOST"), + database=os.getenv("CREDENTIALS_SUPERSET_DATABASE")) + +# Flask-WTF flag for CSRF +CSRF_ENABLED = True + +# Set this API key to enable Mapbox visualizations +MAPBOX_API_KEY = os.getenv("CREDENTIALS_MAPBOX_API_KEY") +ENABLE_PROXY_FIX = True diff --git a/upstream b/upstream new file mode 160000 index 000000000000..62d0db7457d4 --- /dev/null +++ b/upstream @@ -0,0 +1 @@ +Subproject commit 62d0db7457d4a3293f0da9aff3212103a165eb3d