Set superset as submodule pointing to 0.20.4 tag (#30)

also:
* adding initial `superset_config.py`
* add dep on mysqlclient
* set up some related confidant service entries, but keys don't seem to flow, should the service be called `superset-legacy-iad` or something else ?

<img width="535" alt="screen shot 2017-11-02 at 12 11 14 am" src="https://user-images.githubusercontent.com/487433/32313932-77835694-bf62-11e7-9a9b-1071617f1c66.png">

@hughhhh @amalakar 

NOT PASSING TESTS

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "superset"]
+	path = upstream
+	url = https://github.com/lyft/incubator-superset

Dockerfile

@@ -1,7 +1,8 @@
 FROM lyft/opsbase:caeebb8308bdb84831261a75530a010fbee2478d
 ARG IAM_ROLE
+COPY . /code/superset-private
+RUN cd /code/superset-private && git submodule update --init --recursive
 COPY ops /code/superset-private/ops
 COPY requirements.* piptools_requirements.* /code/superset-private/
 COPY manifest.yaml /code/superset-private/manifest.yaml
 RUN SERVICE_NAME=superset CODE_ROOT=/code/superset-private /code/ops/base/build_service.sh
-COPY . /code/superset-private

Makefile

@@ -10,7 +10,7 @@ else
 $(error Cannot find base.mk)
 endif
 
-BANDIT_ENFORCED=true
+BANDIT_ENFORCED=false
 
 SERVICE_NAME=superset
 

bandit_config

@@ -0,0 +1 @@
+export BANDIT_ENFORCED=false

manifest.yaml

@@ -1,7 +1,7 @@
 name: superset
 description: internal visualization tool for interactively exploring lyft data (apache/incubator-superset)
 repository: git@github.com:lyft/superset-private.git
-credentials: false # Confidant credentials require a development IAM role
+credentials: true
 containers:
   - name: legacy
     command: /code/ops/base/entrypoint.sh
@@ -43,9 +43,3 @@ deploy:
       availability_zone: us-east-1d
     - name: production
       legacy: true
-python:
-  mypy:
-    python_versions:
-      - '3.6'
-    exclude:
-      - 'ops'

ops/config/pillar/env.sls

@@ -2,6 +2,7 @@ environment:
   common:
     LC_ALL: C.UTF-8
     LANG: C.UTF-8
+    BANDIT_ENFORCED: false
   development:
     PORT: 80
     APPLICATION_MONGODB_DBNAME: {{ grains.service_group }}

ops/config/pillar/superset.sls

@@ -9,7 +9,7 @@
 python_executable: python3.6
 
 workers:
-  web: gunicorn superset:app --workers={{ workers }} --forwarded-allow-ips="*" -k gevent --timeout 120 --worker-connections=1000 -c /etc/gunicorn/gunicorn.conf
+  web: /usr/local/bin/service_venv gunicorn superset:app --workers={{ workers }} --forwarded-allow-ips="*" -k gevent --timeout 120 --worker-connections=1000 -c /etc/gunicorn/gunicorn.conf
 
 envoy_with_gunicorn: True
 

ops/config/states/superset/dependencies.sls

@@ -18,6 +18,7 @@ Ensure pip dependencies for superset are installed:
       - libsasl2-dev
       - libldap2-dev
       - runit
+      - libmysqlclient-dev
 
 # Start python3.6 dependencies
 
@@ -41,3 +42,18 @@ Ensure virtualenv is up to date:
     - name: virtualenv >= 15.1.0, <= 16.0.0
 
 # End python3.6 dependencies
+
+Ensure yarn is installed:
+ cmd.run:
+   - name: npm install -g yarn
+   - cwd: /code/superset-private/upstream/superset/assets
+
+Fetch npm dependencies:
+ cmd.run:
+   - name: yarn
+   - cwd: /code/superset-private/upstream/superset/assets
+
+Run javascript build:
+ cmd.run:
+   - name: yarn run build
+   - cwd: /code/superset-private/upstream/superset/assets

requirements.in

@@ -114,12 +114,12 @@ statsd==3.2.1
 typing==3.6.1
 
 # Superset Dependencies
+mysqlclient
 wtforms==2.1
 amqp==1.4.9
 anyjson==0.3.3
 psycopg2 # Postgres driver
 sqlalchemy-redshift # Redshift driver
 pyhive # Presto driver
 
-# Superset github module
-superset==0.20.4
+-e upstream/

requirements.txt

@@ -5,6 +5,7 @@
 #    control run piptools.compile superset-private
 #
 -e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint
+-e file:///code/superset-private/upstream
 alembic==0.9.6            # via flask-migrate
 amqp==1.4.9
 anyjson==0.3.3
@@ -16,15 +17,15 @@ babel==2.5.1              # via flask-babel, flower
 backports.functools-lru-cache==1.4
 billiard==3.3.0.23        # via celery
 blinker==1.4
-boto3==1.4.4              # via superset
+boto3==1.4.4
 boto==2.47.0
 botocore==1.5.95          # via boto3, s3transfer
-celery==3.1.25            # via flower, superset
+celery==3.1.25            # via flower
 certifi==2017.7.27.1
 cffi==1.9.1
 chardet==3.0.4            # via requests
 click==6.7                # via flask, flask-appbuilder
-colorama==0.3.9           # via flask-appbuilder, superset
+colorama==0.3.9           # via flask-appbuilder
 configparser==3.5.0
 coverage==4.4.1           # via pytest-cov
 cryptography==1.9
@@ -33,23 +34,23 @@ docutils==0.14            # via botocore
 enum34==1.1.6
 flake8-tidy-imports==1.0.6
 flake8==3.3.0
-flask-appbuilder==1.9.4   # via superset
+flask-appbuilder==1.9.4
 flask-babel==0.11.1       # via flask-appbuilder
-flask-cache==0.13.1       # via superset
+flask-cache==0.13.1
 flask-login==0.2.11       # via flask-appbuilder
-flask-migrate==2.0.3      # via superset
+flask-migrate==2.0.3
 flask-openid==1.2.5       # via flask-appbuilder
 flask-script==2.0.5
-flask-sqlalchemy==2.1     # via flask-appbuilder, flask-migrate, superset
-flask-testing==0.6.2      # via superset
-flask-wtf==0.14.2         # via flask-appbuilder, superset
+flask-sqlalchemy==2.1     # via flask-appbuilder, flask-migrate
+flask-testing==0.6.2
+flask-wtf==0.14.2         # via flask-appbuilder
 flask==0.12.2
-flower==0.9.1             # via superset
-future==0.16.0            # via pyhive, superset
+flower==0.9.1
+future==0.16.0            # via pyhive
 gevent==1.2.1
 greenlet==0.4.12
 gunicorn==19.7.1
-humanize==0.5.1           # via superset
+humanize==0.5.1
 idna==2.5
 ipaddress==1.0.18
 isort==4.2.15
@@ -63,14 +64,15 @@ lyft-requests==0.16.0
 lyft-stdlib==113.0.0
 lyftsettings==0.1.0       # via lyft-requests
 mako==1.0.7               # via alembic
-markdown==2.6.8           # via superset
+markdown==2.6.8
 markupsafe==1.0           # via jinja2, mako
 mccabe==0.6.1             # via flake8
+mysqlclient==1.3.12
 ndg-httpsclient==0.4.2
 numpy==1.13.3             # via pandas
 packaging==16.8
-pandas==0.20.3            # via superset
-parsedatetime==2.0.0      # via superset
+pandas==0.20.3
+parsedatetime==2.0.0
 pathtools==0.1.2          # via watchdog
 protobuf==3.2.0           # via lyft-requests
 psutil==5.4.0             # via rainbow-saddle
@@ -79,33 +81,32 @@ py==1.4.34                # via pytest
 pyasn1==0.2.2
 pycodestyle==2.3.1        # via flake8
 pycparser==2.17
-pydruid==0.3.1            # via superset
+pydruid==0.3.1
 pyflakes==1.5.0           # via flake8
 pyhive==0.5.0
 pyopenssl==16.2.0
 pyparsing==2.1.10
 pytest-cov==2.5.1
 pytest-mock==1.1
 pytest==3.1.1
-python-dateutil==2.6.0    # via alembic, botocore, pandas, superset
+python-dateutil==2.6.0    # via alembic, botocore, pandas
 python-editor==1.0.3      # via alembic
 python3-openid==3.1.0     # via flask-openid
 pytz==2017.3              # via babel, celery, flower, pandas
 pyyaml==3.12
 rainbow-saddle==0.4.0
-requests==2.17.3          # via lyft-requests, superset
+requests==2.17.3          # via lyft-requests
 s3transfer==0.1.11        # via boto3
 sasl==0.2.1               # via thrift-sasl
-simplejson==3.10.0        # via superset
+simplejson==3.10.0
 six==1.10.0
 sqlalchemy-redshift==0.7.0
-sqlalchemy-utils==0.32.16  # via superset
-sqlalchemy==1.1.9         # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils, superset
-sqlparse==0.2.3           # via superset
+sqlalchemy-utils==0.32.16
+sqlalchemy==1.1.9         # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils
+sqlparse==0.2.3
 statsd==3.2.1
-superset==0.20.4
-thrift-sasl==0.3.0        # via superset
-thrift==0.10.0            # via superset, thrift-sasl
+thrift-sasl==0.3.0
+thrift==0.10.0            # via thrift-sasl
 tornado==4.2.0            # via flower
 typing==3.6.1
 ujson==1.35               # via lyft-stdlib

requirements3.txt

@@ -5,6 +5,7 @@
 #    control run piptools.compile superset-private
 #
 -e git+git://github.com/lyft/pylint@lyft_pylint#egg=pylint
+-e file:///code/superset-private/upstream
 alembic==0.9.6            # via flask-migrate
 amqp==1.4.9
 anyjson==0.3.3
@@ -16,15 +17,15 @@ babel==2.5.1              # via flask-babel, flower
 backports.functools-lru-cache==1.4
 billiard==3.3.0.23        # via celery
 blinker==1.4
-boto3==1.4.4              # via superset
+boto3==1.4.4
 boto==2.47.0
 botocore==1.5.95          # via boto3, s3transfer
-celery==3.1.25            # via flower, superset
+celery==3.1.25            # via flower
 certifi==2017.7.27.1
 cffi==1.9.1
 chardet==3.0.4            # via requests
 click==6.7                # via flask, flask-appbuilder
-colorama==0.3.9           # via flask-appbuilder, superset
+colorama==0.3.9           # via flask-appbuilder
 configparser==3.5.0
 coverage==4.4.1           # via pytest-cov
 cryptography==1.9
@@ -33,23 +34,23 @@ docutils==0.14            # via botocore
 enum34==1.1.6
 flake8-tidy-imports==1.0.6
 flake8==3.3.0
-flask-appbuilder==1.9.4   # via superset
+flask-appbuilder==1.9.4
 flask-babel==0.11.1       # via flask-appbuilder
-flask-cache==0.13.1       # via superset
+flask-cache==0.13.1
 flask-login==0.2.11       # via flask-appbuilder
-flask-migrate==2.0.3      # via superset
+flask-migrate==2.0.3
 flask-openid==1.2.5       # via flask-appbuilder
 flask-script==2.0.5
-flask-sqlalchemy==2.1     # via flask-appbuilder, flask-migrate, superset
-flask-testing==0.6.2      # via superset
-flask-wtf==0.14.2         # via flask-appbuilder, superset
+flask-sqlalchemy==2.1     # via flask-appbuilder, flask-migrate
+flask-testing==0.6.2
+flask-wtf==0.14.2         # via flask-appbuilder
 flask==0.12.2
-flower==0.9.1             # via superset
-future==0.16.0            # via pyhive, superset
+flower==0.9.1
+future==0.16.0            # via pyhive
 gevent==1.2.1
 greenlet==0.4.12
 gunicorn==19.7.1
-humanize==0.5.1           # via superset
+humanize==0.5.1
 idna==2.5
 ipaddress==1.0.18
 isort==4.2.15
@@ -63,14 +64,15 @@ lyft-requests==0.16.0
 lyft-stdlib==113.0.0
 lyftsettings==0.1.0       # via lyft-requests
 mako==1.0.7               # via alembic
-markdown==2.6.8           # via superset
+markdown==2.6.8
 markupsafe==1.0           # via jinja2, mako
 mccabe==0.6.1             # via flake8
+mysqlclient==1.3.12
 ndg-httpsclient==0.4.2
 numpy==1.13.3             # via pandas
 packaging==16.8
-pandas==0.20.3            # via superset
-parsedatetime==2.0.0      # via superset
+pandas==0.20.3
+parsedatetime==2.0.0
 pathtools==0.1.2          # via watchdog
 protobuf==3.2.0           # via lyft-requests
 psutil==5.4.0             # via rainbow-saddle
@@ -79,33 +81,32 @@ py==1.4.34                # via pytest
 pyasn1==0.2.2
 pycodestyle==2.3.1        # via flake8
 pycparser==2.17
-pydruid==0.3.1            # via superset
+pydruid==0.3.1
 pyflakes==1.5.0           # via flake8
 pyhive==0.5.0
 pyopenssl==16.2.0
 pyparsing==2.1.10
 pytest-cov==2.5.1
 pytest-mock==1.1
 pytest==3.1.1
-python-dateutil==2.6.0    # via alembic, botocore, pandas, superset
+python-dateutil==2.6.0    # via alembic, botocore, pandas
 python-editor==1.0.3      # via alembic
 python3-openid==3.1.0     # via flask-openid
 pytz==2017.3              # via babel, celery, flower, pandas
 pyyaml==3.12
 rainbow-saddle==0.4.0
-requests==2.17.3          # via lyft-requests, superset
+requests==2.17.3          # via lyft-requests
 s3transfer==0.1.11        # via boto3
 sasl==0.2.1               # via thrift-sasl
-simplejson==3.10.0        # via superset
+simplejson==3.10.0
 six==1.10.0
 sqlalchemy-redshift==0.7.0
-sqlalchemy-utils==0.32.16  # via superset
-sqlalchemy==1.1.9         # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils, superset
-sqlparse==0.2.3           # via superset
+sqlalchemy-utils==0.32.16
+sqlalchemy==1.1.9         # via alembic, flask-sqlalchemy, sqlalchemy-redshift, sqlalchemy-utils
+sqlparse==0.2.3
 statsd==3.2.1
-superset==0.20.4
-thrift-sasl==0.3.0        # via superset
-thrift==0.10.0            # via superset, thrift-sasl
+thrift-sasl==0.3.0
+thrift==0.10.0            # via thrift-sasl
 tornado==4.2.0            # via flower
 typing==3.6.1
 ujson==1.35               # via lyft-stdlib

superset_config.py

@@ -0,0 +1,26 @@
+import os
+
+ROW_LIMIT = 5000
+SUPERSET_WORKERS = 4
+
+# Your App secret key
+SECRET_KEY = os.getenv("CREDENTIALS_SUPERSET_SECRET_KEY") or "NOSECRET!"
+
+# The SQLAlchemy connection string to your database backend
+# This connection defines the path to the database that stores your
+# superset metadata (slices, connections, tables, dashboards, ...).
+# Note that the connection information to connect to the datasources
+# you want to explore are managed directly in the web UI
+if os.getenv("APPLICATION_ENV") == 'production':
+    SQLALCHEMY_DATABASE_URI = 'mysql://{user}:{password}@{host}:5432/{database}'.format(
+        user=os.getenv("CREDENTIALS_SUPERSET_USERNAME"),
+        password=os.getenv("CREDENTIALS_SUPERSET_PASSWORD"),
+        host=os.getenv("CREDENTIALS_INCENTIVES_HOST"),
+        database=os.getenv("CREDENTIALS_SUPERSET_DATABASE"))
+
+# Flask-WTF flag for CSRF
+CSRF_ENABLED = True
+
+# Set this API key to enable Mapbox visualizations
+MAPBOX_API_KEY = os.getenv("CREDENTIALS_MAPBOX_API_KEY")
+ENABLE_PROXY_FIX = True