-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Superset installation to Airflow demo. (#4509)
Co-authored-by: Abhi Vaidyanatha <abhivaidyanatha@Abhis-MacBook-Pro.local>
- Loading branch information
1 parent
d725aa9
commit b250514
Showing
13 changed files
with
625 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
# | ||
# Licensed to the Apache Software Foundation (ASF) under one or more | ||
# contributor license agreements. See the NOTICE file distributed with | ||
# this work for additional information regarding copyright ownership. | ||
# The ASF licenses this file to You under the Apache License, Version 2.0 | ||
# (the "License"); you may not use this file except in compliance with | ||
# the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
x-superset-image: &superset-image apache/superset:latest-dev | ||
x-superset-depends-on: &superset-depends-on | ||
- db | ||
- redis | ||
x-superset-volumes: &superset-volumes | ||
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container | ||
- ./docker:/app/docker | ||
- superset_home:/app/superset_home | ||
|
||
version: "3.7" | ||
services: | ||
redis: | ||
image: redis:latest | ||
container_name: superset_cache | ||
restart: unless-stopped | ||
volumes: | ||
- redis:/data | ||
|
||
db: | ||
env_file: docker/.env | ||
image: postgres:10 | ||
container_name: superset_db | ||
restart: unless-stopped | ||
volumes: | ||
- db_home:/var/lib/postgresql/data | ||
|
||
superset: | ||
env_file: docker/.env-non-dev | ||
image: *superset-image | ||
container_name: superset_app | ||
command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] | ||
user: "root" | ||
restart: unless-stopped | ||
ports: | ||
- 8088:8088 | ||
depends_on: *superset-depends-on | ||
volumes: *superset-volumes | ||
|
||
superset-init: | ||
image: *superset-image | ||
container_name: superset_init | ||
command: ["/app/docker/docker-init.sh"] | ||
env_file: docker/.env-non-dev | ||
depends_on: *superset-depends-on | ||
user: "root" | ||
volumes: *superset-volumes | ||
|
||
superset-worker: | ||
image: *superset-image | ||
container_name: superset_worker | ||
command: ["/app/docker/docker-bootstrap.sh", "worker"] | ||
env_file: docker/.env-non-dev | ||
restart: unless-stopped | ||
depends_on: *superset-depends-on | ||
user: "root" | ||
volumes: *superset-volumes | ||
|
||
superset-worker-beat: | ||
image: *superset-image | ||
container_name: superset_worker_beat | ||
command: ["/app/docker/docker-bootstrap.sh", "beat"] | ||
env_file: docker/.env-non-dev | ||
restart: unless-stopped | ||
depends_on: *superset-depends-on | ||
user: "root" | ||
volumes: *superset-volumes | ||
|
||
volumes: | ||
superset_home: | ||
external: false | ||
db_home: | ||
external: false | ||
redis: | ||
external: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
# | ||
# Licensed to the Apache Software Foundation (ASF) under one or more | ||
# contributor license agreements. See the NOTICE file distributed with | ||
# this work for additional information regarding copyright ownership. | ||
# The ASF licenses this file to You under the Apache License, Version 2.0 | ||
# (the "License"); you may not use this file except in compliance with | ||
# the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
COMPOSE_PROJECT_NAME=superset | ||
|
||
# database configurations (do not modify) | ||
DATABASE_DB=superset | ||
DATABASE_HOST=db | ||
DATABASE_PASSWORD=superset | ||
DATABASE_USER=superset | ||
|
||
# database engine specific environment variables | ||
# change the below if you prefers another database engine | ||
DATABASE_PORT=5432 | ||
DATABASE_DIALECT=postgresql | ||
POSTGRES_DB=superset | ||
POSTGRES_USER=superset | ||
POSTGRES_PASSWORD=superset | ||
#MYSQL_DATABASE=superset | ||
#MYSQL_USER=superset | ||
#MYSQL_PASSWORD=superset | ||
#MYSQL_RANDOM_ROOT_PASSWORD=yes | ||
|
||
# Add the mapped in /app/pythonpath_docker which allows devs to override stuff | ||
PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev | ||
REDIS_HOST=redis | ||
REDIS_PORT=6379 | ||
|
||
FLASK_ENV=development | ||
SUPERSET_ENV=development | ||
SUPERSET_LOAD_EXAMPLES=yes | ||
CYPRESS_CONFIG=false | ||
SUPERSET_PORT=8088 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
# | ||
# Licensed to the Apache Software Foundation (ASF) under one or more | ||
# contributor license agreements. See the NOTICE file distributed with | ||
# this work for additional information regarding copyright ownership. | ||
# The ASF licenses this file to You under the Apache License, Version 2.0 | ||
# (the "License"); you may not use this file except in compliance with | ||
# the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
COMPOSE_PROJECT_NAME=superset | ||
|
||
# database configurations (do not modify) | ||
DATABASE_DB=superset | ||
DATABASE_HOST=db | ||
DATABASE_PASSWORD=superset | ||
DATABASE_USER=superset | ||
|
||
# database engine specific environment variables | ||
# change the below if you prefers another database engine | ||
DATABASE_PORT=5432 | ||
DATABASE_DIALECT=postgresql | ||
POSTGRES_DB=superset | ||
POSTGRES_USER=superset | ||
POSTGRES_PASSWORD=superset | ||
#MYSQL_DATABASE=superset | ||
#MYSQL_USER=superset | ||
#MYSQL_PASSWORD=superset | ||
#MYSQL_RANDOM_ROOT_PASSWORD=yes | ||
|
||
# Add the mapped in /app/pythonpath_docker which allows devs to override stuff | ||
PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev | ||
REDIS_HOST=redis | ||
REDIS_PORT=6379 | ||
|
||
FLASK_ENV=production | ||
SUPERSET_ENV=production | ||
SUPERSET_LOAD_EXAMPLES=yes | ||
CYPRESS_CONFIG=false | ||
SUPERSET_PORT=8088 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
#!/usr/bin/env bash | ||
# | ||
# Licensed to the Apache Software Foundation (ASF) under one or more | ||
# contributor license agreements. See the NOTICE file distributed with | ||
# this work for additional information regarding copyright ownership. | ||
# The ASF licenses this file to You under the Apache License, Version 2.0 | ||
# (the "License"); you may not use this file except in compliance with | ||
# the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
|
||
set -eo pipefail | ||
|
||
REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt" | ||
# If Cypress run – overwrite the password for admin and export env variables | ||
if [ "$CYPRESS_CONFIG" == "true" ]; then | ||
export SUPERSET_CONFIG=tests.integration_tests.superset_test_config | ||
export SUPERSET_TESTENV=true | ||
export ENABLE_REACT_CRUD_VIEWS=true | ||
export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset | ||
fi | ||
# | ||
# Make sure we have dev requirements installed | ||
# | ||
if [ -f "${REQUIREMENTS_LOCAL}" ]; then | ||
echo "Installing local overrides at ${REQUIREMENTS_LOCAL}" | ||
pip install -r "${REQUIREMENTS_LOCAL}" | ||
else | ||
echo "Skipping local overrides" | ||
fi | ||
|
||
if [[ "${1}" == "worker" ]]; then | ||
echo "Starting Celery worker..." | ||
celery --app=superset.tasks.celery_app:app worker -Ofair -l INFO | ||
elif [[ "${1}" == "beat" ]]; then | ||
echo "Starting Celery beat..." | ||
celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule | ||
elif [[ "${1}" == "app" ]]; then | ||
echo "Starting web app..." | ||
flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0 | ||
elif [[ "${1}" == "app-gunicorn" ]]; then | ||
echo "Starting web app..." | ||
/app/docker/docker-entrypoint.sh | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
#!/usr/bin/env bash | ||
# | ||
# Licensed to the Apache Software Foundation (ASF) under one or more | ||
# contributor license agreements. See the NOTICE file distributed with | ||
# this work for additional information regarding copyright ownership. | ||
# The ASF licenses this file to You under the Apache License, Version 2.0 | ||
# (the "License"); you may not use this file except in compliance with | ||
# the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
/app/docker/docker-init.sh | ||
|
||
# TODO: copy config overrides from ENV vars | ||
|
||
# TODO: run celery in detached state | ||
|
||
# start up the web server | ||
gunicorn \ | ||
--bind "0.0.0.0:${SUPERSET_PORT}" \ | ||
--access-logfile '-' \ | ||
--error-logfile '-' \ | ||
--workers 1 \ | ||
--worker-class gthread \ | ||
--threads 8 \ | ||
--timeout 60 \ | ||
--limit-request-line 0 \ | ||
--limit-request-field_size 0 \ | ||
"${FLASK_APP}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
#!/bin/bash | ||
# | ||
# Licensed to the Apache Software Foundation (ASF) under one or more | ||
# contributor license agreements. See the NOTICE file distributed with | ||
# this work for additional information regarding copyright ownership. | ||
# The ASF licenses this file to You under the Apache License, Version 2.0 | ||
# (the "License"); you may not use this file except in compliance with | ||
# the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
set -eo pipefail | ||
|
||
if [ "${#}" -ne 0 ]; then | ||
exec "${@}" | ||
else | ||
gunicorn \ | ||
--bind "0.0.0.0:${SUPERSET_PORT}" \ | ||
--access-logfile '-' \ | ||
--error-logfile '-' \ | ||
--workers 1 \ | ||
--worker-class gthread \ | ||
--threads 20 \ | ||
--timeout ${GUNICORN_TIMEOUT:-60} \ | ||
--limit-request-line 0 \ | ||
--limit-request-field_size 0 \ | ||
"${FLASK_APP}" | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
#!/usr/bin/env bash | ||
# | ||
# Licensed to the Apache Software Foundation (ASF) under one or more | ||
# contributor license agreements. See the NOTICE file distributed with | ||
# this work for additional information regarding copyright ownership. | ||
# The ASF licenses this file to You under the Apache License, Version 2.0 | ||
# (the "License"); you may not use this file except in compliance with | ||
# the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
# | ||
set -e | ||
|
||
cd /app/superset-frontend | ||
npm install -g npm@7 | ||
npm install -f --no-optional --global webpack webpack-cli | ||
npm install -f --no-optional | ||
|
||
echo "Running frontend" | ||
npm run dev |
Oops, something went wrong.