Merge branch 'master' into feat/dashboard_extra_jwt

* master: (26 commits) chore: bump to new superset-ui version (#13932) fix: do not run containers as root by default in Helm chart (#13917) feat(explore): adhoc column formatting for Table chart (#13758) fix(sqla-query): order by aggregations in Presto and Hive (#13739) feat(alert/report): add ALERTS_ATTACH_REPORTS feature flags + feature (#13894) test: Fixes PropertiesModal_spec (#13548) fix: Pin Prophet dependency after breaking changes (#13852) test: Adds tests to dnd controls (#13650) test: Adds tests to the AnnotationLayer component (#13748) test: Refactor and enhance tests for the Explore DatasourcePanel Component (#13799) Add tests (#13778) test: DisplayQueryButton (#13750) Fixing condition around left margin for dashboard layout. Fixes #13863 (#13905) Revert "fix: select table overlay (#13694)" (#13901) test: Adds tests to the OptionControls component (#13729) test: DatasourceControl (#13605) tests for function handleScroll (#13896) test: Adds tests to the CustomFrame component (#13675) test: Adds tests to the AdvancedFrame component (#13664) test: DataTableControl (#13668) ...
apache · Apr 2, 2021 · 308c659 · 308c659
2 parents 51b9e71 + f6f412b
commit 308c659
Show file tree

Hide file tree

Showing 95 changed files with 60,745 additions and 1,186 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -802,6 +802,130 @@ npm install
 npm run cypress open
 ```
 
+### Debugging Server App
+
+Follow these instructions to debug the Flask app running inside a docker container.
+
+First add the following to the ./docker-compose.yaml file
+
+```diff
+superset:
+    env_file: docker/.env
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app"]
+    restart: unless-stopped
++   cap_add:
++     - SYS_PTRACE
+    ports:
+      - 8088:8088
++     - 5678:5678
+    user: "root"
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+    environment:
+      CYPRESS_CONFIG: "${CYPRESS_CONFIG}"
+```
+
+Start Superset as usual
+```bash
+docker-compose up
+```
+
+Install the required libraries and packages to the docker container
+
+Enter the superset_app container
+```bash
+docker exec -it superset_app /bin/bash
+root@39ce8cf9d6ab:/app#
+```
+
+Run the following commands inside the container
+```bash
+apt update
+apt install -y gdb
+apt install -y net-tools
+pip install debugpy
+```
+
+Find the PID for the Flask process. Make sure to use the first PID. The Flask app will re-spawn a sub-process everytime you change any of the python code. So it's important to use the first PID.
+
+```bash
+ps -ef
+
+UID        PID  PPID  C STIME TTY          TIME CMD
+root         1     0  0 14:09 ?        00:00:00 bash /app/docker/docker-bootstrap.sh app
+root         6     1  4 14:09 ?        00:00:04 /usr/local/bin/python /usr/bin/flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+root        10     6  7 14:09 ?        00:00:07 /usr/local/bin/python /usr/bin/flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+```
+
+Inject debugpy into the running Flask process. In this case PID 6.
+```bash
+python3 -m debugpy --listen 0.0.0.0:5678 --pid 6
+```
+
+Verify that debugpy is listening on port 5678
+```bash
+netstat -tunap
+
+Active Internet connections (servers and established)
+Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
+tcp        0      0 0.0.0.0:5678            0.0.0.0:*               LISTEN      462/python
+tcp        0      0 0.0.0.0:8088            0.0.0.0:*               LISTEN      6/python
+```
+
+You are now ready to attach a debugger to the process. Using VSCode you can configure a launch configuration file .vscode/launch.json like so.
+```
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Attach to Superset App in Docker Container",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "127.0.0.1",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/app"
+                }
+            ]
+        },
+    ]
+}
+```
+
+VSCode will not stop on breakpoints right away. We've attached to PID 6 however it does not yet know of any sub-processes. In order to "wakeup" the debugger you need to modify a python file. This will trigger Flask to reload the code and create a new sub-process. This new sub-process will be detected by VSCode and breakpoints will be activated.
+
+
+### Debugging Server App in Kubernetes Environment
+
+To debug Flask running in POD inside kubernetes cluster. You'll need to make sure the pod runs as root and is granted the SYS_TRACE capability.These settings should not be used in production environments.
+
+```
+  securityContext:
+    capabilities:
+      add: ["SYS_PTRACE"]
+```
+
+See (set capabilities for a container)[https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-capabilities-for-a-container] for more details.
+
+Once the pod is running as root and has the SYS_PTRACE capability it will be able to debug the Flask app.
+
+You can follow the same instructions as in the docker-compose. Enter the pod and install the required library and packages; gdb, netstat and debugpy.
+
+Often in a kuernetes environment nodes are not addressable from ouside the cluster. VSCode will thus be unable to remotely connect to port 5678 on a kubernetes node. In order to do this you need to create a tunnel that port forwards 5678 to your local machine.
+
+```
+kubectl port-forward  pod/superset-<some random id> 5678:5678
+```
+
+You can now launch your VSCode debugger with the same config as above. VSCode will connect to to 127.0.0.1:5678 which is forwarded by kubectl to your remote kubernetes POD.
+
+
 ### Storybook
 
 Superset includes a [Storybook](https://storybook.js.org/) to preview the layout/styling of various Superset components, and variations thereof. To open and view the Storybook:

diff --git a/helm/superset/templates/_helpers.tpl b/helm/superset/templates/_helpers.tpl
@@ -49,19 +49,6 @@ Create chart name and version as used by the chart label.
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
-{{- define "superset-bootstrap" -}}
-#!/bin/sh
-{{ if .Values.additionalAptPackages }}
-apt-get update -y \
-  && apt-get install -y --no-install-recommends \
-     {{ range .Values.additionalAptPackages }}{{ . }} {{ end }}\
-  && rm -rf /var/lib/apt/lists/*
-{{ end -}}
-{{ if .Values.additionalRequirements }}
-pip install {{ range .Values.additionalRequirements }}{{ . }} {{ end }}
-{{ end -}}
-{{ end -}}
-
 {{- define "superset-config" }}
 import os
 from cachelib.redis import RedisCache

diff --git a/helm/superset/templates/deployment-beat.yaml b/helm/superset/templates/deployment-beat.yaml
@@ -49,7 +49,7 @@ spec:
         release: {{ .Release.Name }}
     spec:
       securityContext:
-        runAsUser: 0 # Needed in order to allow pip install to work in bootstrap
+        runAsUser: {{ .Values.runAsUser }}
       {{- if .Values.supersetCeleryBeat.initContainers }}
       initContainers:
       {{-  tpl (toYaml .Values.supersetCeleryBeat.initContainers) . | nindent 6 }}

diff --git a/helm/superset/templates/deployment-worker.yaml b/helm/superset/templates/deployment-worker.yaml
@@ -47,7 +47,7 @@ spec:
         release: {{ .Release.Name }}
     spec:
       securityContext:
-        runAsUser: 0 # Needed in order to allow pip install to work in bootstrap
+        runAsUser: {{ .Values.runAsUser }}
       {{- if .Values.supersetWorker.initContainers }}
       initContainers:
       {{-  tpl (toYaml .Values.supersetWorker.initContainers) . | nindent 6 }}

diff --git a/helm/superset/templates/deployment.yaml b/helm/superset/templates/deployment.yaml
@@ -35,7 +35,7 @@ spec:
         # Force reload on config changes
         checksum/superset_config.py: {{ include "superset-config" . | sha256sum }}
         checksum/superset_init.sh: {{ tpl .Values.init.initscript . | sha256sum }}
-        checksum/superset_bootstrap.sh: {{ include "superset-bootstrap" . | sha256sum }}
+        checksum/superset_bootstrap.sh: {{ tpl .Values.bootstrapScript . | sha256sum }}
         checksum/connections: {{ .Values.supersetNode.connections | toYaml | sha256sum }}
         checksum/extraConfigs: {{ .Values.extraConfigs | toYaml | sha256sum }}
         checksum/extraSecrets: {{ .Values.extraSecrets | toYaml | sha256sum }}
@@ -50,7 +50,7 @@ spec:
         release: {{ .Release.Name }}
     spec:
       securityContext:
-        runAsUser: 0 # Needed in order to allow pip install to work in bootstrap
+        runAsUser: {{ .Values.runAsUser }}
       {{- if .Values.supersetNode.initContainers }}
       initContainers:
       {{-  tpl (toYaml .Values.supersetNode.initContainers) . | nindent 6 }}

diff --git a/helm/superset/templates/init-job.yaml b/helm/superset/templates/init-job.yaml
@@ -28,7 +28,7 @@ spec:
       name: {{ template "superset.name" . }}-init-db
     spec:
       securityContext:
-        runAsUser: 0 # Needed in order to allow pip install to work in bootstrap
+        runAsUser: {{ .Values.runAsUser }}
       {{- if .Values.init.initContainers }}
       initContainers:
       {{-  tpl (toYaml .Values.init.initContainers) . | nindent 6 }}
@@ -57,6 +57,8 @@ spec:
             readOnly: true
         {{- end }}
         command: {{  tpl (toJson .Values.init.command) . }}
+        resources:
+{{ toYaml .Values.init.resources | indent 10 }}
       volumes:
         - name: superset-config
           secret:

diff --git a/helm/superset/templates/secret-superset-config.yaml b/helm/superset/templates/secret-superset-config.yaml
@@ -30,7 +30,7 @@ stringData:
   superset_init.sh: |
 {{- tpl .Values.init.initscript . | nindent 4 }}
   superset_bootstrap.sh: |
-{{- include "superset-bootstrap" . | nindent 4 }}
+{{- tpl .Values.bootstrapScript . | nindent 4 }}
 
 {{- if .Values.extraSecrets }}
 {{- range $path, $config := .Values.extraSecrets }}

diff --git a/helm/superset/values.yaml b/helm/superset/values.yaml
@@ -21,15 +21,19 @@
 
 replicaCount: 1
 
-## These requirements are used to build a requirements file which is then applied on init
-## of superset containers
-additionalRequirements:
-  - "psycopg2==2.8.5"
-  - "redis==3.2.1"
-
-## These apt packages are applied on init of superset containers
-additionalAptPackages: {}
-  # - nano
+# User ID directive. This user must have enough permissions to run the bootstrap script
+# Runn containers as root is not recommended in production. Change this to another UID - e.g. 1000 to be more secure
+runAsUser: 0
+
+# Install additional packages and do any other bootstrap configuration in this script
+# For production clusters it's recommended to build own image with this step done in CI
+bootstrapScript: |
+  #!/bin/bash
+  apt-get update -y &&\
+   apt-get install -y --no-install-recommends nano &&\
+   rm -rf /var/lib/apt/lists/*
+  pip install psycopg2==2.8.5 redis==3.2.1
+  if [ ! -f ~/bootstrap ]; then echo "Running Superset with uid {{ .Values.runAsUser }}" > ~/bootstrap; fi
 
 ## The name of the secret which we will use to generate a superset_config.py file
 ## Note: this secret must have the key superset_config.py in it and can include other files as well
@@ -198,6 +202,16 @@ supersetCeleryBeat:
 ##
 ## Init job configuration
 init:
+  # Configure resources
+  # Warning: fab commant consumes a lot of ram and can
+  # cause the process to be killed due to OOM if it exceeds limit
+  resources: {}
+    # limits:
+    #   cpu:
+    #   memory:
+    # requests:
+    #   cpu:
+    #   memory:
   command:
     - "/bin/sh"
     - "-c"

diff --git a/setup.py b/setup.py
@@ -144,7 +144,7 @@ def get_git_sha():
         "postgres": ["psycopg2-binary==2.8.5"],
         "presto": ["pyhive[presto]>=0.4.0"],
         "trino": ["sqlalchemy-trino>=0.2"],
-        "prophet": ["fbprophet>=0.6, <0.7"],
+        "prophet": ["fbprophet>=0.7.1, <0.8", "pystan<3.0"],
         "redshift": ["sqlalchemy-redshift>=0.8.1, < 0.9"],
         "snowflake": ["snowflake-sqlalchemy>=1.2.3, <1.3"],
         "teradata": ["sqlalchemy-teradata==0.9.0.dev0"],

diff --git a/superset-frontend/cypress-base/cypress/integration/explore/visualizations/table.test.ts b/superset-frontend/cypress-base/cypress/integration/explore/visualizations/table.test.ts
@@ -193,7 +193,7 @@ describe('Visualization > Table', () => {
     });
 
     // should display in raw records mode
-    cy.get('div[data-test="query_mode"] .btn.active').contains('Raw Records');
+    cy.get('div[data-test="query_mode"] .btn.active').contains('Raw records');
     cy.get('div[data-test="all_columns"]').should('be.visible');
     cy.get('div[data-test="groupby"]').should('not.exist');