Merge branch 'dashboard-native-filters' of https://github.com/preset-…

…io/incubator-superset into dashboard-native-filters

.github/workflows/bashlib.sh

@@ -88,6 +88,8 @@ build-instrumented-assets() {
 }
 
 setup-postgres() {
+  say "::group::Install dependency for unit tests"
+  sudo apt-get update && sudo apt-get install --yes libecpg-dev
   say "::group::Initialize database"
   psql "postgresql://superset:superset@127.0.0.1:15432/superset" <<-EOF
     DROP SCHEMA IF EXISTS sqllab_test_db CASCADE;

.github/workflows/docker_build_push.sh

@@ -21,14 +21,14 @@ SHA=$(git rev-parse HEAD)
 REPO_NAME="apache/incubator-superset"
 
 if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
-  REFSPEC="${GITHUB_HEAD_REF/[^a-zA-Z0-9]/-}"
+  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/' | head -c 20)
   PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
   LATEST_TAG="pr-${PR_NUM}"
 elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
   REFSPEC="${GITHUB_REF}"
   LATEST_TAG="${REFSPEC}"
 else
-  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/')
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/' | head -c 20)
   LATEST_TAG="${REFSPEC}"
 fi
 

Dockerfile

@@ -28,6 +28,7 @@ RUN mkdir /app \
             default-libmysqlclient-dev \
             libpq-dev \
             libsasl2-dev \
+            libecpg-dev \
         && rm -rf /var/lib/apt/lists/*
 
 # First, we just wanna install requirements, which will allow us to utilize the cache

RELEASING/changelog.py

@@ -14,12 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# pylint: disable=no-value-for-parameter
+
+import csv as lib_csv
 import json
 import os
 import re
+import sys
 from dataclasses import dataclass
 from time import sleep
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Iterator, List, Optional, Union
 from urllib import request
 from urllib.error import HTTPError
 
@@ -37,14 +41,15 @@ class GitLog:
     time: str
     message: str
     pr_number: Union[int, None] = None
+    author_email: str = ""
 
     def __eq__(self, other: object) -> bool:
         """ A log entry is considered equal if it has the same PR number """
         if isinstance(other, self.__class__):
             return other.pr_number == self.pr_number
         return False
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f"[{self.pr_number}]: {self.message} {self.time} {self.author}"
 
 
@@ -73,23 +78,24 @@ def _wait_github_rate_limit(self) -> None:
             sleep(self._wait)
         print()
 
-    def _fetch_github_rate_limit(self) -> Dict[str, Any]:
+    @staticmethod
+    def _fetch_github_rate_limit() -> Dict[str, Any]:
         """
         Fetches current github rate limit info
         """
-        with request.urlopen(f"https://api.github.com/rate_limit") as response:
+        with request.urlopen("https://api.github.com/rate_limit") as response:
             payload = json.loads(response.read())
         return payload
 
-    def _fetch_github_pr(self, pr_number) -> Dict[str, Any]:
+    def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]:
         """
         Fetches a github PR info
         """
         payload = {}
         try:
             self._wait_github_rate_limit()
             with request.urlopen(
-                f"https://api.github.com/repos/apache/incubator-superset/pulls/"
+                "https://api.github.com/repos/apache/incubator-superset/pulls/"
                 f"{pr_number}"
             ) as response:
                 payload = json.loads(response.read())
@@ -105,19 +111,20 @@ def _get_github_login(self, git_log: GitLog) -> Optional[str]:
         github_login = self._github_login_cache.get(author_name)
         if github_login:
             return github_login
-        pr_info = self._fetch_github_pr(git_log.pr_number)
-        if pr_info:
-            github_login = pr_info["user"]["login"]
-        else:
-            github_login = author_name
+        if git_log.pr_number:
+            pr_info = self._fetch_github_pr(git_log.pr_number)
+            if pr_info:
+                github_login = pr_info["user"]["login"]
+            else:
+                github_login = author_name
         # set cache
         self._github_login_cache[author_name] = github_login
         return github_login
 
-    def _get_changelog_version_head(self):
+    def _get_changelog_version_head(self) -> str:
         return f"### {self._version} ({self._logs[0].time})"
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         result = f"\n{self._get_changelog_version_head()}\n"
         for i, log in enumerate(self._logs):
             github_login = self._get_github_login(log)
@@ -131,6 +138,19 @@ def __repr__(self):
             print(f"\r {i}/{len(self._logs)}", end="", flush=True)
         return result
 
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        for log in self._logs:
+            yield {
+                "pr_number": log.pr_number,
+                "pr_link": f"https://github.com/apache/incubator-superset/pull/"
+                f"{log.pr_number}",
+                "message": log.message,
+                "time": log.time,
+                "author": log.author,
+                "email": log.author_email,
+                "sha": log.sha,
+            }
+
 
 class GitLogs:
     """
@@ -151,54 +171,57 @@ def git_ref(self) -> str:
     def logs(self) -> List[GitLog]:
         return self._logs
 
-    def fetch(self):
+    def fetch(self) -> None:
         self._logs = list(map(self._parse_log, self._git_logs()))[::-1]
 
     def diff(self, git_logs: "GitLogs") -> List[GitLog]:
         return [log for log in git_logs.logs if log not in self._logs]
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f"{self._git_ref}, Log count:{len(self._logs)}"
 
-    def _git_get_current_head(self) -> str:
+    @staticmethod
+    def _git_get_current_head() -> str:
         output = os.popen("git status | head -1").read()
         match = re.match("(?:HEAD detached at|On branch) (.*)", output)
         if not match:
             return ""
         return match.group(1)
 
-    def _git_checkout(self, git_ref: str):
+    def _git_checkout(self, git_ref: str) -> None:
         os.popen(f"git checkout {git_ref}").read()
         current_head = self._git_get_current_head()
         if current_head != git_ref:
             print(f"Could not checkout {git_ref}")
-            exit(1)
+            sys.exit(1)
 
     def _git_logs(self) -> List[str]:
         # let's get current git ref so we can revert it back
         current_git_ref = self._git_get_current_head()
         self._git_checkout(self._git_ref)
         output = (
-            os.popen('git --no-pager log --pretty=format:"%h|%an|%ad|%s|"')
+            os.popen('git --no-pager log --pretty=format:"%h|%an|%ae|%ad|%s|"')
             .read()
             .split("\n")
         )
         # revert to git ref, let's be nice
         self._git_checkout(current_git_ref)
         return output
 
-    def _parse_log(self, log_item: str) -> GitLog:
+    @staticmethod
+    def _parse_log(log_item: str) -> GitLog:
         pr_number = None
         split_log_item = log_item.split("|")
         # parse the PR number from the log message
-        match = re.match(".*\(\#(\d*)\)", split_log_item[3])
+        match = re.match(r".*\(\#(\d*)\)", split_log_item[4])
         if match:
             pr_number = int(match.group(1))
         return GitLog(
             sha=split_log_item[0],
             author=split_log_item[1],
-            time=split_log_item[2],
-            message=split_log_item[3],
+            author_email=split_log_item[2],
+            time=split_log_item[3],
+            message=split_log_item[4],
             pr_number=pr_number,
         )
 
@@ -217,13 +240,9 @@ def print_title(message: str) -> None:
 
 @click.group()
 @click.pass_context
-@click.option(
-    "--previous_version", help="The previous release version",
-)
-@click.option(
-    "--current_version", help="The current release version",
-)
-def cli(ctx, previous_version: str, current_version: str):
+@click.option("--previous_version", help="The previous release version", required=True)
+@click.option("--current_version", help="The current release version", required=True)
+def cli(ctx, previous_version: str, current_version: str) -> None:
     """ Welcome to change log generator  """
     previous_logs = GitLogs(previous_version)
     current_logs = GitLogs(current_version)
@@ -235,7 +254,7 @@ def cli(ctx, previous_version: str, current_version: str):
 
 @cli.command("compare")
 @click.pass_obj
-def compare(base_parameters):
+def compare(base_parameters: BaseParameters) -> None:
     """ Compares both versions (by PR) """
     previous_logs = base_parameters.previous_logs
     current_logs = base_parameters.current_logs
@@ -255,14 +274,33 @@ def compare(base_parameters):
 
 
 @cli.command("changelog")
+@click.option(
+    "--csv", help="The csv filename to export the changelog to",
+)
 @click.pass_obj
-def changelog(base_parameters):
+def change_log(base_parameters: BaseParameters, csv: str) -> None:
     """ Outputs a changelog (by PR) """
     previous_logs = base_parameters.previous_logs
     current_logs = base_parameters.current_logs
     previous_diff_logs = previous_logs.diff(current_logs)
-    print("Fetching github usernames, this may take a while:")
-    print(GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1]))
+    logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])
+    if csv:
+        with open(csv, "w") as csv_file:
+            log_items = list(logs)
+            field_names = log_items[0].keys()
+            writer = lib_csv.DictWriter(
+                csv_file,
+                delimiter=",",
+                quotechar='"',
+                quoting=lib_csv.QUOTE_ALL,
+                fieldnames=field_names,
+            )
+            writer.writeheader()
+            for log in logs:
+                writer.writerow(log)
+    else:
+        print("Fetching github usernames, this may take a while:")
+        print(logs)
 
 
 cli()

UPDATING.md

@@ -24,6 +24,7 @@ assists people when migrating to a new version.
 
 ## Next
 
+- [11920](https://github.com/apache/incubator-superset/pull/11920): Undos the DB migration from [11714](https://github.com/apache/incubator-superset/pull/11714) to prevent adding new columns to the logs table. Deploying a sha between these two PRs may result in locking your DB.
 - [11704](https://github.com/apache/incubator-superset/pull/11704) Breaking change: Jinja templating for SQL queries has been updated, removing default modules such as `datetime` and `random` and enforcing static template values. To restore or extend functionality, use `JINJA_CONTEXT_ADDONS` and `CUSTOM_TEMPLATE_PROCESSORS` in `superset_config.py`.
 - [11714](https://github.com/apache/incubator-superset/pull/11714): Logs
   significantly more analytics events (roughly double?), and when

requirements/base.txt

@@ -6,7 +6,7 @@
 #    pip-compile-multi
 #
 -e file:.                 # via -r requirements/base.in
-aiohttp==3.6.2            # via slackclient
+aiohttp==3.7.2            # via slackclient
 alembic==1.4.3            # via flask-migrate
 amqp==2.6.1               # via kombu
 apispec[yaml]==3.3.2      # via flask-appbuilder
@@ -22,9 +22,9 @@ celery==4.4.7             # via apache-superset
 cffi==1.14.3              # via cryptography
 chardet==3.0.4            # via aiohttp
 click==7.1.2              # via apache-superset, flask, flask-appbuilder
-colorama==0.4.3           # via apache-superset, flask-appbuilder
+colorama==0.4.4           # via apache-superset, flask-appbuilder
 contextlib2==0.6.0.post1  # via apache-superset
-croniter==0.3.34          # via apache-superset
+croniter==0.3.36          # via apache-superset
 cryptography==3.2.1       # via apache-superset
 decorator==4.4.2          # via retry
 defusedxml==0.6.0         # via python3-openid
@@ -33,7 +33,7 @@ email-validator==1.1.1    # via flask-appbuilder
 flask-appbuilder==3.1.1   # via apache-superset
 flask-babel==1.0.0        # via flask-appbuilder
 flask-caching==1.9.0      # via apache-superset
-flask-compress==1.5.0     # via apache-superset
+flask-compress==1.8.0     # via apache-superset
 flask-jwt-extended==3.24.1  # via flask-appbuilder
 flask-login==0.4.1        # via flask-appbuilder
 flask-migrate==2.5.3      # via apache-superset
@@ -45,28 +45,29 @@ flask==1.1.2              # via apache-superset, flask-appbuilder, flask-babel,
 geographiclib==1.50       # via geopy
 geopy==2.0.0              # via apache-superset
 gunicorn==20.0.4          # via apache-superset
-humanize==2.6.0           # via apache-superset
+humanize==3.1.0           # via apache-superset
 idna==2.10                # via email-validator, yarl
-importlib-metadata==1.7.0  # via -r requirements/base.in, jsonschema, kombu, markdown
+importlib-metadata==1.7.0  # via -r requirements/base.in
 isodate==0.6.0            # via apache-superset
 itsdangerous==1.1.0       # via flask, flask-wtf
 jinja2==2.11.2            # via flask, flask-babel
 jsonschema==3.2.0         # via flask-appbuilder
 kombu==4.6.11             # via celery
 mako==1.1.3               # via alembic
-markdown==3.2.2           # via apache-superset
+markdown==3.3.3           # via apache-superset
 markupsafe==1.1.1         # via jinja2, mako, wtforms
 marshmallow-enum==1.5.1   # via flask-appbuilder
 marshmallow-sqlalchemy==0.23.1  # via flask-appbuilder
-marshmallow==3.8.0        # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
+marshmallow==3.9.0        # via flask-appbuilder, marshmallow-enum, marshmallow-sqlalchemy
 msgpack==1.0.0            # via apache-superset
-multidict==4.7.6          # via aiohttp, yarl
+multidict==5.0.0          # via aiohttp, yarl
 natsort==7.0.1            # via croniter
-numpy==1.19.2             # via pandas, pyarrow
+numpy==1.19.4             # via pandas, pyarrow
 packaging==20.4           # via bleach
-pandas==1.1.2             # via apache-superset
+pandas==1.1.4             # via apache-superset
 parsedatetime==2.6        # via apache-superset
 pathlib2==2.3.5           # via apache-superset
+pgsanity==0.2.9           # via apache-superset
 polyline==1.4.0           # via apache-superset
 prison==0.1.3             # via flask-appbuilder
 py==1.9.0                 # via retry
@@ -76,29 +77,29 @@ pyjwt==1.7.1              # via flask-appbuilder, flask-jwt-extended
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.1        # via -r requirements/base.in, jsonschema
 python-dateutil==2.8.1    # via alembic, apache-superset, croniter, flask-appbuilder, pandas
-python-dotenv==0.14.0     # via apache-superset
+python-dotenv==0.15.0     # via apache-superset
 python-editor==1.0.4      # via alembic
 python-geohash==0.8.5     # via apache-superset
 python3-openid==3.2.0     # via flask-openid
-pytz==2020.1              # via babel, celery, flask-babel, pandas
+pytz==2020.4              # via babel, celery, flask-babel, pandas
 pyyaml==5.3.1             # via apache-superset, apispec
 retry==0.9.2              # via apache-superset
 selenium==3.141.0         # via apache-superset
 simplejson==3.17.2        # via apache-superset
 six==1.15.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
 slackclient==2.5.0        # via apache-superset
 sqlalchemy-utils==0.36.8  # via apache-superset, flask-appbuilder
-sqlalchemy==1.3.19        # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.3.20        # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 sqlparse==0.3.0           # via apache-superset
-typing-extensions==3.7.4.3  # via yarl
-urllib3==1.25.10          # via selenium
+typing-extensions==3.7.4.3  # via aiohttp
+urllib3==1.25.11          # via selenium
 vine==1.3.0               # via amqp, celery
 webencodings==0.5.1       # via bleach
 werkzeug==1.0.1           # via flask, flask-jwt-extended
 wtforms-json==0.3.3       # via apache-superset
 wtforms==2.3.3            # via flask-wtf, wtforms-json
-yarl==1.5.1               # via aiohttp
-zipp==3.2.0               # via importlib-metadata
+yarl==1.6.2               # via aiohttp
+zipp==3.4.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools