From ffcacc339366afa845e88b6ec8c49096ee553ff1 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Fri, 23 Apr 2021 10:50:49 -0700 Subject: [PATCH] fix: new import/export CLI (#13921) * fix: CLI for import/export * Add tests * Remove debug --- requirements/base.txt | 8 +- requirements/development.txt | 327 ++++++++++++-- requirements/docker.txt | 207 ++++++++- requirements/integration.txt | 74 ++- requirements/local.txt | 265 ++++++++++- requirements/testing.in | 1 + requirements/testing.txt | 421 ++++++++++++++++-- superset/cli.py | 57 +-- .../commands/importers/v1/__init__.py | 2 + .../dashboards/commands/importers/v1/utils.py | 5 +- .../datasets/commands/importers/v1/utils.py | 8 +- tests/cli_tests.py | 208 +++++++++ tests/conftest.py | 6 + 13 files changed, 1450 insertions(+), 139 deletions(-) create mode 100644 tests/cli_tests.py diff --git a/requirements/base.txt b/requirements/base.txt index 3596304d9892..cd7db4d06361 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -122,11 +122,7 @@ idna==2.10 # email-validator # yarl importlib-metadata==2.1.1 -# via -# -r requirements/base.in -# jsonschema -# kombu -# markdown +# via -r requirements/base.in isodate==0.6.0 # via apache-superset itsdangerous==1.1.0 @@ -270,6 +266,7 @@ sqlalchemy==1.3.20 # via # alembic # apache-superset +# flask-appbuilder # flask-sqlalchemy # marshmallow-sqlalchemy # sqlalchemy-utils @@ -279,7 +276,6 @@ typing-extensions==3.7.4.3 # via # aiohttp # apache-superset -# yarl urllib3==1.25.11 # via selenium vine==1.3.0 diff --git a/requirements/development.txt b/requirements/development.txt index f0eb4938a388..cd6eef8693c0 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -6,38 +6,301 @@ # pip-compile-multi # -r base.txt --e file:. # via -r requirements/base.in -boto3==1.16.10 # via tabulator -botocore==1.19.10 # via boto3, s3transfer -cached-property==1.5.2 # via tableschema -certifi==2020.6.20 # via requests -deprecated==1.2.11 # via pygithub -et-xmlfile==1.0.1 # via openpyxl -flask-cors==3.0.9 # via -r requirements/development.in -future==0.18.2 # via pyhive -ijson==3.1.2.post0 # via tabulator -jdcal==1.4.1 # via openpyxl -jmespath==0.10.0 # via boto3, botocore -jsonlines==1.2.0 # via tabulator -linear-tsv==1.1.0 # via tabulator -mysqlclient==1.4.2.post1 # via -r requirements/development.in -openpyxl==3.0.5 # via tabulator -pillow==7.2.0 # via -r requirements/development.in -psycopg2-binary==2.8.5 # via -r requirements/development.in -pydruid==0.6.1 # via -r requirements/development.in -pygithub==1.54.1 # via -r requirements/development.in -pyhive[hive]==0.6.3 # via -r requirements/development.in -requests==2.24.0 # via pydruid, pygithub, tableschema, tabulator -rfc3986==1.4.0 # via tableschema -s3transfer==0.3.3 # via boto3 -sasl==0.2.1 # via pyhive, thrift-sasl -tableschema==1.20.0 # via -r requirements/development.in -tabulator==1.52.5 # via tableschema -thrift-sasl==0.4.2 # via pyhive -thrift==0.13.0 # via -r requirements/development.in, pyhive, thrift-sasl -unicodecsv==0.14.1 # via tableschema, tabulator -wrapt==1.12.1 # via deprecated -xlrd==1.2.0 # via tabulator +-e file:. +# via -r requirements/base.in +# via slackclient +# via flask-migrate +# via kombu +# via flask-appbuilder +# via aiohttp +# via +# aiohttp +# jsonschema +# via flask-babel +# via apache-superset +# via celery +# via apache-superset +boto3==1.16.10 +# via tabulator +botocore==1.19.10 +# via +# boto3 +# s3transfer +# via flask-compress +cached-property==1.5.2 +# via tableschema +# via apache-superset +# via apache-superset +certifi==2020.6.20 +# via requests +# via cryptography +# via +# aiohttp +# requests +# tabulator +# via +# apache-superset +# flask +# flask-appbuilder +# tableschema +# tabulator +# via +# apache-superset +# flask-appbuilder +# via apache-superset +# via holidays +# via apache-superset +# via apache-superset +# via apache-superset +# via retry +# via python3-openid +deprecated==1.2.11 +# via pygithub +# via email-validator +# via flask-appbuilder +et-xmlfile==1.0.1 +# via openpyxl +# via apache-superset +# via flask-appbuilder +# via apache-superset +# via apache-superset +flask-cors==3.0.9 +# via -r requirements/development.in +# via flask-appbuilder +# via flask-appbuilder +# via apache-superset +# via flask-appbuilder +# via +# flask-appbuilder +# flask-migrate +# via apache-superset +# via +# apache-superset +# flask-appbuilder +# via +# apache-superset +# flask-appbuilder +# flask-babel +# flask-caching +# flask-compress +# flask-cors +# flask-jwt-extended +# flask-login +# flask-migrate +# flask-openid +# flask-sqlalchemy +# flask-wtf +future==0.18.2 +# via pyhive +# via geopy +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +# via +# email-validator +# requests +# yarl +ijson==3.1.2.post0 +# via tabulator +# via -r requirements/base.in +# via +# apache-superset +# tableschema +# via +# flask +# flask-wtf +jdcal==1.4.1 +# via openpyxl +# via +# flask +# flask-babel +jmespath==0.10.0 +# via +# boto3 +# botocore +jsonlines==1.2.0 +# via tabulator +# via +# flask-appbuilder +# tableschema +# via celery +# via holidays +linear-tsv==1.1.0 +# via tabulator +# via alembic +# via apache-superset +# via +# jinja2 +# mako +# wtforms +# via flask-appbuilder +# via flask-appbuilder +# via +# flask-appbuilder +# marshmallow-enum +# marshmallow-sqlalchemy +# via apache-superset +# via +# aiohttp +# yarl +mysqlclient==1.4.2.post1 +# via -r requirements/development.in +# via croniter +# via +# pandas +# pyarrow +openpyxl==3.0.5 +# via tabulator +# via bleach +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +pillow==7.2.0 +# via -r requirements/development.in +# via apache-superset +# via flask-appbuilder +psycopg2-binary==2.8.5 +# via -r requirements/development.in +# via retry +# via apache-superset +# via cffi +pydruid==0.6.1 +# via -r requirements/development.in +pygithub==1.54.1 +# via -r requirements/development.in +pyhive[hive]==0.6.3 +# via -r requirements/development.in +# via +# apache-superset +# flask-appbuilder +# flask-jwt-extended +# pygithub +# via convertdate +# via +# apache-superset +# packaging +# via +# -r requirements/base.in +# jsonschema +# via +# alembic +# apache-superset +# botocore +# croniter +# flask-appbuilder +# holidays +# pandas +# pyhive +# tableschema +# via apache-superset +# via alembic +# via apache-superset +# via flask-openid +# via +# babel +# celery +# convertdate +# flask-babel +# pandas +# via +# apache-superset +# apispec +# via apache-superset +requests==2.24.0 +# via +# pydruid +# pygithub +# tableschema +# tabulator +# via apache-superset +rfc3986==1.4.0 +# via tableschema +s3transfer==0.3.3 +# via boto3 +sasl==0.2.1 +# via +# pyhive +# thrift-sasl +# via apache-superset +# via apache-superset +# via +# bleach +# cryptography +# flask-cors +# flask-jwt-extended +# flask-talisman +# holidays +# isodate +# jsonlines +# jsonschema +# linear-tsv +# packaging +# pathlib2 +# polyline +# prison +# pyrsistent +# python-dateutil +# sasl +# sqlalchemy-utils +# tableschema +# tabulator +# thrift +# thrift-sasl +# wtforms-json +# via apache-superset +# via +# apache-superset +# flask-appbuilder +# via +# alembic +# apache-superset +# flask-appbuilder +# flask-sqlalchemy +# marshmallow-sqlalchemy +# sqlalchemy-utils +# tabulator +# via apache-superset +tableschema==1.20.0 +# via -r requirements/development.in +tabulator==1.52.5 +# via tableschema +thrift-sasl==0.4.2 +# via pyhive +thrift==0.13.0 +# via +# -r requirements/development.in +# pyhive +# thrift-sasl +# via +# aiohttp +# apache-superset +unicodecsv==0.14.1 +# via +# tableschema +# tabulator +# via +# botocore +# requests +# selenium +# via +# amqp +# celery +# via bleach +# via +# flask +# flask-jwt-extended +wrapt==1.12.1 +# via deprecated +# via apache-superset +# via +# flask-wtf +# wtforms-json +xlrd==1.2.0 +# via tabulator +# via aiohttp +# via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/docker.txt b/requirements/docker.txt index 3df74655f5b6..c7ef8f684cfb 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -6,12 +6,207 @@ # pip-compile-multi # -r base.txt --e file:. # via -r requirements/base.in -gevent==20.9.0 # via -r requirements/docker.in -greenlet==0.4.17 # via gevent -psycopg2-binary==2.8.6 # via -r requirements/docker.in -zope.event==4.5.0 # via gevent -zope.interface==5.1.2 # via gevent +-e file:. +# via -r requirements/base.in +# via slackclient +# via flask-migrate +# via kombu +# via flask-appbuilder +# via aiohttp +# via +# aiohttp +# jsonschema +# via flask-babel +# via apache-superset +# via celery +# via apache-superset +# via flask-compress +# via apache-superset +# via apache-superset +# via cryptography +# via aiohttp +# via +# apache-superset +# flask +# flask-appbuilder +# via +# apache-superset +# flask-appbuilder +# via apache-superset +# via holidays +# via apache-superset +# via apache-superset +# via apache-superset +# via retry +# via python3-openid +# via email-validator +# via flask-appbuilder +# via apache-superset +# via flask-appbuilder +# via apache-superset +# via apache-superset +# via flask-appbuilder +# via flask-appbuilder +# via apache-superset +# via flask-appbuilder +# via +# flask-appbuilder +# flask-migrate +# via apache-superset +# via +# apache-superset +# flask-appbuilder +# via +# apache-superset +# flask-appbuilder +# flask-babel +# flask-caching +# flask-compress +# flask-jwt-extended +# flask-login +# flask-migrate +# flask-openid +# flask-sqlalchemy +# flask-wtf +# via geopy +# via apache-superset +gevent==20.9.0 +# via -r requirements/docker.in +greenlet==0.4.17 +# via gevent +# via apache-superset +# via apache-superset +# via apache-superset +# via +# email-validator +# yarl +# via -r requirements/base.in +# via apache-superset +# via +# flask +# flask-wtf +# via +# flask +# flask-babel +# via flask-appbuilder +# via celery +# via holidays +# via alembic +# via apache-superset +# via +# jinja2 +# mako +# wtforms +# via flask-appbuilder +# via flask-appbuilder +# via +# flask-appbuilder +# marshmallow-enum +# marshmallow-sqlalchemy +# via apache-superset +# via +# aiohttp +# yarl +# via croniter +# via +# pandas +# pyarrow +# via bleach +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +# via flask-appbuilder +psycopg2-binary==2.8.6 +# via -r requirements/docker.in +# via retry +# via apache-superset +# via cffi +# via +# apache-superset +# flask-appbuilder +# flask-jwt-extended +# via convertdate +# via +# apache-superset +# packaging +# via +# -r requirements/base.in +# jsonschema +# via +# alembic +# apache-superset +# croniter +# flask-appbuilder +# holidays +# pandas +# via apache-superset +# via alembic +# via apache-superset +# via flask-openid +# via +# babel +# celery +# convertdate +# flask-babel +# pandas +# via +# apache-superset +# apispec +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +# via +# bleach +# cryptography +# flask-jwt-extended +# flask-talisman +# holidays +# isodate +# jsonschema +# packaging +# pathlib2 +# polyline +# prison +# pyrsistent +# python-dateutil +# sqlalchemy-utils +# wtforms-json +# via apache-superset +# via +# apache-superset +# flask-appbuilder +# via +# alembic +# apache-superset +# flask-appbuilder +# flask-sqlalchemy +# marshmallow-sqlalchemy +# sqlalchemy-utils +# via apache-superset +# via +# aiohttp +# apache-superset +# via selenium +# via +# amqp +# celery +# via bleach +# via +# flask +# flask-jwt-extended +# via apache-superset +# via +# flask-wtf +# wtforms-json +# via aiohttp +# via importlib-metadata +zope.event==4.5.0 +# via gevent +zope.interface==5.1.2 +# via gevent # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/integration.txt b/requirements/integration.txt index ea7eb06b14cf..fcbde556df30 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -5,28 +5,58 @@ # # pip-compile-multi # -appdirs==1.4.4 # via virtualenv -cfgv==3.2.0 # via pre-commit -click==7.1.2 # via pip-compile-multi, pip-tools -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via tox, virtualenv -identify==1.5.9 # via pre-commit -importlib-metadata==2.1.1 # via pluggy, pre-commit, tox, virtualenv -nodeenv==1.5.0 # via pre-commit -packaging==20.4 # via tox -pip-compile-multi==2.1.0 # via -r requirements/integration.in -pip-tools==5.3.1 # via pip-compile-multi -pluggy==0.13.1 # via tox -pre-commit==2.8.2 # via -r requirements/integration.in -py==1.9.0 # via tox -pyparsing==2.4.7 # via packaging -pyyaml==5.4.1 # via pre-commit -six==1.15.0 # via packaging, pip-tools, tox, virtualenv -toml==0.10.2 # via pre-commit, tox -toposort==1.5 # via pip-compile-multi -tox==3.20.1 # via -r requirements/integration.in -virtualenv==20.1.0 # via pre-commit, tox -zipp==3.4.0 # via importlib-metadata +appdirs==1.4.4 +# via virtualenv +cfgv==3.2.0 +# via pre-commit +click==7.1.2 +# via +# pip-compile-multi +# pip-tools +distlib==0.3.1 +# via virtualenv +filelock==3.0.12 +# via +# tox +# virtualenv +identify==1.5.9 +# via pre-commit +nodeenv==1.5.0 +# via pre-commit +packaging==20.4 +# via tox +pip-compile-multi==2.1.0 +# via -r requirements/integration.in +pip-tools==5.3.1 +# via pip-compile-multi +pluggy==0.13.1 +# via tox +pre-commit==2.8.2 +# via -r requirements/integration.in +py==1.9.0 +# via tox +pyparsing==2.4.7 +# via packaging +pyyaml==5.4.1 +# via pre-commit +six==1.15.0 +# via +# packaging +# pip-tools +# tox +# virtualenv +toml==0.10.2 +# via +# pre-commit +# tox +toposort==1.5 +# via pip-compile-multi +tox==3.20.1 +# via -r requirements/integration.in +virtualenv==20.1.0 +# via +# pre-commit +# tox # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/local.txt b/requirements/local.txt index 8ab0756cd02f..c6a1f1a9dcf7 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -6,7 +6,270 @@ # pip-compile-multi # -r development.txt --e file:. # via -r requirements/base.in +-e file:. +# via -r requirements/base.in +# via slackclient +# via flask-migrate +# via kombu +# via flask-appbuilder +# via aiohttp +# via +# aiohttp +# jsonschema +# via flask-babel +# via apache-superset +# via celery +# via apache-superset +# via tabulator +# via +# boto3 +# s3transfer +# via flask-compress +# via tableschema +# via apache-superset +# via apache-superset +# via requests +# via cryptography +# via +# aiohttp +# requests +# tabulator +# via +# apache-superset +# flask +# flask-appbuilder +# tableschema +# tabulator +# via +# apache-superset +# flask-appbuilder +# via apache-superset +# via holidays +# via apache-superset +# via apache-superset +# via apache-superset +# via retry +# via python3-openid +# via pygithub +# via email-validator +# via flask-appbuilder +# via openpyxl +# via apache-superset +# via flask-appbuilder +# via apache-superset +# via apache-superset +# via -r requirements/development.in +# via flask-appbuilder +# via flask-appbuilder +# via apache-superset +# via flask-appbuilder +# via +# flask-appbuilder +# flask-migrate +# via apache-superset +# via +# apache-superset +# flask-appbuilder +# via +# apache-superset +# flask-appbuilder +# flask-babel +# flask-caching +# flask-compress +# flask-cors +# flask-jwt-extended +# flask-login +# flask-migrate +# flask-openid +# flask-sqlalchemy +# flask-wtf +# via pyhive +# via geopy +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +# via +# email-validator +# requests +# yarl +# via tabulator +# via -r requirements/base.in +# via +# apache-superset +# tableschema +# via +# flask +# flask-wtf +# via openpyxl +# via +# flask +# flask-babel +# via +# boto3 +# botocore +# via tabulator +# via +# flask-appbuilder +# tableschema +# via celery +# via holidays +# via tabulator +# via alembic +# via apache-superset +# via +# jinja2 +# mako +# wtforms +# via flask-appbuilder +# via flask-appbuilder +# via +# flask-appbuilder +# marshmallow-enum +# marshmallow-sqlalchemy +# via apache-superset +# via +# aiohttp +# yarl +# via -r requirements/development.in +# via croniter +# via +# pandas +# pyarrow +# via tabulator +# via bleach +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +# via -r requirements/development.in +# via apache-superset +# via flask-appbuilder +# via -r requirements/development.in +# via retry +# via apache-superset +# via cffi +# via -r requirements/development.in +# via -r requirements/development.in +# via -r requirements/development.in +# via +# apache-superset +# flask-appbuilder +# flask-jwt-extended +# pygithub +# via convertdate +# via +# apache-superset +# packaging +# via +# -r requirements/base.in +# jsonschema +# via +# alembic +# apache-superset +# botocore +# croniter +# flask-appbuilder +# holidays +# pandas +# pyhive +# tableschema +# via apache-superset +# via alembic +# via apache-superset +# via flask-openid +# via +# babel +# celery +# convertdate +# flask-babel +# pandas +# via +# apache-superset +# apispec +# via apache-superset +# via +# pydruid +# pygithub +# tableschema +# tabulator +# via apache-superset +# via tableschema +# via boto3 +# via +# pyhive +# thrift-sasl +# via apache-superset +# via apache-superset +# via +# bleach +# cryptography +# flask-cors +# flask-jwt-extended +# flask-talisman +# holidays +# isodate +# jsonlines +# jsonschema +# linear-tsv +# packaging +# pathlib2 +# polyline +# prison +# pyrsistent +# python-dateutil +# sasl +# sqlalchemy-utils +# tableschema +# tabulator +# thrift +# thrift-sasl +# wtforms-json +# via apache-superset +# via +# apache-superset +# flask-appbuilder +# via +# alembic +# apache-superset +# flask-appbuilder +# flask-sqlalchemy +# marshmallow-sqlalchemy +# sqlalchemy-utils +# tabulator +# via apache-superset +# via -r requirements/development.in +# via tableschema +# via pyhive +# via +# -r requirements/development.in +# pyhive +# thrift-sasl +# via +# aiohttp +# apache-superset +# via +# tableschema +# tabulator +# via +# botocore +# requests +# selenium +# via +# amqp +# celery +# via bleach +# via +# flask +# flask-jwt-extended +# via deprecated +# via apache-superset +# via +# flask-wtf +# wtforms-json +# via tabulator +# via aiohttp +# via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/testing.in b/requirements/testing.in index 6bb9b6eb49fc..e95cab68f04c 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -26,6 +26,7 @@ ipython==7.16.1 openapi-spec-validator openpyxl parameterized +pyfakefs pyhive[presto]>=0.6.3 pylint pytest diff --git a/requirements/testing.txt b/requirements/testing.txt index 8ea641b1dfa4..a08f4c503ec7 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,4 +1,4 @@ -# SHA1:9d449781bc4ef88cd346b9dd5db55240472d5f0c +# SHA1:1b285a0aa0e721283892b052553751d44f5dd81f # # This file is autogenerated by pip-compile-multi # To update, run: @@ -7,39 +7,392 @@ # -r development.txt -r integration.txt --e file:. # via -r requirements/base.in -appnope==0.1.0 # via ipython -astroid==2.4.2 # via pylint -backcall==0.2.0 # via ipython -coverage==5.3 # via pytest-cov -docker==4.3.1 # via -r requirements/testing.in -flask-testing==0.8.0 # via -r requirements/testing.in -freezegun==1.0.0 # via -r requirements/testing.in -iniconfig==1.1.1 # via pytest -ipdb==0.13.4 # via -r requirements/testing.in -ipython-genutils==0.2.0 # via traitlets -ipython==7.16.1 # via -r requirements/testing.in, ipdb -isort==5.6.4 # via pylint -jedi==0.17.2 # via ipython -lazy-object-proxy==1.4.3 # via astroid -mccabe==0.6.1 # via pylint -openapi-spec-validator==0.2.9 # via -r requirements/testing.in -parameterized==0.7.4 # via -r requirements/testing.in -parso==0.7.1 # via jedi -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -prompt-toolkit==3.0.8 # via ipython -ptyprocess==0.6.0 # via pexpect -pygments==2.7.2 # via ipython -pyhive[hive,presto]==0.6.3 # via -r requirements/development.in, -r requirements/testing.in -pylint==2.6.0 # via -r requirements/testing.in -pytest-cov==2.10.1 # via -r requirements/testing.in -pytest==6.1.2 # via -r requirements/testing.in, pytest-cov -statsd==3.3.0 # via -r requirements/testing.in -traitlets==5.0.5 # via ipython -typed-ast==1.4.1 # via astroid -wcwidth==0.2.5 # via prompt-toolkit -websocket-client==0.57.0 # via docker +-e file:. +# via -r requirements/base.in +# via slackclient +# via flask-migrate +# via kombu +# via flask-appbuilder +# via virtualenv +appnope==0.1.0 +# via ipython +astroid==2.4.2 +# via pylint +# via aiohttp +# via +# aiohttp +# jsonschema +# pytest +# via flask-babel +backcall==0.2.0 +# via ipython +# via apache-superset +# via celery +# via apache-superset +# via tabulator +# via +# boto3 +# s3transfer +# via flask-compress +# via tableschema +# via apache-superset +# via apache-superset +# via requests +# via cryptography +# via pre-commit +# via +# aiohttp +# requests +# tabulator +# via +# apache-superset +# flask +# flask-appbuilder +# pip-compile-multi +# pip-tools +# tableschema +# tabulator +# via +# apache-superset +# flask-appbuilder +# via apache-superset +# via holidays +coverage==5.3 +# via pytest-cov +# via apache-superset +# via apache-superset +# via apache-superset +# via +# ipython +# retry +# via python3-openid +# via pygithub +# via virtualenv +# via email-validator +docker==4.3.1 +# via -r requirements/testing.in +# via flask-appbuilder +# via openpyxl +# via +# tox +# virtualenv +# via apache-superset +# via flask-appbuilder +# via apache-superset +# via apache-superset +# via -r requirements/development.in +# via flask-appbuilder +# via flask-appbuilder +# via apache-superset +# via flask-appbuilder +# via +# flask-appbuilder +# flask-migrate +# via apache-superset +flask-testing==0.8.0 +# via -r requirements/testing.in +# via +# apache-superset +# flask-appbuilder +# via +# apache-superset +# flask-appbuilder +# flask-babel +# flask-caching +# flask-compress +# flask-cors +# flask-jwt-extended +# flask-login +# flask-migrate +# flask-openid +# flask-sqlalchemy +# flask-testing +# flask-wtf +freezegun==1.0.0 +# via -r requirements/testing.in +# via pyhive +# via geopy +# via apache-superset +# via apache-superset +# via apache-superset +# via apache-superset +# via pre-commit +# via +# email-validator +# requests +# yarl +# via tabulator +# via -r requirements/base.in +iniconfig==1.1.1 +# via pytest +ipdb==0.13.4 +# via -r requirements/testing.in +ipython-genutils==0.2.0 +# via traitlets +ipython==7.16.1 +# via +# -r requirements/testing.in +# ipdb +# via +# apache-superset +# tableschema +isort==5.6.4 +# via pylint +# via +# flask +# flask-wtf +# via openpyxl +jedi==0.17.2 +# via ipython +# via +# flask +# flask-babel +# via +# boto3 +# botocore +# via tabulator +# via +# flask-appbuilder +# openapi-spec-validator +# tableschema +# via celery +# via holidays +lazy-object-proxy==1.4.3 +# via astroid +# via tabulator +# via alembic +# via apache-superset +# via +# jinja2 +# mako +# wtforms +# via flask-appbuilder +# via flask-appbuilder +# via +# flask-appbuilder +# marshmallow-enum +# marshmallow-sqlalchemy +mccabe==0.6.1 +# via pylint +# via apache-superset +# via +# aiohttp +# yarl +# via -r requirements/development.in +# via croniter +# via pre-commit +# via +# pandas +# pyarrow +openapi-spec-validator==0.2.9 +# via -r requirements/testing.in +# via +# -r requirements/testing.in +# tabulator +# via +# bleach +# pytest +# tox +# via apache-superset +parameterized==0.7.4 +# via -r requirements/testing.in +# via apache-superset +parso==0.7.1 +# via jedi +# via apache-superset +pexpect==4.8.0 +# via ipython +# via apache-superset +pickleshare==0.7.5 +# via ipython +# via -r requirements/development.in +# via -r requirements/integration.in +# via pip-compile-multi +# via +# pytest +# tox +# via apache-superset +# via -r requirements/integration.in +# via flask-appbuilder +prompt-toolkit==3.0.8 +# via ipython +# via -r requirements/development.in +ptyprocess==0.6.0 +# via pexpect +# via +# pytest +# retry +# tox +# via apache-superset +# via cffi +# via -r requirements/development.in +pyfakefs==4.4.0 +# via -r requirements/testing.in +# via -r requirements/development.in +pygments==2.7.2 +# via ipython +pyhive[hive,presto]==0.6.3 +# via +# -r requirements/development.in +# -r requirements/testing.in +# via +# apache-superset +# flask-appbuilder +# flask-jwt-extended +# pygithub +pylint==2.6.0 +# via -r requirements/testing.in +# via convertdate +# via +# apache-superset +# packaging +# via +# -r requirements/base.in +# jsonschema +pytest-cov==2.10.1 +# via -r requirements/testing.in +pytest==6.1.2 +# via +# -r requirements/testing.in +# pytest-cov +# via +# alembic +# apache-superset +# botocore +# croniter +# flask-appbuilder +# freezegun +# holidays +# pandas +# pyhive +# tableschema +# via apache-superset +# via alembic +# via apache-superset +# via flask-openid +# via +# babel +# celery +# convertdate +# flask-babel +# pandas +# via +# apache-superset +# apispec +# openapi-spec-validator +# pre-commit +# via apache-superset +# via +# docker +# pydruid +# pygithub +# pyhive +# tableschema +# tabulator +# via apache-superset +# via tableschema +# via boto3 +# via +# pyhive +# thrift-sasl +# via apache-superset +# via apache-superset +# via +# astroid +# bleach +# cryptography +# docker +# flask-cors +# flask-jwt-extended +# flask-talisman +# holidays +# isodate +# jsonlines +# jsonschema +# linear-tsv +# openapi-spec-validator +# packaging +# pathlib2 +# pip-tools +# polyline +# prison +# pyrsistent +# python-dateutil +# sasl +# sqlalchemy-utils +# tableschema +# tabulator +# thrift +# thrift-sasl +# tox +# virtualenv +# websocket-client +# wtforms-json +# via apache-superset +# via +# apache-superset +# flask-appbuilder +# via +# alembic +# apache-superset +# flask-appbuilder +# flask-sqlalchemy +# marshmallow-sqlalchemy +# sqlalchemy-utils +# tabulator +# via apache-superset +statsd==3.3.0 +# via -r requirements/testing.in +# via -r requirements/development.in +# via tableschema +# via pyhive +# via +# -r requirements/development.in +# pyhive +# thrift-sasl +# via +# pre-commit +# pylint +# pytest +# tox +# via pip-compile-multi +# via -r requirements/integration.in +traitlets==5.0.5 +# via ipython +# via +# aiohttp +# apache-superset +# via +# tableschema +# tabulator +# via +# botocore +# requests +# selenium +# via +# amqp +# celery +# via +# pre-commit +# tox +wcwidth==0.2.5 +# via prompt-toolkit +# via bleach +websocket-client==0.57.0 +# via docker +# via +# flask +# flask-jwt-extended +# via +# astroid +# deprecated +# via apache-superset +# via +# flask-wtf +# wtforms-json +# via tabulator +# via aiohttp +# via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/superset/cli.py b/superset/cli.py index c0cd98482509..83ab5260a949 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -21,7 +21,7 @@ from datetime import datetime, timedelta from subprocess import Popen from typing import Any, Dict, List, Optional, Type, Union -from zipfile import ZipFile +from zipfile import is_zipfile, ZipFile import click import yaml @@ -250,12 +250,9 @@ def refresh_druid(datasource: str, merge: bool) -> None: @superset.command() @with_appcontext @click.option( - "--dashboard-file", - "-f", - default="dashboard_export_YYYYMMDDTHHMMSS", - help="Specify the the file to export to", + "--dashboard-file", "-f", help="Specify the the file to export to", ) - def export_dashboards(dashboard_file: Optional[str]) -> None: + def export_dashboards(dashboard_file: Optional[str] = None) -> None: """Export dashboards to ZIP file""" from superset.dashboards.commands.export import ExportDashboardsCommand from superset.models.dashboard import Dashboard @@ -284,12 +281,9 @@ def export_dashboards(dashboard_file: Optional[str]) -> None: @superset.command() @with_appcontext @click.option( - "--datasource-file", - "-f", - default="dataset_export_YYYYMMDDTHHMMSS", - help="Specify the the file to export to", + "--datasource-file", "-f", help="Specify the the file to export to", ) - def export_datasources(datasource_file: Optional[str]) -> None: + def export_datasources(datasource_file: Optional[str] = None) -> None: """Export datasources to ZIP file""" from superset.connectors.sqla.models import SqlaTable from superset.datasets.commands.export import ExportDatasetsCommand @@ -325,15 +319,20 @@ def export_datasources(datasource_file: Optional[str]) -> None: ) def import_dashboards(path: str, username: Optional[str]) -> None: """Import dashboards from ZIP file""" + from superset.commands.importers.v1.utils import get_contents_from_bundle from superset.dashboards.commands.importers.dispatcher import ( ImportDashboardsCommand, ) if username is not None: g.user = security_manager.find_user(username=username) - contents = {path: open(path).read()} + if is_zipfile(path): + with ZipFile(path) as bundle: + contents = get_contents_from_bundle(bundle) + else: + contents = {path: open(path).read()} try: - ImportDashboardsCommand(contents).run() + ImportDashboardsCommand(contents, overwrite=True).run() except Exception: # pylint: disable=broad-except logger.exception( "There was an error when importing the dashboards(s), please check " @@ -343,36 +342,22 @@ def import_dashboards(path: str, username: Optional[str]) -> None: @superset.command() @with_appcontext @click.option( - "--path", - "-p", - help="Path to a single YAML file or path containing multiple YAML " - "files to import (*.yaml or *.yml)", - ) - @click.option( - "--sync", - "-s", - "sync", - default="", - help="comma seperated list of element types to synchronize " - 'e.g. "metrics,columns" deletes metrics and columns in the DB ' - "that are not specified in the YAML file", - ) - @click.option( - "--recursive", - "-r", - is_flag=True, - default=False, - help="recursively search the path for yaml files", + "--path", "-p", help="Path to a single ZIP file", ) def import_datasources(path: str) -> None: """Import datasources from ZIP file""" + from superset.commands.importers.v1.utils import get_contents_from_bundle from superset.datasets.commands.importers.dispatcher import ( ImportDatasetsCommand, ) - contents = {path: open(path).read()} + if is_zipfile(path): + with ZipFile(path) as bundle: + contents = get_contents_from_bundle(bundle) + else: + contents = {path: open(path).read()} try: - ImportDatasetsCommand(contents).run() + ImportDatasetsCommand(contents, overwrite=True).run() except Exception: # pylint: disable=broad-except logger.exception( "There was an error when importing the dataset(s), please check the " @@ -482,7 +467,7 @@ def export_datasources( help="Specify the user name to assign dashboards to", ) def import_dashboards(path: str, recursive: bool, username: str) -> None: - """Import dashboards from ZIP file""" + """Import dashboards from JSON file""" from superset.dashboards.commands.importers.v0 import ImportDashboardsCommand path_object = Path(path) diff --git a/superset/dashboards/commands/importers/v1/__init__.py b/superset/dashboards/commands/importers/v1/__init__.py index 6bfb1c8bd27b..c698e89af66e 100644 --- a/superset/dashboards/commands/importers/v1/__init__.py +++ b/superset/dashboards/commands/importers/v1/__init__.py @@ -124,6 +124,8 @@ def _import( config = update_id_refs(config, chart_ids) dashboard = import_dashboard(session, config, overwrite=overwrite) for uuid in find_chart_uuids(config["position"]): + if uuid not in chart_ids: + break chart_id = chart_ids[uuid] if (dashboard.id, chart_id) not in existing_relationships: dashboard_chart_ids.append((dashboard.id, chart_id)) diff --git a/superset/dashboards/commands/importers/v1/utils.py b/superset/dashboards/commands/importers/v1/utils.py index 09d786ce3839..7b2ad91fbf32 100644 --- a/superset/dashboards/commands/importers/v1/utils.py +++ b/superset/dashboards/commands/importers/v1/utils.py @@ -51,7 +51,9 @@ def update_id_refs(config: Dict[str, Any], chart_ids: Dict[str, int]) -> Dict[st # build map old_id => new_id old_ids = build_uuid_to_id_map(fixed["position"]) - id_map = {old_id: chart_ids[uuid] for uuid, old_id in old_ids.items()} + id_map = { + old_id: chart_ids[uuid] for uuid, old_id in old_ids.items() if uuid in chart_ids + } # fix metadata metadata = fixed.get("metadata", {}) @@ -97,6 +99,7 @@ def update_id_refs(config: Dict[str, Any], chart_ids: Dict[str, int]) -> Dict[st isinstance(child, dict) and child["type"] == "CHART" and "uuid" in child["meta"] + and child["meta"]["uuid"] in chart_ids ): child["meta"]["chartId"] = chart_ids[child["meta"]["uuid"]] diff --git a/superset/datasets/commands/importers/v1/utils.py b/superset/datasets/commands/importers/v1/utils.py index 882faf2ce5e4..48e1f797fdde 100644 --- a/superset/datasets/commands/importers/v1/utils.py +++ b/superset/datasets/commands/importers/v1/utils.py @@ -116,7 +116,13 @@ def import_dataset( session.flush() example_database = get_example_database() - table_exists = example_database.has_table_by_name(dataset.table_name) + try: + table_exists = example_database.has_table_by_name(dataset.table_name) + except Exception as ex: + # MySQL doesn't play nice with GSheets table names + logger.warning("Couldn't check if table %s exists, stopping import") + raise ex + if data_uri and (not table_exists or force_data): load_data(data_uri, dataset, example_database, session) diff --git a/tests/cli_tests.py b/tests/cli_tests.py new file mode 100644 index 000000000000..7ea4b9350bb3 --- /dev/null +++ b/tests/cli_tests.py @@ -0,0 +1,208 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib +import json +from pathlib import Path +from unittest import mock +from zipfile import is_zipfile, ZipFile + +import pytest +import yaml +from freezegun import freeze_time + +import superset.cli +from superset import app +from tests.fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices + + +@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") +def test_export_dashboards_original(app_context, fs): + """ + Test that a JSON file is exported. + """ + # pylint: disable=reimported, redefined-outer-name + import superset.cli # noqa: F811 + + # reload to define export_dashboards correctly based on the + # feature flags + importlib.reload(superset.cli) + + runner = app.test_cli_runner() + response = runner.invoke(superset.cli.export_dashboards, ("-f", "dashboards.json")) + + assert response.exit_code == 0 + assert Path("dashboards.json").exists() + + # check that file is valid JSON + with open("dashboards.json") as fp: + contents = fp.read() + json.loads(contents) + + +@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") +def test_export_datasources_original(app_context, fs): + """ + Test that a YAML file is exported. + """ + # pylint: disable=reimported, redefined-outer-name + import superset.cli # noqa: F811 + + # reload to define export_dashboards correctly based on the + # feature flags + importlib.reload(superset.cli) + + runner = app.test_cli_runner() + response = runner.invoke( + superset.cli.export_datasources, ("-f", "datasources.yaml") + ) + + assert response.exit_code == 0 + assert Path("datasources.yaml").exists() + + # check that file is valid JSON + with open("datasources.yaml") as fp: + contents = fp.read() + yaml.safe_load(contents) + + +@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") +@mock.patch.dict( + "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True +) +def test_export_dashboards_versioned_export(app_context, fs): + """ + Test that a ZIP file is exported. + """ + # pylint: disable=reimported, redefined-outer-name + import superset.cli # noqa: F811 + + # reload to define export_dashboards correctly based on the + # feature flags + importlib.reload(superset.cli) + + runner = app.test_cli_runner() + with freeze_time("2021-01-01T00:00:00Z"): + response = runner.invoke(superset.cli.export_dashboards, ()) + + assert response.exit_code == 0 + assert Path("dashboard_export_20210101T000000.zip").exists() + + assert is_zipfile("dashboard_export_20210101T000000.zip") + + +@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") +@mock.patch.dict( + "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True +) +def test_export_datasources_versioned_export(app_context, fs): + """ + Test that a ZIP file is exported. + """ + # pylint: disable=reimported, redefined-outer-name + import superset.cli # noqa: F811 + + # reload to define export_dashboards correctly based on the + # feature flags + importlib.reload(superset.cli) + + runner = app.test_cli_runner() + with freeze_time("2021-01-01T00:00:00Z"): + response = runner.invoke(superset.cli.export_datasources, ()) + + assert response.exit_code == 0 + assert Path("dataset_export_20210101T000000.zip").exists() + + assert is_zipfile("dataset_export_20210101T000000.zip") + + +@mock.patch.dict( + "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True +) +@mock.patch("superset.dashboards.commands.importers.dispatcher.ImportDashboardsCommand") +def test_import_dashboards_versioned_export(import_dashboards_command, app_context, fs): + """ + Test that both ZIP and JSON can be imported. + """ + # pylint: disable=reimported, redefined-outer-name + import superset.cli # noqa: F811 + + # reload to define export_dashboards correctly based on the + # feature flags + importlib.reload(superset.cli) + + # write JSON file + with open("dashboards.json", "w") as fp: + fp.write('{"hello": "world"}') + + runner = app.test_cli_runner() + response = runner.invoke(superset.cli.import_dashboards, ("-p", "dashboards.json")) + + assert response.exit_code == 0 + expected_contents = {"dashboards.json": '{"hello": "world"}'} + import_dashboards_command.assert_called_with(expected_contents, overwrite=True) + + # write ZIP file + with ZipFile("dashboards.zip", "w") as bundle: + with bundle.open("dashboards/dashboard.yaml", "w") as fp: + fp.write(b"hello: world") + + runner = app.test_cli_runner() + response = runner.invoke(superset.cli.import_dashboards, ("-p", "dashboards.zip")) + + assert response.exit_code == 0 + expected_contents = {"dashboard.yaml": "hello: world"} + import_dashboards_command.assert_called_with(expected_contents, overwrite=True) + + +@mock.patch.dict( + "superset.config.DEFAULT_FEATURE_FLAGS", {"VERSIONED_EXPORT": True}, clear=True +) +@mock.patch("superset.datasets.commands.importers.dispatcher.ImportDatasetsCommand") +def test_import_datasets_versioned_export(import_datasets_command, app_context, fs): + """ + Test that both ZIP and YAML can be imported. + """ + # pylint: disable=reimported, redefined-outer-name + import superset.cli # noqa: F811 + + # reload to define export_datasets correctly based on the + # feature flags + importlib.reload(superset.cli) + + # write YAML file + with open("datasets.yaml", "w") as fp: + fp.write("hello: world") + + runner = app.test_cli_runner() + response = runner.invoke(superset.cli.import_datasources, ("-p", "datasets.yaml")) + + assert response.exit_code == 0 + expected_contents = {"datasets.yaml": "hello: world"} + import_datasets_command.assert_called_with(expected_contents, overwrite=True) + + # write ZIP file + with ZipFile("datasets.zip", "w") as bundle: + with bundle.open("datasets/dataset.yaml", "w") as fp: + fp.write(b"hello: world") + + runner = app.test_cli_runner() + response = runner.invoke(superset.cli.import_datasources, ("-p", "datasets.zip")) + + assert response.exit_code == 0 + expected_contents = {"dataset.yaml": "hello: world"} + import_datasets_command.assert_called_with(expected_contents, overwrite=True) diff --git a/tests/conftest.py b/tests/conftest.py index 3bd5170d4943..b04a76c9062d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,12 @@ ADMIN_SCHEMA_NAME = "admin_database" +@pytest.fixture +def app_context(): + with app.app_context(): + yield + + @pytest.fixture(autouse=True, scope="session") def setup_sample_data() -> Any: with app.app_context():