Apply V10 beta period feedback / fixes (getredash#5611)

The following PR's were cherry-picked: * Excel & CSV query runner (getredash#2478) * Pin python3 image version (getredash#5570) * Fix: Edit Source button disappeared for users without CanEdit perms (getredash#5568) * Fix: Specify the protobuf version (getredash#5608) Plus one additional change exclusive to this branch: * Replace reference to yarn with NPM This happened because we cherry-picked getredash#5570 but did not also incorporate getredash#5541 into V10. Co-authored-by: deecay <deecay@users.noreply.github.com> Co-authored-by: Levko Kravets <levko.ne@gmail.com> Co-authored-by: zoomdot <gninggoon@gmail.com>
denisov-vlad · Oct 1, 2021 · f312adf · f312adf
1 parent 92e5d78
commit f312adf
Show file tree

Hide file tree

Showing 7 changed files with 205 additions and 3 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -22,7 +22,8 @@ RUN if [ "x$skip_frontend_build" = "x" ] ; then npm ci --unsafe-perm; fi
 COPY --chown=redash client /frontend/client
 COPY --chown=redash webpack.config.js /frontend/
 RUN if [ "x$skip_frontend_build" = "x" ] ; then npm run build; else mkdir -p /frontend/client/dist && touch /frontend/client/dist/multi_org.html && touch /frontend/client/dist/index.html; fi
-FROM python:3.7-slim
+
+FROM python:3.7-slim-buster
 
 EXPOSE 5000
 

diff --git a/client/app/assets/images/db-logos/excel.png b/client/app/assets/images/db-logos/excel.png
diff --git a/client/app/pages/queries/components/QueryPageHeader.jsx b/client/app/pages/queries/components/QueryPageHeader.jsx
@@ -179,7 +179,7 @@ export default function QueryPageHeader({
 
         {!queryFlags.isNew && queryFlags.canViewSource && (
           <span>
-            {!sourceMode && queryFlags.canEdit && (
+            {!sourceMode && (
               <Link.Button className="m-r-5" href={query.getUrl(true, selectedVisualization)}>
                 <i className="fa fa-pencil-square-o" aria-hidden="true" />
                 <span className="m-l-5">Edit Source</span>

diff --git a/redash/query_runner/csv.py b/redash/query_runner/csv.py
@@ -0,0 +1,100 @@
+import logging
+import yaml
+import requests
+import io
+
+from redash import settings
+from redash.query_runner import *
+from redash.utils import json_dumps
+
+logger = logging.getLogger(__name__)
+
+try:
+    import pandas as pd
+    import numpy as np
+    enabled = True
+except ImportError:
+    enabled = False
+
+
+class CSV(BaseQueryRunner):
+    should_annotate_query = False
+
+    @classmethod
+    def name(cls):
+        return "CSV"
+
+    @classmethod
+    def enabled(cls):
+        return enabled
+
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            'type': 'object',
+            'properties': {},
+        }
+
+    def __init__(self, configuration):
+        super(CSV, self).__init__(configuration)
+        self.syntax = "yaml"
+
+    def test_connection(self):
+        pass
+
+    def run_query(self, query, user):
+        path = ""
+        ua = ""
+        args = {}
+        try:
+            args = yaml.safe_load(query)
+            path = args['url']
+            args.pop('url', None)
+            ua = args['user-agent']
+            args.pop('user-agent', None)
+
+            if is_private_address(path) and settings.ENFORCE_PRIVATE_ADDRESS_BLOCK:
+                raise Exception("Can't query private addresses.")
+        except:
+            pass
+
+        try:
+            response = requests.get(url=path, headers={"User-agent": ua})
+            workbook = pd.read_csv(io.BytesIO(response.content),sep=",", **args)
+
+            df = workbook.copy()
+            data = {'columns': [], 'rows': []}
+            conversions = [
+                {'pandas_type': np.integer, 'redash_type': 'integer',},
+                {'pandas_type': np.inexact, 'redash_type': 'float',},
+                {'pandas_type': np.datetime64, 'redash_type': 'datetime', 'to_redash': lambda x: x.strftime('%Y-%m-%d %H:%M:%S')},
+                {'pandas_type': np.bool_, 'redash_type': 'boolean'},
+                {'pandas_type': np.object, 'redash_type': 'string'}
+            ]
+            labels = []
+            for dtype, label in zip(df.dtypes, df.columns):
+                for conversion in conversions:
+                    if issubclass(dtype.type, conversion['pandas_type']):
+                        data['columns'].append({'name': label, 'friendly_name': label, 'type': conversion['redash_type']})
+                        labels.append(label)
+                        func = conversion.get('to_redash')
+                        if func:
+                            df[label] = df[label].apply(func)
+                        break
+            data['rows'] = df[labels].replace({np.nan: None}).to_dict(orient='records')
+
+            json_data = json_dumps(data)
+            error = None
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            error = "Error reading {0}. {1}".format(path, str(e))
+            json_data = None
+
+        return json_data, error
+
+    def get_schema(self):
+        raise NotSupported()
+
+register(CSV)
diff --git a/redash/query_runner/excel.py b/redash/query_runner/excel.py
@@ -0,0 +1,96 @@
+import logging
+import yaml
+import requests
+
+from redash import settings
+from redash.query_runner import *
+from redash.utils import json_dumps
+
+logger = logging.getLogger(__name__)
+
+try:
+    import pandas as pd
+    import xlrd
+    import openpyxl
+    import numpy as np
+    enabled = True
+except ImportError:
+    enabled = False
+
+class Excel(BaseQueryRunner):
+    should_annotate_query = False
+
+    @classmethod
+    def enabled(cls):
+        return enabled
+
+    @classmethod
+    def configuration_schema(cls):
+        return {
+            'type': 'object',
+            'properties': {},
+        }
+
+    def __init__(self, configuration):
+        super(Excel, self).__init__(configuration)
+        self.syntax = "yaml"
+
+    def test_connection(self):
+        pass
+
+    def run_query(self, query, user):
+        path = ""
+        ua = ""
+        args = {}
+        try:
+            args = yaml.safe_load(query)
+            path = args['url']
+            args.pop('url', None)
+            ua = args['user-agent']
+            args.pop('user-agent', None)
+
+            if is_private_address(path) and settings.ENFORCE_PRIVATE_ADDRESS_BLOCK:
+                raise Exception("Can't query private addresses.")
+        except:
+            pass
+
+        try:
+            response = requests.get(url=path, headers={"User-agent": ua})
+            workbook = pd.read_excel(response.content, **args)
+
+            df = workbook.copy()
+            data = {'columns': [], 'rows': []}
+            conversions = [
+                {'pandas_type': np.integer, 'redash_type': 'integer',},
+                {'pandas_type': np.inexact, 'redash_type': 'float',},
+                {'pandas_type': np.datetime64, 'redash_type': 'datetime', 'to_redash': lambda x: x.strftime('%Y-%m-%d %H:%M:%S')},
+                {'pandas_type': np.bool_, 'redash_type': 'boolean'},
+                {'pandas_type': np.object, 'redash_type': 'string'}
+            ]
+            labels = []
+            for dtype, label in zip(df.dtypes, df.columns):
+                for conversion in conversions:
+                    if issubclass(dtype.type, conversion['pandas_type']):
+                        data['columns'].append({'name': label, 'friendly_name': label, 'type': conversion['redash_type']})
+                        labels.append(label)
+                        func = conversion.get('to_redash')
+                        if func:
+                            df[label] = df[label].apply(func)
+                        break
+            data['rows'] = df[labels].replace({np.nan: None}).to_dict(orient='records')
+
+            json_data = json_dumps(data)
+            error = None
+        except KeyboardInterrupt:
+            error = "Query cancelled by user."
+            json_data = None
+        except Exception as e:
+            error = "Error reading {0}. {1}".format(path, str(e))
+            json_data = None
+
+        return json_data, error
+
+    def get_schema(self):
+        raise NotSupported()
+
+register(Excel)
diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py
@@ -380,7 +380,9 @@ def email_server_is_configured():
     "redash.query_runner.cloudwatch",
     "redash.query_runner.cloudwatch_insights",
     "redash.query_runner.corporate_memory",
-    "redash.query_runner.sparql_endpoint"
+    "redash.query_runner.sparql_endpoint",
+    "redash.query_runner.excel",
+    "redash.query_runner.csv"
 ]
 
 enabled_query_runners = array_from_string(

diff --git a/requirements_all_ds.txt b/requirements_all_ds.txt
@@ -1,4 +1,5 @@
 google-api-python-client==1.7.11
+protobuf==3.17.3
 gspread==3.1.0
 impyla==0.16.0
 influxdb==5.2.3
@@ -37,3 +38,5 @@ python-rapidjson==0.8.0
 pyodbc==4.0.28
 trino~=0.305
 cmem-cmempy==21.2.3
+xlrd==2.0.1
+openpyxl==3.0.7