From accca51f39a69ac822e6335c05c8a02541dbd666 Mon Sep 17 00:00:00 2001 From: Arik Fraimovich Date: Wed, 18 Mar 2015 10:10:42 +0200 Subject: [PATCH 1/3] Feature: web interface to edit datasources * Web interface to add and delete data sources, without the need to ssh into the server. * Ability to safely delete datasources -- query results from this data sources are deleted, while queries get assigned null datasource. * Updated the BigQuery datasource to use the JSON key file from Google Developer console. Also both BigQuery and the Google Spreadsheets datasource no longer store their key on the filesystem, but rather in the DB. * Minor updates to the Flask Admin. --- bin/test_multithreading.py | 63 --------------- migrations/0010_allow_deleting_datasources.py | 18 +++++ migrations/0011_migrate_bigquery_to_json.py | 44 +++++++++++ rd_ui/app/index.html | 8 +- rd_ui/app/scripts/app.js | 12 ++- .../scripts/controllers/admin_controllers.js | 2 +- rd_ui/app/scripts/controllers/data_sources.js | 47 ++++++++++++ rd_ui/app/scripts/controllers/query_view.js | 16 ++-- .../directives/data_source_directives.js | 76 +++++++++++++++++++ rd_ui/app/scripts/services/resources.js | 47 ++++++++++-- .../scripts/ui-bootstrap-tpls-0.5.0.min.js | 2 - rd_ui/app/styles/redash.css | 14 ++++ rd_ui/app/views/data_sources/edit.html | 11 +++ rd_ui/app/views/data_sources/form.html | 20 +++++ rd_ui/app/views/data_sources/list.html | 18 +++++ rd_ui/app/views/query.html | 2 +- rd_ui/bower.json | 3 +- rd_ui/package.json | 3 +- redash/admin.py | 27 +++---- redash/controllers.py | 38 +++++++++- redash/models.py | 19 ++++- redash/query_runner/__init__.py | 6 +- redash/query_runner/big_query.py | 18 ++--- redash/query_runner/google_spreadsheets.py | 15 ++-- tests/test_controllers.py | 2 +- 25 files changed, 406 insertions(+), 125 deletions(-) delete mode 100644 bin/test_multithreading.py create mode 100644 migrations/0010_allow_deleting_datasources.py create mode 100644 migrations/0011_migrate_bigquery_to_json.py create mode 100644 rd_ui/app/scripts/controllers/data_sources.js create mode 100644 rd_ui/app/scripts/directives/data_source_directives.js delete mode 100644 rd_ui/app/scripts/ui-bootstrap-tpls-0.5.0.min.js create mode 100644 rd_ui/app/views/data_sources/edit.html create mode 100644 rd_ui/app/views/data_sources/form.html create mode 100644 rd_ui/app/views/data_sources/list.html diff --git a/bin/test_multithreading.py b/bin/test_multithreading.py deleted file mode 100644 index 5eae91623e4..00000000000 --- a/bin/test_multithreading.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -Script to test concurrency (multithreading/multiprocess) issues with the workers. Use with caution. -""" -import json -import atfork -atfork.monkeypatch_os_fork_functions() -import atfork.stdlib_fixer -atfork.stdlib_fixer.fix_logging_module() - -import time -from redash.data import worker -from redash import models, data_manager, redis_connection - -if __name__ == '__main__': - models.create_db(True, False) - - print "Creating data source..." - data_source = models.DataSource.create(name="Concurrency", type="pg", options="dbname=postgres") - - print "Clear jobs/hashes:" - redis_connection.delete("jobs") - query_hashes = redis_connection.keys("query_hash_*") - if query_hashes: - redis_connection.delete(*query_hashes) - - starting_query_results_count = models.QueryResult.select().count() - jobs_count = 5000 - workers_count = 10 - - print "Creating jobs..." - for i in xrange(jobs_count): - query = "SELECT {}".format(i) - print "Inserting: {}".format(query) - data_manager.add_job(query=query, priority=worker.Job.LOW_PRIORITY, - data_source=data_source) - - print "Starting workers..." - workers = data_manager.start_workers(workers_count) - - print "Waiting for jobs to be done..." - keep_waiting = True - while keep_waiting: - results_count = models.QueryResult.select().count() - starting_query_results_count - print "QueryResults: {}".format(results_count) - time.sleep(5) - if results_count == jobs_count: - print "Yay done..." - keep_waiting = False - - data_manager.stop_workers() - - qr_count = 0 - for qr in models.QueryResult.select(): - number = int(qr.query.split()[1]) - data_number = json.loads(qr.data)['rows'][0].values()[0] - - if number != data_number: - print "Oops? {} != {} ({})".format(number, data_number, qr.id) - qr_count += 1 - - print "Verified {} query results.".format(qr_count) - - print "Done." \ No newline at end of file diff --git a/migrations/0010_allow_deleting_datasources.py b/migrations/0010_allow_deleting_datasources.py new file mode 100644 index 00000000000..4188c615b1a --- /dev/null +++ b/migrations/0010_allow_deleting_datasources.py @@ -0,0 +1,18 @@ +from playhouse.migrate import PostgresqlMigrator, migrate + +from redash.models import db + +if __name__ == '__main__': + db.connect_db() + migrator = PostgresqlMigrator(db.database) + + with db.database.transaction(): + migrate( + migrator.drop_not_null('queries', 'data_source_id'), + ) + + db.close_db(None) + + + + diff --git a/migrations/0011_migrate_bigquery_to_json.py b/migrations/0011_migrate_bigquery_to_json.py new file mode 100644 index 00000000000..dd805f98aff --- /dev/null +++ b/migrations/0011_migrate_bigquery_to_json.py @@ -0,0 +1,44 @@ +from base64 import b64encode +import json +from redash.models import DataSource + + +def convert_p12_to_pem(p12file): + from OpenSSL import crypto + with open(p12file, 'rb') as f: + p12 = crypto.load_pkcs12(f.read(), "notasecret") + + return crypto.dump_privatekey(crypto.FILETYPE_PEM, p12.get_privatekey()) + +if __name__ == '__main__': + + for ds in DataSource.all(): + + if ds.type == 'bigquery': + options = json.loads(ds.options) + + if 'jsonKeyFile' in options: + continue + + new_options = { + 'projectId': options['projectId'], + 'jsonKeyFile': b64encode(json.dumps({ + 'client_email': options['serviceAccount'], + 'private_key': convert_p12_to_pem(options['privateKey']) + })) + } + + ds.options = json.dumps(new_options) + ds.save() + elif ds.type == 'google_spreadsheets': + options = json.loads(ds.options) + if 'jsonKeyFile' in options: + continue + + with open(options['credentialsFilePath']) as f: + new_options = { + 'jsonKeyFile': b64encode(f.read()) + } + + ds.options = json.dumps(new_options) + ds.save() diff --git a/rd_ui/app/index.html b/rd_ui/app/index.html index 3e491b748c1..2925c70aabf 100644 --- a/rd_ui/app/index.html +++ b/rd_ui/app/index.html @@ -76,6 +76,9 @@
  • Alerts
    • +
  • + Data Sources +