From 6de50b5eff23c33e2767b9e795aa3af7607f1a9b Mon Sep 17 00:00:00 2001 From: Luca Albertalli Date: Fri, 27 May 2016 15:15:26 -0700 Subject: [PATCH 1/7] Created migration to fix the bug --- ...3_fix_wrong_constraint_on_table_columns.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py diff --git a/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py b/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py new file mode 100644 index 000000000000..e80f4208f795 --- /dev/null +++ b/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py @@ -0,0 +1,24 @@ +"""Fix wrong constraint on table columns + +Revision ID: 1226819ee0e3 +Revises: 956a063c52b3 +Create Date: 2016-05-27 15:03:32.980343 + +""" + +# revision identifiers, used by Alembic. +revision = '1226819ee0e3' +down_revision = '956a063c52b3' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.drop_constraint('columns_ibfk_1', 'columns') + op.create_foreign_key('columns_ibfk_1', 'columns', 'datasources', ['datasource_name'], ['datasource_name']) + + +def downgrade(): + op.drop_constraint('columns_ibfk_1', 'columns') + op.create_foreign_key('columns_ibfk_1', 'columns', 'datasources', ['columns'], ['datasource_name']) From 29ffe03be64103f766eea4e37fda2e3d74f7cba0 Mon Sep 17 00:00:00 2001 From: Luca Albertalli Date: Fri, 27 May 2016 15:26:45 -0700 Subject: [PATCH 2/7] Working also on MySQL --- .../1226819ee0e3_fix_wrong_constraint_on_table_columns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py b/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py index e80f4208f795..b63a762a1b58 100644 --- a/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py +++ b/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py @@ -15,10 +15,10 @@ def upgrade(): - op.drop_constraint('columns_ibfk_1', 'columns') + op.drop_constraint('columns_ibfk_1', 'columns', type_="foreignkey") op.create_foreign_key('columns_ibfk_1', 'columns', 'datasources', ['datasource_name'], ['datasource_name']) def downgrade(): - op.drop_constraint('columns_ibfk_1', 'columns') + op.drop_constraint('columns_ibfk_1', 'columns', type_="foreignkey") op.create_foreign_key('columns_ibfk_1', 'columns', 'datasources', ['columns'], ['datasource_name']) From 839fb020614ebc0b89d50ec115a88da5b5dff921 Mon Sep 17 00:00:00 2001 From: Jiayu Liu Date: Wed, 1 Jun 2016 12:07:14 +0800 Subject: [PATCH 3/7] Update models.py (#541) removing duplicated `user_id` def --- caravel/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/caravel/models.py b/caravel/models.py index ca069b451de3..9727598a1934 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -1276,7 +1276,6 @@ class Log(Model): user_id = Column(Integer, ForeignKey('ab_user.id')) dashboard_id = Column(Integer) slice_id = Column(Integer) - user_id = Column(Integer, ForeignKey('ab_user.id')) json = Column(Text) user = relationship('User', backref='logs', foreign_keys=[user_id]) dttm = Column(DateTime, default=func.now()) From 1ff530429ad8be3ba255104530d193d64fb2a1eb Mon Sep 17 00:00:00 2001 From: Luca Albertalli Date: Tue, 31 May 2016 21:09:04 -0700 Subject: [PATCH 4/7] Added support for Vertica Grains (#515) --- caravel/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/caravel/models.py b/caravel/models.py index 9727598a1934..0a91ee8f78d4 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -424,6 +424,7 @@ def grains(self): ), } db_time_grains['redshift'] = db_time_grains['postgresql'] + db_time_grains['vertica'] = db_time_grains['postgresql'] for db_type, grains in db_time_grains.items(): if self.sqlalchemy_uri.startswith(db_type): return grains From e283af969a86b7909fd7a345e5985d0f314fc76a Mon Sep 17 00:00:00 2001 From: Riccardo Magliocchetti Date: Wed, 1 Jun 2016 06:10:28 +0200 Subject: [PATCH 5/7] i18n: Fix typo in Druid cluster broker port label (#512) --- babel/messages.pot | 166 +++++++++++++++++++++------------------------ caravel/views.py | 2 +- 2 files changed, 80 insertions(+), 88 deletions(-) diff --git a/babel/messages.pot b/babel/messages.pot index d5c7a1be7dc8..28a096d93cc2 100644 --- a/babel/messages.pot +++ b/babel/messages.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2016-05-20 20:30-0700\n" +"POT-Creation-Date: 2016-05-25 12:39+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -23,314 +23,306 @@ msgid "" "by this type of chart" msgstr "" -#: caravel/models.py:1243 +#: caravel/models.py:1247 msgid "No data was returned." msgstr "" -#: caravel/views.py:124 +#: caravel/views.py:123 msgid "" "Whether to make this column available as a [Time Granularity] option, " "column has to be DATETIME or DATETIME-like" msgstr "" -#: caravel/views.py:133 caravel/views.py:161 +#: caravel/views.py:132 caravel/views.py:160 msgid "Column" msgstr "" -#: caravel/views.py:134 caravel/views.py:194 caravel/views.py:223 +#: caravel/views.py:133 caravel/views.py:193 caravel/views.py:222 msgid "Verbose Name" msgstr "" -#: caravel/views.py:135 caravel/views.py:193 caravel/views.py:222 -#: caravel/views.py:400 caravel/views.py:535 +#: caravel/views.py:134 caravel/views.py:192 caravel/views.py:221 +#: caravel/views.py:399 caravel/views.py:534 msgid "Description" msgstr "" -#: caravel/views.py:136 caravel/views.py:164 +#: caravel/views.py:135 caravel/views.py:163 msgid "Groupable" msgstr "" -#: caravel/views.py:137 caravel/views.py:165 +#: caravel/views.py:136 caravel/views.py:164 msgid "Filterable" msgstr "" -#: caravel/views.py:138 caravel/views.py:197 caravel/views.py:308 -#: caravel/views.py:406 +#: caravel/views.py:137 caravel/views.py:196 caravel/views.py:307 +#: caravel/views.py:405 msgid "Table" msgstr "" -#: caravel/views.py:139 caravel/views.py:166 +#: caravel/views.py:138 caravel/views.py:165 msgid "Count Distinct" msgstr "" -#: caravel/views.py:140 caravel/views.py:167 +#: caravel/views.py:139 caravel/views.py:166 msgid "Sum" msgstr "" -#: caravel/views.py:141 caravel/views.py:168 +#: caravel/views.py:140 caravel/views.py:167 msgid "Min" msgstr "" -#: caravel/views.py:142 caravel/views.py:169 +#: caravel/views.py:141 caravel/views.py:168 msgid "Max" msgstr "" -#: caravel/views.py:143 +#: caravel/views.py:142 msgid "Expression" msgstr "" -#: caravel/views.py:144 +#: caravel/views.py:143 msgid "Is temporal" msgstr "" -#: caravel/views.py:162 caravel/views.py:195 caravel/views.py:224 -#: caravel/views.py:424 +#: caravel/views.py:161 caravel/views.py:194 caravel/views.py:223 +#: caravel/views.py:423 msgid "Type" msgstr "" -#: caravel/views.py:163 caravel/views.py:399 +#: caravel/views.py:162 caravel/views.py:398 msgid "Datasource" msgstr "" -#: caravel/views.py:192 caravel/views.py:221 +#: caravel/views.py:191 caravel/views.py:220 msgid "Metric" msgstr "" -#: caravel/views.py:196 +#: caravel/views.py:195 msgid "SQL Expression" msgstr "" -#: caravel/views.py:225 caravel/views.py:503 +#: caravel/views.py:224 caravel/views.py:502 msgid "JSON" msgstr "" -#: caravel/views.py:226 +#: caravel/views.py:225 msgid "Druid Datasource" msgstr "" -#: caravel/views.py:257 caravel/views.py:310 +#: caravel/views.py:256 caravel/views.py:309 msgid "Database" msgstr "" -#: caravel/views.py:258 +#: caravel/views.py:257 msgid "SQL link" msgstr "" -#: caravel/views.py:259 caravel/views.py:397 caravel/views.py:459 +#: caravel/views.py:258 caravel/views.py:396 caravel/views.py:458 msgid "Creator" msgstr "" -#: caravel/views.py:260 caravel/views.py:311 +#: caravel/views.py:259 caravel/views.py:310 msgid "Last Changed" msgstr "" -#: caravel/views.py:261 +#: caravel/views.py:260 msgid "SQLAlchemy URI" msgstr "" -#: caravel/views.py:262 caravel/views.py:317 caravel/views.py:396 -#: caravel/views.py:541 +#: caravel/views.py:261 caravel/views.py:316 caravel/views.py:395 +#: caravel/views.py:540 msgid "Cache Timeout" msgstr "" -#: caravel/views.py:263 +#: caravel/views.py:262 msgid "Extra" msgstr "" -#: caravel/views.py:279 +#: caravel/views.py:278 msgid "Databases" msgstr "" -#: caravel/views.py:281 caravel/views.py:337 caravel/views.py:369 +#: caravel/views.py:280 caravel/views.py:336 caravel/views.py:368 msgid "Sources" msgstr "" -#: caravel/views.py:309 +#: caravel/views.py:308 msgid "Changed By" msgstr "" -#: caravel/views.py:312 +#: caravel/views.py:311 msgid "SQL Editor" msgstr "" -#: caravel/views.py:313 caravel/views.py:537 +#: caravel/views.py:312 caravel/views.py:536 msgid "Is Featured" msgstr "" -#: caravel/views.py:314 +#: caravel/views.py:313 msgid "Schema" msgstr "" -#: caravel/views.py:315 caravel/views.py:539 +#: caravel/views.py:314 caravel/views.py:538 msgid "Default Endpoint" msgstr "" -#: caravel/views.py:316 +#: caravel/views.py:315 msgid "Offset" msgstr "" -#: caravel/views.py:354 caravel/views.py:534 +#: caravel/views.py:353 caravel/views.py:533 msgid "Cluster" msgstr "" -#: caravel/views.py:355 +#: caravel/views.py:354 msgid "Coordinator Host" msgstr "" -#: caravel/views.py:356 +#: caravel/views.py:355 msgid "Coordinator Port" msgstr "" -#: caravel/views.py:357 +#: caravel/views.py:356 msgid "Coordinator Endpoint" msgstr "" -#: caravel/views.py:358 +#: caravel/views.py:357 msgid "Broker Host" msgstr "" -#: caravel/views.py:359 -msgid "Borker Port" +#: caravel/views.py:358 +msgid "Broker Port" msgstr "" -#: caravel/views.py:360 +#: caravel/views.py:359 msgid "Broker Endpoint" msgstr "" -#: caravel/views.py:398 caravel/views.py:479 +#: caravel/views.py:397 caravel/views.py:478 msgid "Dashboards" msgstr "" -#: caravel/views.py:401 +#: caravel/views.py:400 msgid "Last Modified" msgstr "" -#: caravel/views.py:402 caravel/views.py:458 +#: caravel/views.py:401 caravel/views.py:457 msgid "Owners" msgstr "" -#: caravel/views.py:403 +#: caravel/views.py:402 msgid "Parameters" msgstr "" -#: caravel/views.py:404 caravel/views.py:425 +#: caravel/views.py:403 caravel/views.py:424 msgid "Slice" msgstr "" -#: caravel/views.py:405 +#: caravel/views.py:404 msgid "Name" msgstr "" -#: caravel/views.py:407 caravel/views.py:426 +#: caravel/views.py:406 caravel/views.py:425 msgid "Visualization Type" msgstr "" -#: caravel/views.py:441 +#: caravel/views.py:440 msgid "" "This json object describes the positioning of the widgets in the " "dashboard. It is dynamically generated when adjusting the widgets size " "and positions by using drag & drop in the dashboard view" msgstr "" -#: caravel/views.py:446 +#: caravel/views.py:445 msgid "" "The css for individual dashboards can be altered here, or in the " "dashboard view where changes are immediately visible" msgstr "" -#: caravel/views.py:450 +#: caravel/views.py:449 msgid "To get a readable URL for your dashboard" msgstr "" -#: caravel/views.py:454 +#: caravel/views.py:453 msgid "Dashboard" msgstr "" -#: caravel/views.py:455 +#: caravel/views.py:454 msgid "Title" msgstr "" -#: caravel/views.py:456 +#: caravel/views.py:455 msgid "Slug" msgstr "" -#: caravel/views.py:457 +#: caravel/views.py:456 msgid "Slices" msgstr "" -#: caravel/views.py:460 +#: caravel/views.py:459 msgid "Modified" msgstr "" -#: caravel/views.py:461 +#: caravel/views.py:460 msgid "Position JSON" msgstr "" -#: caravel/views.py:462 +#: caravel/views.py:461 msgid "CSS" msgstr "" -#: caravel/views.py:463 +#: caravel/views.py:462 msgid "JSON Metadata" msgstr "" -#: caravel/views.py:500 +#: caravel/views.py:499 msgid "User" msgstr "" -#: caravel/views.py:501 +#: caravel/views.py:500 msgid "Action" msgstr "" -#: caravel/views.py:502 +#: caravel/views.py:501 msgid "dttm" msgstr "" -#: caravel/views.py:509 +#: caravel/views.py:508 msgid "Action Log" msgstr "" -#: caravel/views.py:510 +#: caravel/views.py:509 msgid "Security" msgstr "" -#: caravel/views.py:527 +#: caravel/views.py:526 msgid "Timezone offset (in hours) for this datasource" msgstr "" -#: caravel/views.py:533 +#: caravel/views.py:532 msgid "Data Source" msgstr "" -#: caravel/views.py:536 +#: caravel/views.py:535 msgid "Owner" msgstr "" -#: caravel/views.py:538 +#: caravel/views.py:537 msgid "Is Hidden" msgstr "" -#: caravel/views.py:540 +#: caravel/views.py:539 msgid "Time Offset" msgstr "" -#: caravel/views.py:555 +#: caravel/views.py:554 msgid "Druid Datasources" msgstr "" -#: caravel/views.py:639 -msgid "The datasource seems to have been deleted" -msgstr "" - -#: caravel/views.py:647 -msgid "You don't seem to have access to this datasource" -msgstr "" - -#: caravel/views.py:970 +#: caravel/views.py:969 msgid "This view requires the `all_datasource_access` permission" msgstr "" -#: caravel/views.py:1081 +#: caravel/views.py:1064 msgid "CSS Templates" msgstr "" diff --git a/caravel/views.py b/caravel/views.py index 7c6da2155d5b..b52abfda99e6 100644 --- a/caravel/views.py +++ b/caravel/views.py @@ -355,7 +355,7 @@ class DruidClusterModelView(CaravelModelView, DeleteMixin): # noqa 'coordinator_port': _("Coordinator Port"), 'coordinator_endpoint': _("Coordinator Endpoint"), 'broker_host': _("Broker Host"), - 'broker_port': _("Borker Port"), + 'broker_port': _("Broker Port"), 'broker_endpoint': _("Broker Endpoint"), } From 1650138617757e2dc050b76f29ec79572474f240 Mon Sep 17 00:00:00 2001 From: Luca Albertalli Date: Tue, 31 May 2016 21:16:32 -0700 Subject: [PATCH 6/7] Fix #529 1 "This Session's transaction has been rolled back" (#530) * Fixing the specific issue * Added an additional fix for a similar error in #529 Background: - When an object is modified by SQLAlchemy, it is invalidated so need to be fetched again from the DB - If there's an exception during a transaction, SQLAlchemy performs a rollback and mark the connection as dirty. Bug: - When handling exceptions, the exception handler tries to access the name of the cluster in the main object. Since the name has been invalidated due to a write, SQLAlchemy tries to fetch it on a 'dirty' connection and spits out an error. Solution: - Fetch the information for handling the exception before starting the process. --- caravel/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/caravel/views.py b/caravel/views.py index b52abfda99e6..b3bb3b4ec76a 100644 --- a/caravel/views.py +++ b/caravel/views.py @@ -317,13 +317,14 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa } def post_add(self, table): + table_name = table.table_name try: table.fetch_metadata() except Exception as e: logging.exception(e) flash( "Table [{}] doesn't seem to exist, " - "couldn't fetch metadata".format(table.table_name), + "couldn't fetch metadata".format(table_name), "danger") utils.merge_perm(sm, 'datasource_access', table.perm) @@ -1001,12 +1002,13 @@ def refresh_datasources(self): """endpoint that refreshes druid datasources metadata""" session = db.session() for cluster in session.query(models.DruidCluster).all(): + cluster_name = cluster.cluster_name try: cluster.refresh_datasources() except Exception as e: flash( "Error while processing cluster '{}'\n{}".format( - cluster, str(e)), + cluster_name, str(e)), "danger") logging.exception(e) return redirect('/druidclustermodelview/list/') From 938f6cdb033fb885e1780cef2d472c9bccec2c70 Mon Sep 17 00:00:00 2001 From: Luca Albertalli Date: Tue, 31 May 2016 23:16:19 -0700 Subject: [PATCH 7/7] Modified the migration function to to automatically detect the the foreign keys based on the signature. It supports also sqlite using batch migrations --- ...3_fix_wrong_constraint_on_table_columns.py | 36 ++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py b/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py index b63a762a1b58..af96538849aa 100644 --- a/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py +++ b/caravel/migrations/versions/1226819ee0e3_fix_wrong_constraint_on_table_columns.py @@ -13,12 +13,38 @@ from alembic import op import sqlalchemy as sa +naming_convention = { + "fk": + "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", +} + +def find_constraint_name(upgrade = True): + __table = 'columns' + __cols = {'column_name'} if upgrade else {'datasource_name'} + __referenced = 'datasources' + __ref_cols = {'datasource_name'} if upgrade else {'column_name'} + + engine = op.get_bind().engine + m = sa.MetaData({}) + t=sa.Table(__table,m, autoload=True, autoload_with=engine) + + for fk in t.foreign_key_constraints: + if fk.referred_table.name == __referenced and \ + set(fk.column_keys) == __cols: + return fk.name + return None def upgrade(): - op.drop_constraint('columns_ibfk_1', 'columns', type_="foreignkey") - op.create_foreign_key('columns_ibfk_1', 'columns', 'datasources', ['datasource_name'], ['datasource_name']) - + constraint = find_constraint_name() or 'fk_columns_column_name_datasources' + with op.batch_alter_table("columns", + naming_convention=naming_convention) as batch_op: + batch_op.drop_constraint(constraint, type_="foreignkey") + batch_op.create_foreign_key('fk_columns_datasource_name_datasources', 'datasources', ['datasource_name'], ['datasource_name']) def downgrade(): - op.drop_constraint('columns_ibfk_1', 'columns', type_="foreignkey") - op.create_foreign_key('columns_ibfk_1', 'columns', 'datasources', ['columns'], ['datasource_name']) + constraint = find_constraint_name(False) or 'fk_columns_datasource_name_datasources' + with op.batch_alter_table("columns", + naming_convention=naming_convention) as batch_op: + batch_op.drop_constraint(constraint, type_="foreignkey") + batch_op.create_foreign_key('fk_columns_column_name_datasources', 'datasources', ['column_name'], ['datasource_name']) + \ No newline at end of file