From 0997ead6cff3f6bdff6f87e523729624dcd2491b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 8 Dec 2017 23:30:57 -0500 Subject: [PATCH 1/2] adds predenominated to ensure raw measures are returned --- cartoframes/context.py | 102 +++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/cartoframes/context.py b/cartoframes/context.py index 8eb77ed5c..2db30b67a 100644 --- a/cartoframes/context.py +++ b/cartoframes/context.py @@ -1234,11 +1234,12 @@ def data_discovery(self, region, keywords=None, regex=None, time=None, else: filters = '' - numer_query = '\n'.join(s.strip() for s in ( + numer_query = utils.minify_sql(( 'SELECT', ' numer_id,', ' {geom_id} AS geom_id,', - ' {timespan} AS numer_timespan', + ' {timespan} AS numer_timespan,', + ' {normalization} AS normalization', ' FROM', ' OBS_GetAvailableNumerators(', ' (SELECT env FROM envelope),', @@ -1253,14 +1254,16 @@ def data_discovery(self, region, keywords=None, regex=None, time=None, numer_query.format( timespan=('\'{}\''.format(t) if t else 'null'), geom_id=('\'{}\''.format(b) if b else 'null'), + normalization=('\'{}\''.format(n) if n else 'null'), countrytag=countrytag, filters=filters ) for t in time for b in boundaries + for n in ('predenominated', None) ) - query = '\n'.join(s.strip() for s in ( + query = utils.minify_sql(( 'WITH envelope AS (', ' {boundary}', '), numers AS (', @@ -1370,40 +1373,41 @@ def data(self, table_name, metadata, persist_as=None, how='the_geom'): if isinstance(metadata, pd.DataFrame): _meta = metadata.copy().reset_index() elif isinstance(metadata, collections.Iterable): - query = ''' - WITH envelope AS ( - SELECT ST_SetSRID(ST_Extent(the_geom)::geometry, 4326) AS env, - count(*)::int AS cnt - FROM {table_name} - ) - SELECT * - FROM json_to_recordset( - (SELECT OBS_GetMeta( - envelope.env, - ('{meta}')::json, - 10, 1, envelope.cnt - ) AS meta - FROM envelope - GROUP BY env, cnt)) as data( - denom_aggregate text, denom_colname text, - denom_description text, denom_geomref_colname text, - denom_id text, denom_name text, denom_reltype text, - denom_t_description text, denom_tablename text, - denom_type text, geom_colname text, geom_description text, - geom_geomref_colname text, geom_id text, geom_name text, - geom_t_description text, geom_tablename text, - geom_timespan text, geom_type text, id numeric, - max_score_rank text, max_timespan_rank text, - normalization text, num_geoms numeric,numer_aggregate text, - numer_colname text, numer_description text, - numer_geomref_colname text, numer_id text, - numer_name text, numer_t_description text, - numer_tablename text, numer_timespan text, - numer_type text, score numeric, score_rank numeric, - score_rownum numeric, suggested_name text, - target_area text, target_geoms text, timespan_rank numeric, - timespan_rownum numeric) - '''.format( + query = utils.minify_sql(( + 'WITH envelope AS (', + ' SELECT ', + ' ST_SetSRID(ST_Extent(the_geom)::geometry, 4326) AS env,', + ' count(*)::int AS cnt', + ' FROM {table_name}', + ')', + 'SELECT *', + ' FROM json_to_recordset(', + ' (SELECT OBS_GetMeta(', + ' envelope.env,', + ' (\'{meta}\')::json,', + ' 10, 1, envelope.cnt', + ' ) AS meta', + ' FROM envelope', + ' GROUP BY env, cnt)) as data(', + ' denom_aggregate text, denom_colname text,', + ' denom_description text, denom_geomref_colname text,', + ' denom_id text, denom_name text, denom_reltype text,', + ' denom_t_description text, denom_tablename text,', + ' denom_type text, geom_colname text, ', + ' geom_description text,geom_geomref_colname text, ', + ' geom_id text, geom_name text, geom_t_description text, ', + ' geom_tablename text, geom_timespan text, ', + ' geom_type text, id numeric, max_score_rank text, ', + ' max_timespan_rank text, normalization text, num_geoms ', + ' numeric,numer_aggregate text, numer_colname text, ', + ' numer_description text, numer_geomref_colname text, ', + ' numer_id text, numer_name text, numer_t_description ', + ' text, numer_tablename text, numer_timespan text,', + ' numer_type text, score numeric, score_rank numeric,', + ' score_rownum numeric, suggested_name text,', + ' target_area text, target_geoms text, timespan_rank ', + ' numeric, timespan_rownum numeric)', + )).format( table_name=table_name, meta=json.dumps(metadata).replace('\'', '\'\'')) resp = self.sql_client.send(query) @@ -1438,18 +1442,18 @@ def data(self, table_name, metadata, persist_as=None, how='the_geom'): col=row[1]['suggested_name']) for row in _meta.iterrows() ) - query = ''' - SELECT t.*, {cols} - FROM OBS_GetData( - (SELECT array_agg((the_geom, cartodb_id)::geomval) - FROM "{tablename}"), - (SELECT \'{meta}\'::json)) as m, - {tablename} as t - WHERE t.cartodb_id = m.id - '''.format(tablename=table_name, - cols=cols, - meta=_meta.to_json(orient='records').replace( - '\'', '\'\'')) + query = utils.minify_sql(( + 'SELECT t.*, {cols}', + ' FROM OBS_GetData(', + ' (SELECT array_agg((the_geom, cartodb_id)::geomval)', + ' FROM "{tablename}"),', + ' (SELECT \'{meta}\'::json)) as m,', + ' {tablename} as t', + ' WHERE t.cartodb_id = m.id', + )).format(tablename=table_name, + cols=cols, + meta=_meta.to_json(orient='records').replace( + '\'', '\'\'')) return self.query(query, table_name=persist_as) From fd9c3c36b64b0e66058e0bf30782d43320ec276e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 8 Dec 2017 23:43:19 -0500 Subject: [PATCH 2/2] adds pgquote utility function --- cartoframes/context.py | 6 +++--- cartoframes/utils.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cartoframes/context.py b/cartoframes/context.py index 2db30b67a..cd9734faa 100644 --- a/cartoframes/context.py +++ b/cartoframes/context.py @@ -1252,9 +1252,9 @@ def data_discovery(self, region, keywords=None, regex=None, time=None, numers = '\nUNION\n'.join( numer_query.format( - timespan=('\'{}\''.format(t) if t else 'null'), - geom_id=('\'{}\''.format(b) if b else 'null'), - normalization=('\'{}\''.format(n) if n else 'null'), + timespan=utils.pgquote(t), + geom_id=utils.pgquote(b), + normalization=utils.pgquote(n), countrytag=countrytag, filters=filters ) diff --git a/cartoframes/utils.py b/cartoframes/utils.py index 363658508..aebd03a3c 100644 --- a/cartoframes/utils.py +++ b/cartoframes/utils.py @@ -90,3 +90,8 @@ def join_url(*parts): def minify_sql(lines): """eliminate whitespace in sql queries""" return '\n'.join(line.strip() for line in lines) + + +def pgquote(string): + """single-quotes a string if not None, else returns null""" + return '\'{}\''.format(string) if string else 'null'