Skip to content

Commit

Permalink
Merge pull request #335 from CartoDB/310-Fixing_value_naming
Browse files Browse the repository at this point in the history 
Fixing value name. Adding updates
  • Loading branch information
Javier Torres committed Oct 3, 2017
2 parents ff3b8d3 + 5a260f4 commit cf3f507
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 27 deletions.
157 changes: 138 additions & 19 deletions catalog/templates/column.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
SELECT ROW_NUMBER() OVER () AS cartodb_id,
data->0->>'geomref' as geomref,
(data->0->>'value')::Geometry as the_geom,
ST_Transform((data->0->>'value')::Geometry, 3857) as the_geom_webmercator
ST_Transform((data->0->>'value')::Geometry, 3857) AS the_geom_webmercator
FROM OBS_GetData((SELECT ids FROM ids), (SELECT meta FROM meta))

{% else %}
Expand All @@ -76,13 +76,34 @@
{% if denom_reltype == 'denominator' and col.aggregate|lower == 'sum' %}
.. tab:: {"tab_name": "Per {{ denom_name }}"}

Update a field of a table with an estimate of "{{ col.name }}" per {{ denom_name }} for a polygon
using a combination of weighting by area and {{ denom_name }}.

.. code-block:: postgresql

WITH data AS (
SELECT cartodb_id AS id, the_geom FROM <table_to_update>
),
meta AS (
SELECT OBS_GetMeta(ST_SetSRID(ST_Extent(the_geom), 4326),
'[{"numer_id": "{{ col.id }}",
"denom_id": "{{ denom_id }}",
"normalization": "denominated"}]'
, 1, 1) meta
FROM data)
UPDATE <table_to_update> SET <field_to_update> = (data->0->>'value')::{{ col.type }}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta)) obs_data
WHERE <table_to_update>.cartodb_id = obs_data.id

Obtain an estimate of "{{ col.name }}" per {{ denom_name }} for a polygon
using a combination of weighting by area and {{ denom_name }}.

.. code-block:: postgresql

WITH data AS (
SELECT 1 as id,
SELECT 1 AS id,
ST_Buffer(ST_SetSRID(ST_Point({{ col['latlng'][1] }}, {{ col['latlng'][0] }}), 4326), 0.01) the_geom
),
meta AS (
Expand All @@ -92,24 +113,41 @@
"normalization": "denominated"}]'
, 1, 1) meta
FROM data)
SELECT id, (data->0->>'value')::{{ col.type }} as rate_estimate
SELECT id, (data->0->>'value')::{{ col.type }} AS {{col.suggested_name}}_rate_{{col.timespan}}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta))

{% elif col.aggregate in ('median', 'average') and denom_reltype == 'universe' %}
.. tab:: {"tab_name": "Weighted by {{ denom_name }} "}

Obtain an estimate of "{{ col.name }}" for a polygon
Update a field of a table with an estimate of "{{ col.name }}" for a polygon
using a combination of weighting by area and {{ denom_name }}.

{% if col.aggregate == 'median' %}Although this measure is a median, the
estimate will be a weighted average of medians.{% endif %}
.. code-block:: postgresql

WITH data AS (
SELECT cartodb_id AS id, the_geom FROM <table_to_update>
),
meta AS (
SELECT OBS_GetMeta(ST_SetSRID(ST_Extent(the_geom), 4326),
'[{"numer_id": "{{ col.id }}",
"denom_id": "{{ denom_id }}"}]'
, 1, 1) meta
FROM data)
UPDATE <table_to_update> SET <field_to_update> = (data->0->>'value')::{{ col.type }}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta)) obs_data
WHERE <table_to_update>.cartodb_id = obs_data.id

Obtain an estimate of "{{ col.name }}" for a polygon
using a combination of weighting by area and {{ denom_name }}.

.. code-block:: postgresql

WITH data AS (
SELECT 1 as id,
SELECT 1 AS id,
ST_Buffer(ST_SetSRID(ST_Point({{ col['latlng'][1] }}, {{ col['latlng'][0] }}), 4326), 0.01) the_geom
),
meta AS (
Expand All @@ -118,21 +156,45 @@
"denom_id": "{{ denom_id }}"}]'
, 1, 1) meta
FROM data)
SELECT id, (data->0->>'value')::{{ col.type }} as estimate
SELECT id, (data->0->>'value')::{{ col.type }} AS {{col.suggested_name}}_estimate_{{col.timespan}}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta))

{% if col.aggregate == 'median' %}NOTE: Although this measure is a median, the
estimate will be a weighted average of medians.{% endif %}

{% elif denom_reltype == 'denominator' %}
.. tab:: {"tab_name": "Per {{ denom_name }}"}

Update a field of a table with the value of "{{ col.name }}" per {{ denom_name}} for the
smallest available geometry at the selected point.

.. code-block:: postgresql

WITH data AS (
SELECT cartodb_id AS id, the_geom FROM <table_to_update>
),
meta AS (
SELECT OBS_GetMeta(ST_SetSRID(ST_Extent(the_geom), 4326),
'[{"numer_id": "{{ col.id }}",
"denom_id": "{{ denom_id }}",
"normalization": "denominated"}]'
, 1, 1) meta
FROM data)
UPDATE <table_to_update> SET <field_to_update> = (data->0->>'value')::{{ col.type }}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta)) obs_data
WHERE <table_to_update>.cartodb_id = obs_data.id

Obtain value of "{{ col.name }}" per {{ denom_name}} for the
smallest available geometry at the selected point.

.. code-block:: postgresql

WITH data AS (
SELECT 1 as id,
SELECT 1 AS id,
ST_SetSRID(ST_Point({{ col['latlng'][1] }}, {{ col['latlng'][0] }}), 4326) the_geom
),
meta AS (
Expand All @@ -142,7 +204,7 @@
"normalization": "denominated"}]'
, 1, 1) meta
FROM data)
SELECT id, (data->0->>'value')::{{ col.type }} as val
SELECT id, (data->0->>'value')::{{ col.type }} AS {{col.suggested_name}}_{{col.timespan}}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta))
Expand All @@ -154,13 +216,33 @@
{% if col.aggregate == 'sum' %}
.. tab:: {"tab_name": "Per sq km"}

Update a field of a table with an estimate of "{{ col.name }}" per square kilometer using area
weighted interpolation.

.. code-block:: postgresql

WITH data AS (
SELECT cartodb_id AS id, the_geom FROM <table_to_update>
),
meta AS (
SELECT OBS_GetMeta(ST_SetSRID(ST_Extent(the_geom), 4326),
'[{"numer_id": "{{ col.id }}",
"normalization": "areaNormalized"}]'
, 1, 1) meta
FROM data)
UPDATE <table_to_update> SET <field_to_update> = (data->0->>'value')::{{ col.type }}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta)) obs_data
WHERE <table_to_update>.cartodb_id = obs_data.id

Obtain an estimate of "{{ col.name }}" per square kilometer using area
weighted interpolation.

.. code-block:: postgresql

WITH data AS (
SELECT 1 as id,
SELECT 1 AS id,
ST_Buffer(ST_SetSRID(ST_Point({{ col['latlng'][1] }}, {{ col['latlng'][0] }}), 4326), 0.01) the_geom
),
meta AS (
Expand All @@ -169,21 +251,40 @@
"normalization": "areaNormalized"}]'
, 1, 1) meta
FROM data)
SELECT id, (data->0->>'value')::{{ col.type }} as per_sq_km
SELECT id, (data->0->>'value')::{{ col.type }} AS {{col.suggested_name}}_per_sq_km_{{col.timespan}}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta))

.. tab:: {"tab_name": "Raw count"}

Obtain an estimate of "{{ col.name }}" for a polygon using area-weighted
Update a field of a table with an estimate of "{{ col.name }}" for a polygon using area-weighted
interpolation.

.. code-block:: postgresql

WITH data AS (
SELECT cartodb_id AS id, the_geom FROM <table_to_update>
),
meta AS (
SELECT OBS_GetMeta(ST_SetSRID(ST_Extent(the_geom), 4326),
'[{"numer_id": "{{ col.id }}",
"normalization": "predenominated"}]'
, 1, 1) meta
FROM data)
UPDATE <table_to_update> SET <field_to_update> = (data->0->>'value')::{{ col.type }}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta)) obs_data
WHERE <table_to_update>.cartodb_id = obs_data.id

Obtain an estimate of "{{ col.name }}" for a polygon using area-weighted
interpolation.

.. code-block:: postgresql

WITH data AS (
SELECT 1 as id,
SELECT 1 AS id,
ST_Buffer(ST_SetSRID(ST_Point({{ col['latlng'][1] }}, {{ col['latlng'][0] }}), 4326), 0.01) the_geom
),
meta AS (
Expand All @@ -192,30 +293,48 @@
"normalization": "predenominated"}]'
, 1, 1) meta
FROM data)
SELECT id, (data->0->>'value')::{{ col.type }} as per_sq_km
SELECT id, (data->0->>'value')::{{ col.type }} AS {{col.suggested_name}}_{{col.timespan}}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta))


{% else %}
.. tab:: {"tab_name": "Point"}

Obtain the value of "{{ col.name }}" for a point. The value will be from
Update a field of a table with the value of "{{ col.name }}" for a point. The value will be from
the densest bounday with this measure.

.. code-block:: postgresql

WITH data AS (
SELECT cartodb_id AS id, the_geom FROM <table_to_update>
),
meta AS (
SELECT OBS_GetMeta(ST_SetSRID(ST_Extent(the_geom), 4326),
'[{"numer_id": "{{ col.id }}"}]'
, 1, 1) meta
FROM data)
UPDATE <table_to_update> SET <field_to_update> = (data->0->>'value')::{{ col.type }}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta)) obs_data
WHERE <table_to_update>.cartodb_id = obs_data.id

Obtain the value of "{{ col.name }}" for a point. The value will be from
the densest bounday with this measure.

.. code-block:: postgresql

WITH data AS (
SELECT 1 as id,
SELECT 1 AS id,
ST_SetSRID(ST_Point({{ col['latlng'][1] }}, {{ col['latlng'][0] }}), 4326) the_geom
),
meta AS (
SELECT OBS_GetMeta(ST_SetSRID(ST_Extent(the_geom), 4326),
'[{"numer_id": "{{ col.id }}"}]'
, 1, 1) meta
FROM data)
SELECT id, (data->0->>'value')::{{ col.type }} as val
SELECT id, (data->0->>'value')::{{ col.type }} AS {{col.suggested_name}}_{{col.timespan}}
FROM OBS_GetData(
(SELECT ARRAY_AGG((the_geom, id)::geomval) FROM data),
(SELECT meta FROM meta))
Expand Down
16 changes: 11 additions & 5 deletions tasks/sphinx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,8 @@ def _boundaries_tree(self, section_id, subsection_id):
FIRST(ctab.extra),
FIRST(c.aggregate),
JSONB_Object_Agg(t.type || '/' || t.id, t.name),
'name' suggested_name,
FIRST(tab.timespan) timespan,
ARRAY[]::Text[] denoms,
ARRAY[]::Text[],
ST_AsText(ST_Envelope(FIRST(tab.the_geom))) envelope
Expand All @@ -210,7 +212,7 @@ def _boundaries_tree(self, section_id, subsection_id):
AND ct.tag_id = t.id
AND c.id = ctab.column_id
AND tab.id = ctab.table_id
GROUP BY c.id
GROUP BY 1, 8
'''.format("', '".join(boundary_ids)))
boundary_data = self._parse_columns(boundaries_detail_result)
return {k: {} for k in boundary_data.keys()}, boundary_data
Expand Down Expand Up @@ -249,6 +251,8 @@ def _numerators_tree(self, section_id, subsection_id):
numer_extra,
numer_aggregate,
numer_tags,
numer_colname suggested_name,
numer_timespan timespan,
ARRAY_AGG(DISTINCT ARRAY[
denom_reltype,
denom_id,
Expand All @@ -261,15 +265,15 @@ def _numerators_tree(self, section_id, subsection_id):
FIRST(ST_AsText(ST_Envelope(the_geom))) envelope
FROM observatory.obs_meta
WHERE numer_id = ANY (ARRAY['{}'])
GROUP BY 1, 2, 3, 4, 5, 6, 7
GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9
'''.format("', '".join(numerator_ids)))
return numerator_tree, self._parse_columns(numerator_details_result)

def _parse_columns(self, all_columns_result):
all_columns = {}
for col in all_columns_result:
geom_timespans = {}
for gt in col[8]:
for gt in col[10]:
if gt[0] in geom_timespans:
geom_timespans[gt[0]]['timespans'].append(gt[2])
else:
Expand All @@ -288,15 +292,17 @@ def _parse_columns(self, all_columns_result):
'extra': col[4],
'aggregate': col[5],
'tags': col[6],
'suggested_name': col[7],
'timespan': col[8],
'licenses': [tag_id.split('/')[1]
for tag_id, tag_name in col[6].iteritems()
if tag_id.startswith('license/')],
'sources': [tag_id.split('/')[1]
for tag_id, tag_name in col[6].iteritems()
if tag_id.startswith('source/')],
'denoms': col[7],
'denoms': col[9],
'geom_timespans': geom_timespans,
'envelope': col[9]
'envelope': col[11]
}
return all_columns

Expand Down
10 changes: 7 additions & 3 deletions tasks/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2072,9 +2072,13 @@ class RunDiff(WrapperTask):
test_all = BoolParameter(default=False)

def requires(self):
resp = shell("git diff '{compare}' --name-only | grep '^tasks'".format(
compare=self.compare
))
try:
resp = shell("git diff '{compare}' --name-only | grep '^tasks'".format(
compare=self.compare
))
except CalledProcessError:
# No diffs
return
for line in resp.split('\n'):
if not line:
continue
Expand Down

0 comments on commit cf3f507

Please sign in to comment.