Skip to content

Commit

Permalink
Merge pull request #110 from ThreeSixtyGiving/org-output
Browse files Browse the repository at this point in the history 
Funder and Recipient GrantNav export
  • Loading branch information
@michaelwood
michaelwood committed Mar 10, 2022
2 parents 7ec0f58 + e83bdc9 commit 039c6d2
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 23 deletions.
109 changes: 109 additions & 0 deletions datastore/db/management/commands/create_data_package.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os

from django.core.management.base import BaseCommand
from django.db import connection

from db.management.spinner import Spinner
from db.models import Latest
Expand Down Expand Up @@ -49,6 +50,31 @@ def handle(self, *args, **options):
data_all = []
data_all_file = "%s/data_all.json" % options["dir"]

recipients_file = "%s/recipients.jl" % options["dir"]
funders_file = "%s/funders.jl" % options["dir"]

with connection.cursor() as cursor, open(
recipients_file, "w"
) as recipientfp, open(funders_file, "w") as funderfp:
cursor.execute(CREATE_RELATED_ORGIDS)

cursor.execute(ORG_SELECT.format(org="recipient"))
columns = [col[0] for col in cursor.description]
rows = 0
for row in cursor.fetchall():
recipient = json.dumps(dict(zip(columns, row)))
recipientfp.write(recipient)
recipientfp.write("\r\n")
rows += 1

cursor.execute(ORG_SELECT.format(org="funding"))
rows = 0
for row in cursor.fetchall():
funder = json.dumps(dict(zip(columns, row)))
funderfp.write(funder)
funderfp.write("\r\n")
rows += 1

def flatten_grant(in_grant):
"""Add the additional_data inside grant object"""
out_grant = {}
Expand Down Expand Up @@ -86,3 +112,86 @@ def flatten_grant(in_grant):
data_all_fp.write(json.dumps(data_all, indent=options["indent"]))

spinner.stop()


CREATE_RELATED_ORGIDS = """
DROP TABLE IF EXISTS tmp_related_orgids;
CREATE TABLE tmp_related_orgids AS (
with org_name AS (SELECT
data -> 'linked_orgs' ->> 0 AS canonical_orgid,
string_agg(data ->> 'name', '||') AS name,
string_agg(org_id, '||') AS name_org_id
FROM additional_data_orginfocache GROUP BY 1
)
SELECT
linked_org ->> 0 org_id,
data -> 'linked_orgs' tmp_related_orgids,
data -> 'linked_orgs' ->> 0 AS canonical_orgid,
max(name) AS name,
max(name_org_id) AS name_org_id
FROM
additional_data_orginfocache orgi
JOIN LATERAL
jsonb_array_elements(data -> 'linked_orgs') linked_org ON true
JOIN
org_name orgn ON orgi.data -> 'linked_orgs' ->> 0 = canonical_orgid
WHERE data -> 'linked_orgs' ->> 0 is not null
GROUP by 1,2,3
);
"""

ORG_SELECT = """
WITH latest_grant AS (
SELECT
*
FROM
db_grant
JOIN
db_grant_latest ON db_grant.id = db_grant_latest.grant_id
JOIN
db_latest on db_grant_latest.latest_id = db_latest.id
WHERE
db_latest.series = 'CURRENT'
),
{org}_by_currency AS (SELECT
coalesce(o.canonical_orgid, g.data -> '{org}Organization' -> 0 ->> 'id') org_id,
g.data ->> 'currency' AS currency,
coalesce(tmp_related_orgids, to_jsonb(ARRAY[g.data -> '{org}Organization' -> 0 ->> 'id'])) AS orgids,
max(coalesce(
o.name || '||' || (g.data -> '{org}Organization' -> 0 ->> 'name'),
g.data -> '{org}Organization' -> 0 ->> 'name')
) AS name,
max(name_org_id) AS org_ids_charity_finder,
max(o.name) AS name_charity_finder,
count(*) AS grants,
sum((g.data ->> 'amountAwarded')::numeric) total_amount,
max((g.data ->> 'amountAwarded')::numeric) max_amount,
min((g.data ->> 'amountAwarded')::numeric) min_amount,
avg((g.data ->> 'amountAwarded')::numeric) avg_amount,
max(g.data ->> 'awardDate') max_award_date,
min(g.data ->> 'awardDate') min_award_date
FROM
latest_grant g
LEFT JOIN
tmp_related_orgids o ON o.org_id = g.data -> '{org}Organization' -> 0 ->> 'id'
GROUP BY 1, 2, 3)
SELECT
org_id as id,
string_to_array(string_agg(name, '||'), '||') AS "organizationName",
orgids AS "orgIDs",
coalesce(string_to_array(string_agg(org_ids_charity_finder, '||'), '||'), Array[]::text[]) AS "orgIDsCharityFinder",
coalesce(string_to_array(string_agg(name_charity_finder, '||'), '||'), Array[]::text[]) AS "nameCharityFinder",
sum(grants)::int AS grants,
array_agg(currency) as currency,
jsonb_object_agg(currency, grants) AS "currencyGrants",
jsonb_object_agg(currency, total_amount) AS "currencyTotal",
jsonb_object_agg(currency, max_amount) AS "currencyMaxAmount",
jsonb_object_agg(currency, min_amount) AS "currencyMinAmount",
jsonb_object_agg(currency, avg_amount) AS "currencyAvgAmount",
max(max_award_date) "maxAwardDate",
min(min_award_date) "minAwardDate"
FROM
{org}_by_currency
GROUP BY org_id, orgids;
"""
6 changes: 3 additions & 3 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,15 @@ django-filter==2.2.0
# via -r requirements.in
django-prettyjson==0.4.1
# via -r requirements.in
django==2.2.24
django==2.2.27
# via
# -r requirements.in
# django-filter
# django-prettyjson
# djangorestframework
djangorestframework==3.11.2
# via -r requirements.in
et_xmlfile==1.1.0
et-xmlfile==1.1.0
# via
# datagetter
# openpyxl
Expand Down Expand Up @@ -130,7 +130,7 @@ pytz==2021.3
# datagetter
# django
# flattentool
rangedict==0.1.6
rangedict==0.1.7
# via lib360dataquality
requests[socks]==2.26.0
# via
Expand Down
38 changes: 18 additions & 20 deletions requirements_dev.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ backports-datetime-fromisoformat==1.0.0
# via
# datagetter
# flattentool
black==21.11b1
black==22.1.0
# via -r requirements_dev.in
btrees==4.9.2
# via
Expand All @@ -44,19 +44,19 @@ charset-normalizer==2.0.7
# via
# datagetter
# requests
chromedriver-autoinstaller==0.2.2
chromedriver-autoinstaller==0.3.1
# via -r requirements_dev.in
click==8.0.3
click==8.0.4
# via black
contextlib2==21.6.0
# via
# datagetter
# schema
coverage==6.1.2
coverage==6.3.2
# via coveralls
coveralls==3.3.1
# via -r requirements_dev.in
cryptography==36.0.0
cryptography==36.0.1
# via
# pyopenssl
# urllib3
Expand All @@ -70,7 +70,7 @@ django-filter==2.2.0
# via -r requirements.in
django-prettyjson==0.4.1
# via -r requirements.in
django==2.2.24
django==2.2.27
# via
# -r requirements.in
# django-filter
Expand All @@ -80,7 +80,7 @@ djangorestframework==3.11.2
# via -r requirements.in
docopt==0.6.2
# via coveralls
et_xmlfile==1.1.0
et-xmlfile==1.1.0
# via
# datagetter
# openpyxl
Expand All @@ -90,7 +90,7 @@ flattentool==0.17.1
# via
# datagetter
# libcove
h11==0.12.0
h11==0.13.0
# via wsproto
idna==3.3
# via
Expand Down Expand Up @@ -145,7 +145,7 @@ persistent==4.7.0
# btrees
# datagetter
# zodb
platformdirs==2.4.0
platformdirs==2.5.1
# via black
prometheus_client==0.7
# via -r requirements.in
Expand All @@ -159,7 +159,7 @@ pycparser==2.21
# datagetter
pyflakes==2.4.0
# via flake8
pyopenssl==21.0.0
pyopenssl==22.0.0
# via urllib3
pyrsistent==0.18.0
# via
Expand All @@ -169,17 +169,16 @@ pysocks==1.7.1
# via
# datagetter
# requests
# urllib3
python-dateutil==2.8.2
# via lib360dataquality
pytz==2021.3
# via
# datagetter
# django
# flattentool
rangedict==0.1.6
rangedict==0.1.7
# via lib360dataquality
regex==2021.11.10
# via black
requests-mock==1.9.3
# via -r requirements_dev.in
requests[socks]==2.26.0
Expand All @@ -198,14 +197,13 @@ schema==0.7.4
# via
# datagetter
# flattentool
selenium==4.1.0
selenium==4.1.3
# via -r requirements_dev.in
six==1.16.0
# via
# datagetter
# django-prettyjson
# jsonschema
# pyopenssl
# python-dateutil
# requests-mock
# rfc3339-validator
Expand All @@ -220,26 +218,26 @@ standardjson==0.3.1
# via django-prettyjson
strict-rfc3339==0.7
# via datagetter
tomli==1.2.2
tomli==2.0.1
# via black
transaction==3.0.1
# via
# datagetter
# zodb
trio-websocket==0.9.2
# via selenium
trio==0.19.0
trio==0.20.0
# via
# selenium
# trio-websocket
typing-extensions==4.0.0
typing-extensions==4.1.1
# via black
urllib3[secure]==1.26.7
urllib3[secure,socks]==1.26.7
# via
# datagetter
# requests
# selenium
wsproto==1.0.0
wsproto==1.1.0
# via trio-websocket
xmltodict==0.12.0
# via
Expand Down

0 comments on commit 039c6d2

Please sign in to comment.