Skip to content

Commit

Permalink
Tests: Add tests for field_lists across awards/contracts
Browse files Browse the repository at this point in the history 
Test data modified to make sure there were cases where the
award/contract link worked.

Refactor to field_lists code so that the format arguments are put into a
dictionary to be passed to execute.
  • Loading branch information
@kindly
kindly committed Dec 20, 2021
1 parent 8ac6062 commit d3aa60b
Show file tree
Hide file tree
Showing 3 changed files with 178 additions and 118 deletions.
35 changes: 21 additions & 14 deletions manage.py
View file
Expand Up @@ -566,9 +566,7 @@ def _run_field_lists(name, summary_table, tables_only):
db.set_search_path([name, 'public'])

table = SUMMARIES[summary_table]
field_list_table = f'{summary_table}_field_list'
no_field_list_table = f'{summary_table}_no_field_list'
qualified_primary_keys = [(summary_table, field) for field in table.primary_keys]

variables = {}

if tables_only:
Expand All @@ -578,6 +576,19 @@ def _run_field_lists(name, summary_table, tables_only):
no_field_list_type = sql.SQL('TABLE' if table.is_table else 'VIEW')
final_summary_type = sql.SQL('VIEW')

field_list_table = f'{summary_table}_field_list'
format_kwargs = dict(
summary_table=summary_table,
field_list_table=field_list_table,
no_field_list_table=f'{summary_table}_no_field_list',
qualified_primary_keys=[(summary_table, field) for field in table.primary_keys],
no_field_list_type=no_field_list_type,
final_summary_type=final_summary_type,
data_column=table.data_column,
primary_keys=table.primary_keys,
index=f'{field_list_table}_id'
)

counts_per_path_select = """
SELECT
{primary_keys},
Expand All @@ -595,8 +606,10 @@ def _run_field_lists(name, summary_table, tables_only):
if summary_table in ('contracts_summary', 'awards_summary'):
if summary_table == 'contracts_summary':
variables['path_prefix'] = 'awards'
format_kwargs['other_data_column'] = 'award'
else:
variables['path_prefix'] = 'contracts'
format_kwargs['other_data_column'] = 'contract'

counts_per_path_select += """
UNION ALL
Expand All @@ -613,7 +626,7 @@ def _run_field_lists(name, summary_table, tables_only):
awards_summary.id = contracts_summary.id AND
awards_summary.award_id = contracts_summary.awardid
CROSS JOIN
flatten(contracts_summary.contract)
flatten({other_data_column})
GROUP BY
{qualified_primary_keys}, path
Expand Down Expand Up @@ -648,17 +661,13 @@ def _run_field_lists(name, summary_table, tables_only):
GROUP BY
{primary_keys}
""".replace('INNER_SELECT', counts_per_path_select)
db.execute(statement, variables=variables, summary_table=summary_table, field_list_table=field_list_table,
data_column=table.data_column, primary_keys=table.primary_keys,
qualified_primary_keys=qualified_primary_keys)
db.execute(statement, variables=variables, **format_kwargs)

statement = 'CREATE UNIQUE INDEX {index} ON {field_list_table}({primary_keys})'
db.execute(statement, index=f'{field_list_table}_id', field_list_table=field_list_table,
primary_keys=table.primary_keys)
db.execute(statement, **format_kwargs)

statement = 'ALTER {no_field_list_type} {summary_table} RENAME TO {no_field_list_table}'
db.execute(statement, no_field_list_type=no_field_list_type, summary_table=summary_table,
no_field_list_table=no_field_list_table)
db.execute(statement, **format_kwargs)

statement = """
CREATE {final_summary_type} {summary_table} AS
Expand All @@ -670,9 +679,7 @@ def _run_field_lists(name, summary_table, tables_only):
JOIN
{field_list_table} USING ({primary_keys})
"""
db.execute(statement, final_summary_type=final_summary_type, summary_table=summary_table,
no_field_list_table=no_field_list_table, field_list_table=field_list_table,
primary_keys=table.primary_keys)
db.execute(statement, **format_kwargs)

for row in db.all(COLUMN_COMMENTS_SQL, {'schema': name, 'table': f'{summary_table}_no_field_list'}):
statement = 'COMMENT ON COLUMN {table}.{column} IS %(comment)s'
Expand Down
200 changes: 100 additions & 100 deletions tests/fixtures/kingfisher-process.sql
View file

Large diffs are not rendered by default.

61 changes: 57 additions & 4 deletions tests/test_add.py
View file
Expand Up @@ -263,15 +263,15 @@ def test_command(db, tables_only, field_counts, field_lists, tables, views, filt
'award_documents_summary': 11,
'award_items_summary': 26,
'award_suppliers_summary': 28,
'awards_summary': 140,
'awards_summary': 469,
'buyer_summary': 28,
'contract_documents_summary': 11,
'contract_implementation_documents_summary': 11,
'contract_implementation_milestones_summary': 29,
'contract_implementation_transactions_summary': 83,
'contract_items_summary': 26,
'contract_milestones_summary': 27,
'contracts_summary': 328,
'contracts_summary': 469,
'parties_summary': 34,
'planning_documents_summary': 11,
'planning_milestones_summary': 29,
Expand All @@ -291,6 +291,59 @@ def test_command(db, tables_only, field_counts, field_lists, tables, views, filt

assert count == expected[table_name], f'{table_name}: {count} != {expected[table_name]}'

def result_dict(statement):
result = db.one(statement)
return dict((column.name, result) for
(column, result) in zip(db.cursor.description, result))

statement = """
SELECT
count(*) total,
sum(coalesce((field_list ->> 'contracts')::int, 0)) contracts,
sum(coalesce((field_list ->> 'awards')::int, 0)) awards,
sum(coalesce((field_list ->> 'awards/id')::int, 0)) awards_id,
sum(coalesce((field_list ->> 'awards/value/amount')::int, 0)) awards_amount
FROM
view_data_collection_1.contracts_summary
"""

if filters:
assert result_dict(statement) == {'awards': 1,
'awards_amount': 1,
'awards_id': 1,
'contracts': 0,
'total': 1}
else:
assert result_dict(statement) == {'awards': 213,
'awards_amount': 213,
'awards_id': 213,
'contracts': 0,
'total': 285}

statement = """
SELECT
count(*) total,
sum(coalesce((field_list ->> 'awards')::int, 0)) awards,
sum(coalesce((field_list ->> 'contracts')::int, 0)) contracts,
sum(coalesce((field_list ->> 'contracts/id')::int, 0)) contracts_id,
sum(coalesce((field_list ->> 'contracts/value/amount')::int, 0)) contracts_amount
FROM
view_data_collection_1.awards_summary
"""

if filters:
assert result_dict(statement) == {'contracts': 1,
'contracts_amount': 1,
'contracts_id': 1,
'awards': 0,
'total': 4}
else:
assert result_dict(statement) == {'contracts': 213,
'contracts_amount': 213,
'contracts_id': 213,
'awards': 0,
'total': 301}

# All columns have comments.
assert not db.all("""
SELECT
Expand Down Expand Up @@ -530,15 +583,15 @@ def test_command_filter(db, tables_only, field_counts, field_lists, tables, view
'award_documents_summary': 11,
'award_items_summary': 29,
'award_suppliers_summary': 30,
'awards_summary': 164,
'awards_summary': 492,
'buyer_summary': 31,
'contract_documents_summary': 11,
'contract_implementation_documents_summary': 11,
'contract_implementation_milestones_summary': 23,
'contract_implementation_transactions_summary': 83,
'contract_items_summary': 26,
'contract_milestones_summary': 26,
'contracts_summary': 327,
'contracts_summary': 492,
'parties_summary': 30,
'planning_documents_summary': 11,
'planning_milestones_summary': 27,
Expand Down

0 comments on commit d3aa60b

Please sign in to comment.