Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/SatcherInstitute/health-equ…
Browse files Browse the repository at this point in the history 
…ity-tracker
  • Loading branch information
@benhammondmusic
benhammondmusic committed Jun 20, 2024
2 parents 2b50a42 + ad55973 commit 5871b05
Show file tree
Hide file tree
Showing 18 changed files with 114 additions and 174 deletions.
2 changes: 1 addition & 1 deletion .github/linters/.pylintrc
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[FORMAT]
max-line-length=120
[MESSAGES CONTROL]
disable=C0114,E0401,C0201,C0115,R1714,C0411,W0511,R1705,C0116,C0103,R0201,W0104,W0105,R0801,R0913,R0022,C0412,R0914,R0915,W0621,R0903,W0237,W0221,W0201,R0911,R1710,R0902,R0912,W0622,W1514
disable=C0114,E0401,C0201,C0115,R1714,C0411,W0511,R1705,C0116,C0103,R0201,W0104,W0105,R0801,R0913,R0022,C0412,R0914,R0915,W0621,R0903,W0237,W0221,W0201,R0911,R1710,R0902,R0912,W0622,W1514,W0718
20 changes: 9 additions & 11 deletions data_server/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,22 @@ def get_program_name():
def get_metadata():
"""Downloads and returns metadata about available download files."""
try:
metadata = cache.getDataset(os.environ.get('GCS_BUCKET'),
os.environ.get('METADATA_FILENAME'))
metadata = cache.getDataset(os.environ.get('GCS_BUCKET'), os.environ.get('METADATA_FILENAME'))
except Exception as err:
logging.error(err)
return 'Internal server error: {}'.format(err), 500
return f'Internal server error: {err}', 500

def generate_response(data: bytes):
next_row = b'['
for row in data.splitlines():
yield next_row
next_row = row + b','
yield next_row.rstrip(b',') + b']'

headers = Headers()
headers.add('Content-Disposition', 'attachment',
filename=os.environ.get('METADATA_FILENAME'))
headers.add('Content-Disposition', 'attachment', filename=os.environ.get('METADATA_FILENAME'))
headers.add('Vary', 'Accept-Encoding')
return Response(generate_response(metadata), mimetype='application/json',
headers=headers)
return Response(generate_response(metadata), mimetype='application/json', headers=headers)


@app.route('/dataset', methods=['GET'])
Expand All @@ -52,14 +50,15 @@ def get_dataset():
dataset = cache.getDataset(os.environ.get('GCS_BUCKET'), dataset_name)
except Exception as err:
logging.error(err)
return 'Internal server error: {}'.format(err), 500
return f'Internal server error: {err}', 500

def generate_response(data: bytes):
def generate_response():
next_row = b'['
for row in dataset.splitlines():
yield next_row
next_row = row + b','
yield next_row.rstrip(b',') + b']'

headers = Headers()
headers.add('Content-Disposition', 'attachment', filename=dataset_name)
headers.add('Vary', 'Accept-Encoding')
Expand All @@ -71,8 +70,7 @@ def generate_response(data: bytes):
if dataset_name.endswith('.csv'):
return Response(dataset, mimetype='text/csv', headers=headers)

return Response(generate_response(dataset), mimetype='application/json',
headers=headers)
return Response(generate_response(), mimetype='application/json', headers=headers)


if __name__ == "__main__":
Expand Down
10 changes: 0 additions & 10 deletions data_server/requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,8 @@ charset-normalizer==3.3.2
# via requests
click==8.1.7
# via flask
flask==3.0.3
# via
# -r requirements.in
# flask-cors
flask-cors==4.0.1
# via -r requirements.in
google-api-core==2.19.0
# via
# google-cloud-core
# google-cloud-storage
google-auth==2.29.0
# via
# google-api-core
Expand Down Expand Up @@ -75,5 +67,3 @@ requests==2.32.0
# google-cloud-storage
rsa==4.7
# via google-auth
werkzeug==3.0.3
# via flask
19 changes: 0 additions & 19 deletions exporter/requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,6 @@ click==8.1.3
# via flask
db-dtypes==1.2.0
# via -r requirements.in
flask==3.0.3
# via -r requirements.in
google-api-core[grpc]==2.17.1
# via
# google-cloud-bigquery
# google-cloud-core
# google-cloud-pubsub
# google-cloud-storage
google-auth==2.29.0
# via
# google-api-core
Expand All @@ -28,15 +20,11 @@ google-auth==2.29.0
# google-cloud-storage
google-cloud==0.34.0
# via -r requirements.in
google-cloud-bigquery==3.24.0
# via -r requirements.in
google-cloud-core==2.4.1
# via
# -r requirements.in
# google-cloud-bigquery
# google-cloud-storage
google-cloud-pubsub==2.21.3
# via -r requirements.in
google-cloud-storage==2.17.0
# via -r requirements.in
google-crc32c==1.3.0
Expand Down Expand Up @@ -79,11 +67,6 @@ markupsafe==2.1.1
# via
# jinja2
# werkzeug
numpy==1.23.2
# via
# db-dtypes
# pandas
# pyarrow
packaging==21.3
# via
# db-dtypes
Expand Down Expand Up @@ -130,7 +113,5 @@ six==1.16.0
# via python-dateutil
tzdata==2024.1
# via pandas
werkzeug==3.0.3
# via flask
zipp==3.8.1
# via importlib-metadata
36 changes: 18 additions & 18 deletions frontend/src/data/config/DatasetMetadata.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,98 +507,98 @@ export const DatasetMetadataMap: Record<DatasetId, DatasetMetadata> = {

'cdc_restricted_data-by_race_county_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by race/ethnicity and county',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
contains_nh: true,
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_race_state_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by race/ethnicity and state',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
contains_nh: true,
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_race_national_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by race/ethnicity, nationally',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
contains_nh: true,
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_age_county_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by age and county',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_age_state_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by age and state',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_age_national_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by age, nationally',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_sex_county_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by sex and county',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_sex_state_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by sex and state',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_sex_national_processed_time_series': {
name: 'Monthly COVID-19 deaths, cases, and hospitalizations by sex, nationally',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_race_county_processed': {
name: 'COVID-19 deaths, cases, and hospitalizations since January 2020 by race/ethnicity and county',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
contains_nh: true,
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_race_state_processed-with_age_adjust': {
name: 'COVID-19 deaths, cases, and hospitalizations with age-adjusted ratios since January 2020 by race/ethnicity and state',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
contains_nh: true,
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_race_national_processed-with_age_adjust': {
name: 'COVID-19 deaths, cases, and hospitalizations with age-adjusted ratios since January 2020 by race/ethnicity, nationally',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
contains_nh: true,
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_age_county_processed': {
name: 'COVID-19 deaths, cases, and hospitalizations since January 2020 by age and county',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_age_state_processed': {
name: 'COVID-19 deaths, cases, and hospitalizations since January 2020 by age and state',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_age_national_processed': {
name: 'COVID-19 deaths, cases, and hospitalizations since January 2020 by age, nationally',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_sex_county_processed': {
name: 'COVID-19 deaths, cases, and hospitalizations since January 2020 by sex and county',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_sex_state_processed': {
name: 'COVID-19 deaths, cases, and hospitalizations since January 2020 by sex and state',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_restricted_data-by_sex_national_processed': {
name: 'COVID-19 deaths, cases, and hospitalizations since January 2020 by sex, nationally',
original_data_sourced: 'January 2020 - April 2024',
original_data_sourced: 'January 2020 - May 2024',
source_id: 'cdc_restricted',
},
'cdc_vaccination_county-alls_county': {
Expand Down
89 changes: 44 additions & 45 deletions frontend/src/data/providers/MaternalMortalityProvider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,63 +4,62 @@ import { type MetricQuery, MetricQueryResponse } from '../query/MetricQuery';
import VariableProvider from './VariableProvider';
import { appendFipsIfNeeded } from '../utils/datasetutils';
import { type DatasetId } from '../config/DatasetMetadata';
import DataFrame from 'dataframe-js';
import { DataTypeId } from '../config/MetricConfig';

class MaternalMortalityProvider extends VariableProvider {
constructor() {
super('maternal_mortality_provider', [
'maternal_mortality_per_100k',
'maternal_mortality_pct_share',
'maternal_mortality_population_pct',
'maternal_deaths_estimated_total',
'live_births_estimated_total',
]);
}
constructor() {
super('maternal_mortality_provider', [
'maternal_mortality_per_100k',
'maternal_mortality_pct_share',
'maternal_mortality_population_pct',
'maternal_deaths_estimated_total',
'live_births_estimated_total',
]);
}


getDatasetId(breakdowns: Breakdowns,
dataTypeId?: DataTypeId,
timeView?: TimeView): DatasetId | undefined {
if (breakdowns.hasOnlyRace()) {
if (timeView === 'current') {
if (breakdowns.geography === 'state') return "maternal_mortality_data-by_race_state_current";
if (breakdowns.geography === 'national') return 'maternal_mortality_data-by_race_national_current';
}
if (timeView === 'historical') {
if (breakdowns.geography === 'state') return "maternal_mortality_data-by_race_state_historical";
if (breakdowns.geography === 'national') return 'maternal_mortality_data-by_race_national_historical';
}
getDatasetId(breakdowns: Breakdowns,
dataTypeId?: DataTypeId,
timeView?: TimeView): DatasetId | undefined {
if (breakdowns.hasOnlyRace()) {
if (timeView === 'current') {
if (breakdowns.geography === 'state') return "maternal_mortality_data-by_race_state_current";
if (breakdowns.geography === 'national') return 'maternal_mortality_data-by_race_national_current';
}
if (timeView === 'historical') {
if (breakdowns.geography === 'state') return "maternal_mortality_data-by_race_state_historical";
if (breakdowns.geography === 'national') return 'maternal_mortality_data-by_race_national_historical';
}
}
}
}

async getDataInternal(metricQuery: MetricQuery): Promise<MetricQueryResponse> {
const breakdowns = metricQuery.breakdowns;
const datasetId = this.getDatasetId(breakdowns, undefined, metricQuery.timeView);
if (!datasetId) throw Error('DatasetId undefined');
const specificDatasetId = appendFipsIfNeeded(datasetId, breakdowns);
const maternalMortalityDataset = await getDataManager().loadDataset(specificDatasetId);
const consumedDatasetIds = [datasetId];
let df = maternalMortalityDataset.toDataFrame();
async getDataInternal(metricQuery: MetricQuery): Promise<MetricQueryResponse> {
const breakdowns = metricQuery.breakdowns;
const datasetId = this.getDatasetId(breakdowns, undefined, metricQuery.timeView);
if (!datasetId) throw Error('DatasetId undefined');
const specificDatasetId = appendFipsIfNeeded(datasetId, breakdowns);
const maternalMortalityDataset = await getDataManager().loadDataset(specificDatasetId);
const consumedDatasetIds = [datasetId];
let df = maternalMortalityDataset.toDataFrame();

// Filter by geography
df = this.filterByGeo(df, breakdowns);
// Filter by geography
df = this.filterByGeo(df, breakdowns);

if (df.toArray().length === 0) {
return new MetricQueryResponse([], consumedDatasetIds);
}
df = this.renameGeoColumns(df, breakdowns);
if (df.toArray().length === 0) {
return new MetricQueryResponse([], consumedDatasetIds);
}
df = this.renameGeoColumns(df, breakdowns);

// Apply demographic breakdown filters
df = this.applyDemographicBreakdownFilters(df, breakdowns);
df = this.removeUnrequestedColumns(df, metricQuery);
// Apply demographic breakdown filters
df = this.applyDemographicBreakdownFilters(df, breakdowns);
df = this.removeUnrequestedColumns(df, metricQuery);

return new MetricQueryResponse(df.toArray(), consumedDatasetIds);
}
return new MetricQueryResponse(df.toArray(), consumedDatasetIds);
}

allowsBreakdowns(breakdowns: Breakdowns): boolean {
return breakdowns.hasExactlyOneDemographic();
}
allowsBreakdowns(breakdowns: Breakdowns): boolean {
return breakdowns.hasExactlyOneDemographic();
}
}


Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ filterwarnings = [
]

[tool.pylint."MESSAGES CONTROL"]
disable = "C0114,E0401,C0201,C0115,R1714,C0411,W0511,R1705,C0116,C0103,R0201,W0104,W0105,R0801,R0913,R0022,C0412,R0914,R0915,W0621,R0903,W0237,W0221,W0201,R0911,R1710,R0902,R0912,W0622,W1514"
disable = "C0114,E0401,C0201,C0115,R1714,C0411,W0511,R1705,C0116,C0103,R0201,W0104,W0105,R0801,R0913,R0022,C0412,R0914,R0915,W0621,R0903,W0237,W0221,W0201,R0911,R1710,R0902,R0912,W0622,W1514,W0718"

[tool.pylint."FORMAT"]
max-line-length = 120
Loading

0 comments on commit 5871b05

Please sign in to comment.