Skip to content

Commit

Permalink
[Insights] Update endpoint names; correct file path for upload destin…
Browse files Browse the repository at this point in the history
…ation (Recidiviz/recidiviz-data#29462)

## Description of the change

The filepath for the uploaded destination was incorrect because the
`_outliers_bucket()` value already includes the `gs://` prefix. Also
updates naming conventions for the JSON handler and updatees test to
account for both of these changes.

## Type of change

> All pull requests must have at least one of the following labels
applied (otherwise the PR will fail):

| Label | Description |
|-----------------------------
|-----------------------------------------------------------------------------------------------------------
|
| Type: Bug | non-breaking change that fixes an issue |
| Type: Feature | non-breaking change that adds functionality |
| Type: Breaking Change | fix or feature that would cause existing
functionality to not work as expected |
| Type: Non-breaking refactor | change addresses some tech debt item or
prepares for a later change, but does not change functionality |
| Type: Configuration Change | adjusts configuration to achieve some end
related to functionality, development, performance, or security |
| Type: Dependency Upgrade | upgrades a project dependency - these
changes are not included in release notes |

## Related issues

Closes Recidiviz/recidiviz-data#29449

## Checklists

### Development

**This box MUST be checked by the submitter prior to merging**:
- [ ] **Double- and triple-checked that there is no Personally
Identifiable Information (PII) being mistakenly added in this pull
request**

These boxes should be checked by the submitter prior to merging:
- [ ] Tests have been written to cover the code changed/added as part of
this pull request

### Code review

These boxes should be checked by reviewers prior to merging:

- [ ] This pull request has a descriptive title and information useful
to a reviewer
- [ ] Potential security implications or infrastructural changes have
been considered, if relevant

GitOrigin-RevId: b59e83dd448ae94ba52ca615e9e1232f617191df
  • Loading branch information
alexabatino authored and Helper Bot committed May 15, 2024
1 parent 36fa2aa commit 86ed1ff
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 8 deletions.
10 changes: 6 additions & 4 deletions recidiviz/application_data_import/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,8 +252,8 @@ def _import_trigger_outliers() -> Tuple[str, HTTPStatus]:
message=message,
gcs_bucket=gcs_bucket,
import_queue_name=OUTLIERS_DB_IMPORT_QUEUE,
task_prefix="import-outliers-utils",
task_url="/import/outliers/utils",
task_prefix="import-outliers-json-to-csv",
task_url="/import/outliers/json_to_csv",
)
else:
error_msg = f"Unexpected handling of file type .{file_type} for file {file}"
Expand Down Expand Up @@ -410,7 +410,7 @@ def _import_outliers(state_code: str, filename: str) -> Tuple[str, HTTPStatus]:
return "", HTTPStatus.OK


@app.route("/import/outliers/utils/<state_code>/<filename>", methods=["POST"])
@app.route("/import/outliers/json_to_csv/<state_code>/<filename>", methods=["POST"])
def _import_outliers_convert_json_to_csv(
state_code: str, filename: str
) -> Tuple[str, HTTPStatus]:
Expand Down Expand Up @@ -473,9 +473,11 @@ def _import_outliers_convert_json_to_csv(
)

destination_path = GcsfsFilePath.from_absolute_path(
f"gs://{_outliers_bucket()}/{state_code.upper()}/{file_name}.csv"
f"{_outliers_bucket()}/{state_code.upper()}/{file_name}.csv"
)

logging.info("Uploading DataFrame as CSV to %s", destination_path)

gcsfs.upload_from_string(
path=destination_path,
contents=df.to_csv(
Expand Down
20 changes: 16 additions & 4 deletions recidiviz/tests/application_data_import/server_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,9 +491,9 @@ def test_import_trigger_outliers_json(self, mock_task_manager: MagicMock) -> Non
self.assertEqual(HTTPStatus.OK, response.status_code)

mock_task_manager.return_value.create_task.assert_called_with(
absolute_uri=f"http://localhost:5000/import/outliers/utils/{self.state_code}/test-file.json",
absolute_uri=f"http://localhost:5000/import/outliers/json_to_csv/{self.state_code}/test-file.json",
service_account_email="fake-acct@fake-project.iam.gserviceaccount.com",
task_id=f"import-outliers-utils-{self.state_code}-test-file-json",
task_id=f"import-outliers-json-to-csv-{self.state_code}-test-file-json",
)

def test_import_trigger_outliers_bad_message(self) -> None:
Expand Down Expand Up @@ -645,7 +645,7 @@ def test_import_outliers_invalid_table(self, mock_get_database: MagicMock) -> No
response.data,
)

def test_import_outliers_utils_successful(
def test_import_outliers_json_to_csv_successful(
self,
) -> None:
with self.app.test_request_context():
Expand All @@ -667,8 +667,20 @@ def test_import_outliers_utils_successful(
)
self.assertEqual(len(self.fs.files), 1)
response = self.client.post(
f"/import/outliers/utils/{self.state_code}/{self.view}.json",
f"/import/outliers/json_to_csv/{self.state_code}/{filename}",
)

csv_path = GcsfsFilePath.from_absolute_path(
os.path.join(
self.bucket,
self.state_code + "/" + f"{self.view}.csv",
)
)

self.assertEqual(
csv_path.uri(),
"gs://test-project-outliers-etl-data/US_XX/supervision_officers.csv",
)
self.assertTrue(self.fs.exists(csv_path))
self.assertEqual(HTTPStatus.OK, response.status_code)
self.assertEqual(len(self.fs.files), 2)

0 comments on commit 86ed1ff

Please sign in to comment.