Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions f/connectors/arcgis/arcgis_download_feature_layer_anonymously.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ def main(
service_id=service_id,
feature_id=feature_id,
layer_index=li,
storage_path=storage_path,
download_attachments=download_attachments,
output_format=output_format,
storage_path=storage_path,
session=session,
)
results.append(path)
Expand Down Expand Up @@ -362,9 +362,9 @@ def fetch_layer_data(
service_id: str,
feature_id: str,
layer_index: int,
storage_path: Path,
download_attachments: bool = False,
output_format: str = "geojson",
storage_path: Optional[Path] = None,
transformer: Transformer | None = None,
session: Optional[requests.Session] = None,
) -> Path:
Expand All @@ -389,12 +389,12 @@ def fetch_layer_data(

layer_name = slugify(layer_obj.get("name", f"layer_{layer_index}"))

outputs_dir = Path("outputs") / service_id
filename = outputs_dir / f"{layer_name}.{output_format}"
filename = storage_path / f"{layer_name}.{output_format}"
relative_output = Path(storage_path.name) / f"{layer_name}.{output_format}"

if filename.exists():
logger.info("File %s already exists. Skipping download.", filename)
return filename
return relative_output

base_feature_url = f"https://{subdomain}.arcgis.com/{service_id}/arcgis/rest/services/{feature_id}/FeatureServer"

Expand All @@ -414,7 +414,7 @@ def fetch_layer_data(
save_output_geojson(geojson, filename, storage_path)

if download_attachments:
attachments_root = (storage_path or Path.cwd()) / f"{service_id}_attachments"
attachments_root = storage_path / f"{service_id}_attachments"
layer_url = f"{base_feature_url}/{layer_index}"
for rec in records:
objid = rec.get("OBJECTID") or rec.get("objectid") or rec.get("ObjectID")
Expand All @@ -432,4 +432,4 @@ def fetch_layer_data(
)

logger.info("Saved layer %s to %s", layer_name, filename)
return filename
return relative_output
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ def test_script_e2e_excel_format(arcgis_anonymous_server, tmp_path):
def test_script_e2e_csv_format(arcgis_anonymous_server, tmp_path):
"""Test downloading features as CSV format"""
asset_storage = tmp_path / "datalake"
folder_name = "arcgis_csv"

output_files = main(
subdomain=arcgis_anonymous_server.subdomain,
Expand All @@ -122,19 +123,18 @@ def test_script_e2e_csv_format(arcgis_anonymous_server, tmp_path):
layer_index_list=[0],
download_attachments=False,
output_format="csv",
folder_name="arcgis_csv",
folder_name=folder_name,
attachment_root=str(asset_storage),
)

# CSV format saves to outputs/ directory (unlike geojson which saves to storage_path)
# Returns relative path, but file is saved to storage_path
assert len(output_files) == 1
output_file = output_files[0]
assert output_file.exists()
assert output_file.suffix == ".csv"

# Clean up
output_file.unlink()
output_file.parent.rmdir()
# Verify file actually exists at the full path
expected_file = asset_storage / folder_name / "test-anonymous-layer.csv"
assert expected_file.exists()


def test_script_e2e_multiple_layers(arcgis_anonymous_server, tmp_path):
Expand Down