refactor.

datagovharvester/extract/utils.py

@@ -1,10 +1,7 @@
 import requests
 import json
-import os
 from datagovharvester.utils.s3_utilities import (
     upload_dcatus_to_S3,
-    create_s3_client,
-    create_bucket,
     create_s3_payload,
 )
 
@@ -34,19 +31,14 @@ def fetch_url(url):
     return data, success
 
 
-def extract_catalog(url, S3_config, job_id):
+def extract_catalog(url, job_id, S3_client, bucket_name):
     data, fetch_success = fetch_url(url)
 
-    S3, S3_create_error_msg = create_s3_client(S3_config)
-
-    bucket_name = os.getenv("S3FILESTORE__AWS_BUCKET_NAME")
-    bucket_name, bucket_error_msg = create_bucket(S3, bucket_name)
-
     for idx, record in enumerate(data["dataset"]):
         record = json.dumps(record)
         key_name = f"{job_id}_{idx}_extract.json"
         s3_payload = create_s3_payload(record, bucket_name, key_name)
 
-        upload_data, upload_error_message = upload_dcatus_to_S3(S3, s3_payload)
+        upload_data, upload_error_message = upload_dcatus_to_S3(S3_client, s3_payload)
 
     return upload_data, upload_error_message

datagovharvester/utils/s3_utilities.py

@@ -13,7 +13,7 @@ def create_s3_client(s3_config):
     return S3, error_message
 
 
-def create_bucket(S3, bucket_name):
+def create_s3_bucket(S3, bucket_name):
     error_message = None
 
     try:

tests/extract/conftest.py

@@ -2,6 +2,7 @@
 from dotenv import load_dotenv
 import os
 from uuid import uuid4
+from datagovharvester.utils.s3_utilities import create_s3_client, create_s3_bucket
 
 
 @pytest.fixture
@@ -14,7 +15,7 @@ def get_dcatus_job():
 
 
 @pytest.fixture
-def create_s3_client_config():
+def create_client_config():
     config = {}
     load_dotenv()
 
@@ -24,3 +25,14 @@ def create_s3_client_config():
     config["endpoint_url"] = os.getenv("S3FILESTORE__HOST_NAME")
 
     return config
+
+
+@pytest.fixture
+def create_client(create_client_config):
+    return create_s3_client(create_client_config)
+
+
+@pytest.fixture
+def create_bucket(create_client):
+    s3_client, s3_client_msg = create_client
+    return create_s3_bucket(s3_client, "test-bucket")

tests/extract/test_dcatus.py

@@ -1,17 +1,17 @@
 from datagovharvester.extract.utils import extract_catalog
 
 
-def test_extract_dcatus(create_s3_client_config, get_dcatus_job):
+def test_extract_dcatus(get_dcatus_job, create_client, create_bucket):
     success = False
 
     url = get_dcatus_job["url"]
     job_id = get_dcatus_job["job_id"]
 
-    S3_config = create_s3_client_config
+    S3_client, S3_msg = create_client
+    bucket_name, bucket_msg = create_bucket
 
-    final_res, final_error_msg = extract_catalog(url, S3_config, job_id)
+    final_res, final_error_msg = extract_catalog(url, job_id, S3_client, bucket_name)
 
-    print(final_error_msg)
     if final_error_msg is None:
         success = True