Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Use proper GCS prefix for custom data folder #408

Merged
merged 1 commit into from
Jul 11, 2022
Merged

fix: Use proper GCS prefix for custom data folder #408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 10 additions & 27 deletions scripts/deploy_dag.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ def main(
data_folder = (
DATASETS_PATH / dataset_id / "pipelines" / pipeline_path.name / "data"
)
if data_folder.exists() and data_folder.is_dir():

if data_folder.exists() and data_folder.is_dir() and any(data_folder.iterdir()):
copy_data_folder_to_composer_bucket(
dataset_id,
data_folder,
Expand Down Expand Up @@ -301,33 +302,15 @@ def copy_data_folder_to_composer_bucket(
pipeline: str,
composer_bucket: str,
):
gcs_uri = f"gs://{composer_bucket}/data/{dataset_id}/pipeline/{pipeline}/data"
schema_file_dir_pattern = "*"
schema_file_dir = []

schema_file_dir = sorted(data_folder.rglob(schema_file_dir_pattern))
"""
[remote]
gsutil cp * gs://{composer_bucket}/data/{dataset_id}/pipeline/{pipeline}
cd .{ENV}/datasets/{dataset_id}/data
"""

if len(schema_file_dir) > 0:
print(
"\nCopying files from local data folder into Cloud Composer data folder\n"
)

for file in data_folder.iterdir():
print(" Source:\n")
print(" " + str(file) + "\n")
print(" Destination:\n")
print(" " + gcs_uri + "/" + str(file).split("/data/")[1] + "\n")
print(
f"Data folder exists: {data_folder}.\nCopying contents into Composer bucket.."
)
gcs_uri = f"gs://{composer_bucket}/data/{dataset_id}/{pipeline}"
run_gsutil_cmd(["-q", "cp", "-r", f"{data_folder}/*", gcs_uri], data_folder)

run_gsutil_cmd(["cp", schema_file_dir_pattern, gcs_uri], data_folder)
else:
print(
"\n No files in local data folder to copy into Cloud Composer data folder \n"
)
print("Done. Files uploaded to GCS:")
for file in data_folder.iterdir():
print(f" - {gcs_uri}/{file.name}")


def run_cloud_composer_vars_import(
Expand Down
3 changes: 2 additions & 1 deletion tests/scripts/test_deploy_dag.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -720,7 +720,8 @@ def test_script_copy_files_in_data_folder_to_composer_with_folder_created(

data_folder = pipeline_path / "data"
data_folder.mkdir(parents=True)
assert data_folder.exists() and data_folder.is_dir()
(data_folder / "test_file.txt").touch()
assert data_folder.exists() and data_folder.is_dir() and any(data_folder.iterdir())

airflow_version = 2
mocker.patch("scripts.deploy_dag.check_and_configure_airflow_variables")
Expand Down