From 4a46905cb9352fb658ffef594deb2705fb4b025f Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 5 Jul 2023 16:17:04 +0530 Subject: [PATCH 1/3] Migrating Google AutoML example_dags to sys tests --- .../example_automl_nl_text_classification.py | 28 ++++++++++------- .../example_automl_nl_text_sentiment.py | 27 ++++++++++------ .../automl}/example_automl_translation.py | 31 ++++++++++++------- ...utoml_video_intelligence_classification.py | 29 ++++++++++------- ...mple_automl_video_intelligence_tracking.py | 29 ++++++++++------- .../example_automl_vision_object_detection.py | 27 +++++++++------- 6 files changed, 103 insertions(+), 68 deletions(-) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_nl_text_classification.py (82%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_nl_text_sentiment.py (82%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_translation.py (82%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_video_intelligence_classification.py (82%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_video_intelligence_tracking.py (81%) rename {airflow/providers/google/cloud/example_dags => tests/system/providers/google/cloud/automl}/example_automl_vision_object_detection.py (82%) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py similarity index 82% rename from airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py rename to tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py index afe28a9ff82ec..97f54cec02c11 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py @@ -35,17 +35,14 @@ AutoMLTrainModelOperator, ) -GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") -GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_TEXT_CLS_BUCKET = os.environ.get("GCP_AUTOML_TEXT_CLS_BUCKET", "gs://INVALID BUCKET NAME") - -# Example values -DATASET_ID = "" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_automl_classification" +GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +GCP_AUTOML_LOCATION = "us-central1" # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "text_classification_model_metadata": {}, } @@ -55,7 +52,10 @@ "text_classification_dataset_metadata": {"classification_type": "MULTICLASS"}, } -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TEXT_CLS_BUCKET]}} + +DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/dataset.csv" +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id @@ -65,12 +65,13 @@ start_date=datetime(2021, 1, 1), catchup=False, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -78,11 +79,9 @@ location=GCP_AUTOML_LOCATION, input_config=IMPORT_INPUT_CONFIG, ) - MODEL["dataset_id"] = dataset_id create_model = AutoMLTrainModelOperator(task_id="create_model", model=MODEL, location=GCP_AUTOML_LOCATION) - model_id = cast(str, XComArg(create_model, key="model_id")) delete_model_task = AutoMLDeleteModelOperator( @@ -99,10 +98,17 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: # create_dataset_task >> import_dataset_task # create_dataset_task >> create_model # create_dataset_task >> delete_datasets_task + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py similarity index 82% rename from airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py rename to tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py index a823b8af89538..d9fc64b9a9eb4 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_nl_text_sentiment.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py @@ -35,17 +35,14 @@ AutoMLTrainModelOperator, ) -GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") -GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_SENTIMENT_BUCKET = os.environ.get("GCP_AUTOML_SENTIMENT_BUCKET", "gs://INVALID BUCKET NAME") - -# Example values -DATASET_ID = "" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_automl_text_sentiment" +GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +GCP_AUTOML_LOCATION = "us-central1" # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "text_sentiment_model_metadata": {}, } @@ -55,23 +52,26 @@ "text_sentiment_dataset_metadata": {"sentiment_max": 10}, } -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_SENTIMENT_BUCKET]}} +DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/dataset.csv" +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Natural Language Text Sentiment with models.DAG( - "example_automl_text_sentiment", + DAG_ID, start_date=datetime(2021, 1, 1), catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -100,7 +100,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -108,3 +110,8 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_translation.py b/tests/system/providers/google/cloud/automl/example_automl_translation.py similarity index 82% rename from airflow/providers/google/cloud/example_dags/example_automl_translation.py rename to tests/system/providers/google/cloud/automl/example_automl_translation.py index 2bef20caabee3..77820a1a61926 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_translation.py +++ b/tests/system/providers/google/cloud/automl/example_automl_translation.py @@ -35,19 +35,14 @@ AutoMLTrainModelOperator, ) -GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") -GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_TRANSLATION_BUCKET = os.environ.get( - "GCP_AUTOML_TRANSLATION_BUCKET", "gs://INVALID BUCKET NAME/file" -) - -# Example values -DATASET_ID = "TRL123456789" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_automl_translation" +GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +GCP_AUTOML_LOCATION = "us-central1" # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "translation_model_metadata": {}, } @@ -60,19 +55,23 @@ }, } -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TRANSLATION_BUCKET]}} + +DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/file" +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Translation with models.DAG( - "example_automl_translation", + DAG_ID, start_date=datetime(2021, 1, 1), + schedule="@once", catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) @@ -106,7 +105,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -114,3 +115,9 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py similarity index 82% rename from airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py rename to tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py index 2b55c42a8ab96..eaaf6da22839e 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py @@ -35,19 +35,14 @@ AutoMLTrainModelOperator, ) -GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") -GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_VIDEO_BUCKET = os.environ.get( - "GCP_AUTOML_VIDEO_BUCKET", "gs://INVALID BUCKET NAME/hmdb_split1.csv" -) - -# Example values -DATASET_ID = "VCN123455678" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_automl_video" +GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +GCP_AUTOML_LOCATION = "us-central1" # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "video_classification_model_metadata": {}, } @@ -57,24 +52,27 @@ "video_classification_dataset_metadata": {}, } -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_VIDEO_BUCKET]}} +DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/hmdb_split1.csv" +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Video Intelligence Classification with models.DAG( - "example_automl_video", + DAG_ID, start_date=datetime(2021, 1, 1), catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -103,7 +101,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -111,3 +111,8 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py similarity index 81% rename from airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py rename to tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py index daed5748b2399..c0ea0d48e695b 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_video_intelligence_tracking.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py @@ -35,20 +35,15 @@ AutoMLTrainModelOperator, ) -GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") -GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_TRACKING_BUCKET = os.environ.get( - "GCP_AUTOML_TRACKING_BUCKET", - "gs://INVALID BUCKET NAME/youtube_8m_videos_animal_tiny.csv", -) +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_automl_video_tracking" +GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +GCP_AUTOML_LOCATION = "us-central1" -# Example values -DATASET_ID = "VOT123456789" # Example model MODEL = { "display_name": "auto_model_1", - "dataset_id": DATASET_ID, "video_object_tracking_model_metadata": {}, } @@ -58,24 +53,27 @@ "video_object_tracking_dataset_metadata": {}, } -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TRACKING_BUCKET]}} +DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/youtube_8m_videos_animal_tiny.csv" +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Video Intelligence Object Tracking with models.DAG( - "example_automl_video_tracking", + DAG_ID, start_date=datetime(2021, 1, 1), catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -104,7 +102,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -112,3 +112,8 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) diff --git a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py similarity index 82% rename from airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py rename to tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py index 8b9ae271d1126..757ad68234c12 100644 --- a/airflow/providers/google/cloud/example_dags/example_automl_vision_object_detection.py +++ b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py @@ -35,19 +35,14 @@ AutoMLTrainModelOperator, ) -GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") -GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") -GCP_AUTOML_DETECTION_BUCKET = os.environ.get( - "GCP_AUTOML_DETECTION_BUCKET", "gs://INVALID BUCKET NAME/img/openimage/csv/salads_ml_use.csv" -) - -# Example values -DATASET_ID = "" +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_automl_vision_detection" +GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") +GCP_AUTOML_LOCATION = "us-central1" # Example model MODEL = { "display_name": "auto_model", - "dataset_id": DATASET_ID, "image_object_detection_model_metadata": {}, } @@ -57,7 +52,9 @@ "image_object_detection_dataset_metadata": {}, } -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_DETECTION_BUCKET]}} +DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" +AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/csv/salads_ml_use.csv" +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id @@ -69,12 +66,13 @@ catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, tags=["example"], -) as example_dag: +) as dag: create_dataset_task = AutoMLCreateDatasetOperator( task_id="create_dataset_task", dataset=DATASET, location=GCP_AUTOML_LOCATION ) dataset_id = cast(str, XComArg(create_dataset_task, key="dataset_id")) + MODEL["dataset_id"] = dataset_id import_dataset_task = AutoMLImportDataOperator( task_id="import_dataset_task", @@ -103,7 +101,9 @@ project_id=GCP_PROJECT_ID, ) + # TEST BODY import_dataset_task >> create_model + # TEST TEARDOWN delete_model_task >> delete_datasets_task # Task dependencies created via `XComArgs`: @@ -111,3 +111,8 @@ # create_dataset_task >> create_model # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From 9c0690e5cdbfed97962fe9ed2bd1bbc0cf4bf5ed Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Thu, 6 Jul 2023 11:33:38 +0530 Subject: [PATCH 2/3] handling review comments from Adaverse --- .../example_automl_nl_text_classification.py | 11 ++++------- .../automl/example_automl_nl_text_sentiment.py | 13 +++++-------- .../cloud/automl/example_automl_translation.py | 15 +++++++-------- ...e_automl_video_intelligence_classification.py | 15 +++++++-------- ...example_automl_video_intelligence_tracking.py | 16 ++++++++-------- .../example_automl_vision_object_detection.py | 13 ++++++------- 6 files changed, 37 insertions(+), 46 deletions(-) diff --git a/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py index 97f54cec02c11..4b24534e2a81d 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py @@ -35,10 +35,9 @@ AutoMLTrainModelOperator, ) -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "example_automl_classification" -GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") -GCP_AUTOML_LOCATION = "us-central1" +GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") +GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") +GCP_AUTOML_TEXT_CLS_BUCKET = os.environ.get("GCP_AUTOML_TEXT_CLS_BUCKET", "gs://INVALID BUCKET NAME") # Example model MODEL = { @@ -53,9 +52,7 @@ } -DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/dataset.csv" -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TEXT_CLS_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id diff --git a/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py index d9fc64b9a9eb4..e17d8cf927ee0 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py @@ -35,10 +35,9 @@ AutoMLTrainModelOperator, ) -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "example_automl_text_sentiment" -GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") -GCP_AUTOML_LOCATION = "us-central1" +GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") +GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") +GCP_AUTOML_SENTIMENT_BUCKET = os.environ.get("GCP_AUTOML_SENTIMENT_BUCKET", "gs://INVALID BUCKET NAME") # Example model MODEL = { @@ -52,15 +51,13 @@ "text_sentiment_dataset_metadata": {"sentiment_max": 10}, } -DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/dataset.csv" -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_SENTIMENT_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Natural Language Text Sentiment with models.DAG( - DAG_ID, + "example_automl_text_sentiment", start_date=datetime(2021, 1, 1), catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, diff --git a/tests/system/providers/google/cloud/automl/example_automl_translation.py b/tests/system/providers/google/cloud/automl/example_automl_translation.py index 77820a1a61926..6b0686fed9aaf 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_translation.py +++ b/tests/system/providers/google/cloud/automl/example_automl_translation.py @@ -35,10 +35,11 @@ AutoMLTrainModelOperator, ) -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "example_automl_translation" -GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") -GCP_AUTOML_LOCATION = "us-central1" +GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") +GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") +GCP_AUTOML_TRANSLATION_BUCKET = os.environ.get( + "GCP_AUTOML_TRANSLATION_BUCKET", "gs://INVALID BUCKET NAME/file" +) # Example model MODEL = { @@ -56,16 +57,14 @@ } -DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/file" -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TRANSLATION_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Translation with models.DAG( - DAG_ID, + "example_automl_translation", start_date=datetime(2021, 1, 1), schedule="@once", catchup=False, diff --git a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py index eaaf6da22839e..971a75e03927d 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py @@ -35,10 +35,11 @@ AutoMLTrainModelOperator, ) -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "example_automl_video" -GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") -GCP_AUTOML_LOCATION = "us-central1" +GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") +GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") +GCP_AUTOML_VIDEO_BUCKET = os.environ.get( + "GCP_AUTOML_VIDEO_BUCKET", "gs://INVALID BUCKET NAME/hmdb_split1.csv" +) # Example model MODEL = { @@ -52,16 +53,14 @@ "video_classification_dataset_metadata": {}, } -DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/hmdb_split1.csv" -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_VIDEO_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Video Intelligence Classification with models.DAG( - DAG_ID, + "example_automl_video", start_date=datetime(2021, 1, 1), catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, diff --git a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py index c0ea0d48e695b..e8a5700e199b8 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py @@ -35,10 +35,12 @@ AutoMLTrainModelOperator, ) -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "example_automl_video_tracking" -GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") -GCP_AUTOML_LOCATION = "us-central1" +GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") +GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") +GCP_AUTOML_TRACKING_BUCKET = os.environ.get( + "GCP_AUTOML_TRACKING_BUCKET", + "gs://INVALID BUCKET NAME/youtube_8m_videos_animal_tiny.csv", +) # Example model @@ -53,16 +55,14 @@ "video_object_tracking_dataset_metadata": {}, } -DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/youtube_8m_videos_animal_tiny.csv" -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_TRACKING_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id # Example DAG for AutoML Video Intelligence Object Tracking with models.DAG( - DAG_ID, + "example_automl_video_tracking", start_date=datetime(2021, 1, 1), catchup=False, user_defined_macros={"extract_object_id": extract_object_id}, diff --git a/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py index 757ad68234c12..c51e1602561b1 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py +++ b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py @@ -35,10 +35,11 @@ AutoMLTrainModelOperator, ) -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "example_automl_vision_detection" -GCP_PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "default") -GCP_AUTOML_LOCATION = "us-central1" +GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "your-project-id") +GCP_AUTOML_LOCATION = os.environ.get("GCP_AUTOML_LOCATION", "us-central1") +GCP_AUTOML_DETECTION_BUCKET = os.environ.get( + "GCP_AUTOML_DETECTION_BUCKET", "gs://INVALID BUCKET NAME/img/openimage/csv/salads_ml_use.csv" +) # Example model MODEL = { @@ -52,9 +53,7 @@ "image_object_detection_dataset_metadata": {}, } -DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}" -AUTOML_DATASET_BUCKET = f"gs://{DATA_SAMPLE_GCS_BUCKET_NAME}/automl-text/csv/salads_ml_use.csv" -IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [AUTOML_DATASET_BUCKET]}} +IMPORT_INPUT_CONFIG = {"gcs_source": {"input_uris": [GCP_AUTOML_DETECTION_BUCKET]}} extract_object_id = CloudAutoMLHook.extract_object_id From 3c446e59bcdc24405665ff7804fa56c79c63b3fe Mon Sep 17 00:00:00 2001 From: Amogh Date: Fri, 7 Jul 2023 15:42:03 +0530 Subject: [PATCH 3/3] adding systest watchers --- .../cloud/automl/example_automl_nl_text_classification.py | 6 ++++++ .../google/cloud/automl/example_automl_nl_text_sentiment.py | 6 ++++++ .../google/cloud/automl/example_automl_translation.py | 6 ++++++ .../example_automl_video_intelligence_classification.py | 6 ++++++ .../automl/example_automl_video_intelligence_tracking.py | 6 ++++++ .../cloud/automl/example_automl_vision_object_detection.py | 6 ++++++ 6 files changed, 36 insertions(+) diff --git a/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py index 4b24534e2a81d..39129162097e0 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_classification.py @@ -105,6 +105,12 @@ # create_dataset_task >> create_model # create_dataset_task >> delete_datasets_task + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + from tests.system.utils import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) diff --git a/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py index e17d8cf927ee0..6bbe0d2bfbea8 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py +++ b/tests/system/providers/google/cloud/automl/example_automl_nl_text_sentiment.py @@ -108,6 +108,12 @@ # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + from tests.system.utils import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) diff --git a/tests/system/providers/google/cloud/automl/example_automl_translation.py b/tests/system/providers/google/cloud/automl/example_automl_translation.py index 6b0686fed9aaf..87bf7166dcd1f 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_translation.py +++ b/tests/system/providers/google/cloud/automl/example_automl_translation.py @@ -115,6 +115,12 @@ # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + from tests.system.utils import get_test_run # noqa: E402 diff --git a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py index 971a75e03927d..63447a123059d 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_classification.py @@ -111,6 +111,12 @@ # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + from tests.system.utils import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) diff --git a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py index e8a5700e199b8..5ac8fa7457fa6 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py +++ b/tests/system/providers/google/cloud/automl/example_automl_video_intelligence_tracking.py @@ -113,6 +113,12 @@ # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + from tests.system.utils import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) diff --git a/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py index c51e1602561b1..933c2760cbbbd 100644 --- a/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py +++ b/tests/system/providers/google/cloud/automl/example_automl_vision_object_detection.py @@ -111,6 +111,12 @@ # create_model >> delete_model_task # create_dataset_task >> delete_datasets_task + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + from tests.system.utils import get_test_run # noqa: E402 # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)