Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[AIRFLOW-6147] [AIP-21] Rename GoogleCloudStorageToS3Operator #6826

Merged
merged 1 commit into from
Dec 19, 2019
Merged

[AIRFLOW-6147] [AIP-21] Rename GoogleCloudStorageToS3Operator #6826

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion UPDATING.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,7 @@ The following table shows changes in import paths.
|airflow.contrib.operators.gcs_operator.GoogleCloudStorageCreateBucketOperator |airflow.gcp.operators.gcs.GoogleCloudStorageCreateBucketOperator |
|airflow.contrib.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator |airflow.operators.gcs_to_bq.GoogleCloudStorageToBigQueryOperator |
|airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator |airflow.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator |
|airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator |airflow.operators.gcs_to_s3.GoogleCloudStorageToS3Operator |
|airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator |airflow.operators.gcs_to_s3.GCSToS3Operator |
|airflow.contrib.operators.mlengine_operator.MLEngineBatchPredictionOperator |airflow.gcp.operators.mlengine.MLEngineBatchPredictionOperator |
|airflow.contrib.operators.mlengine_operator.MLEngineModelOperator |airflow.gcp.operators.mlengine.MLEngineModelOperator |
|airflow.contrib.operators.mlengine_operator.MLEngineTrainingOperator |airflow.gcp.operators.mlengine.MLEngineTrainingOperator |
Expand Down
15 changes: 13 additions & 2 deletions airflow/contrib/operators/gcs_to_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,21 @@

import warnings

# pylint: disable=unused-import
from airflow.operators.gcs_to_s3 import GoogleCloudStorageToS3Operator # noqa
from airflow.operators.gcs_to_s3 import GCSToS3Operator

warnings.warn(
"This module is deprecated. Please use `airflow.operators.gcs_to_s3`.",
DeprecationWarning, stacklevel=2
)


class GoogleCloudStorageToS3Operator(GCSToS3Operator):
"""
This class is deprecated. Please use `airflow.operators.gcs_to_s3.GCSToS3Operator`.
"""
def __init__(self, *args, **kwargs):
warnings.warn(
"This class is deprecated. Please use `airflow.operators.gcs_to_s3.GCSToS3Operator`.",
DeprecationWarning, stacklevel=2
)
super().__init__(*args, **kwargs)
2 changes: 1 addition & 1 deletion airflow/operators/gcs_to_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from airflow.utils.decorators import apply_defaults


class GoogleCloudStorageToS3Operator(GoogleCloudStorageListOperator):
class GCSToS3Operator(GoogleCloudStorageListOperator):
"""
Synchronizes a Google Cloud Storage bucket with an S3 bucket.

Expand Down
8 changes: 4 additions & 4 deletions docs/howto/define_extra_link.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ You can also add (or override) an extra link to an existing operators
through an Airflow plugin.

For example, the following Airflow plugin will add an Operator Link on all
tasks using :class:`~airflow.operators.gcs_to_s3.GoogleCloudStorageToS3Operator` operator.
tasks using :class:`~airflow.operators.gcs_to_s3.GCSToS3Operator` operator.

**Adding Operator Links to Existing Operators**
``plugins/extra_link.py``:
Expand All @@ -75,14 +75,14 @@ tasks using :class:`~airflow.operators.gcs_to_s3.GoogleCloudStorageToS3Operator`

from airflow.plugins_manager import AirflowPlugin
from airflow.models.baseoperator import BaseOperatorLink
from airflow.operators.gcs_to_s3 import GoogleCloudStorageToS3Operator
from airflow.operators.gcs_to_s3 import GCSToS3Operator

class S3LogLink(BaseOperatorLink):
name = 'S3'

# Add list of all the operators to which you want to add this OperatorLinks
# Example: operators = [GoogleCloudStorageToS3Operator, GoogleCloudStorageToBigQueryOperator]
operators = [GoogleCloudStorageToS3Operator]
# Example: operators = [GCSToS3Operator, GoogleCloudStorageToBigQueryOperator]
operators = [GCSToS3Operator]

def get_link(self, operator, dttm):
return 'https://s3.amazonaws.com/airflow-logs/{dag_id}/{task_id}/{execution_date}'.format(
Expand Down
4 changes: 2 additions & 2 deletions docs/plugins.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ definitions in Airflow.
from airflow.hooks.base_hook import BaseHook
from airflow.models import BaseOperator
from airflow.models.baseoperator import BaseOperatorLink
from airflow.operators.gcs_to_s3 import GoogleCloudStorageToS3Operator
from airflow.operators.gcs_to_s3 import GCSToS3Operator
from airflow.sensors.base_sensor_operator import BaseSensorOperator
from airflow.executors.base_executor import BaseExecutor

Expand Down Expand Up @@ -234,7 +234,7 @@ definitions in Airflow.
# buttons.
class S3LogLink(BaseOperatorLink):
name = 'S3'
operators = [GoogleCloudStorageToS3Operator]
operators = [GCSToS3Operator]

def get_link(self, operator, dttm):
return 'https://s3.amazonaws.com/airflow-logs/{dag_id}/{task_id}/{execution_date}'.format(
Expand Down
74 changes: 37 additions & 37 deletions tests/operators/test_gcs_to_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import unittest

from airflow.operators.gcs_to_s3 import GoogleCloudStorageToS3Operator
from airflow.operators.gcs_to_s3 import GCSToS3Operator
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
from tests.compat import mock

Expand All @@ -36,7 +36,7 @@
MOCK_FILES = ["TEST1.csv", "TEST2.csv", "TEST3.csv"]


class TestGoogleCloudStorageToS3Operator(unittest.TestCase):
class TestGCSToS3Operator(unittest.TestCase):

# Test1: incremental behaviour (just some files missing)
@mock_s3
Expand All @@ -47,13 +47,13 @@ def test_execute_incremental(self, mock_hook, mock_hook2):
mock_hook.return_value.download.return_value = b"testing"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you also rename TestGoogleCloudStorageToS3Operator class to Test GCSToS3Operator?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

mock_hook2.return_value.list.return_value = MOCK_FILES

operator = GoogleCloudStorageToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=False)
operator = GCSToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=False)
# create dest bucket
hook = S3Hook(aws_conn_id=None)
b = hook.get_bucket('bucket')
Expand All @@ -77,13 +77,13 @@ def test_execute_without_replace(self, mock_hook, mock_hook2):
mock_hook.return_value.download.return_value = b"testing"
mock_hook2.return_value.list.return_value = MOCK_FILES

operator = GoogleCloudStorageToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=False)
operator = GCSToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=False)
# create dest bucket with all the files
hook = S3Hook(aws_conn_id=None)
b = hook.get_bucket('bucket')
Expand All @@ -107,13 +107,13 @@ def test_execute(self, mock_hook, mock_hook2):
mock_hook.return_value.download.return_value = b"testing"
mock_hook2.return_value.list.return_value = MOCK_FILES

operator = GoogleCloudStorageToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=False)
operator = GCSToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=False)
# create dest bucket without files
hook = S3Hook(aws_conn_id=None)
b = hook.get_bucket('bucket')
Expand All @@ -136,13 +136,13 @@ def test_execute_with_replace(self, mock_hook, mock_hook2):
mock_hook.return_value.download.return_value = b"testing"
mock_hook2.return_value.list.return_value = MOCK_FILES

operator = GoogleCloudStorageToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=True)
operator = GCSToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=True)
# create dest bucket with all the files
hook = S3Hook(aws_conn_id=None)
b = hook.get_bucket('bucket')
Expand All @@ -166,13 +166,13 @@ def test_execute_incremental_with_replace(self, mock_hook, mock_hook2):
mock_hook.return_value.download.return_value = b"testing"
mock_hook2.return_value.list.return_value = MOCK_FILES

operator = GoogleCloudStorageToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=True)
operator = GCSToS3Operator(task_id=TASK_ID,
bucket=GCS_BUCKET,
prefix=PREFIX,
delimiter=DELIMITER,
dest_aws_conn_id=None,
dest_s3_key=S3_BUCKET,
replace=True)
# create dest bucket with just two files (the first two files in MOCK_FILES)
hook = S3Hook(aws_conn_id=None)
b = hook.get_bucket('bucket')
Expand Down
2 changes: 1 addition & 1 deletion tests/test_core_to_contrib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -572,7 +572,7 @@
"airflow.contrib.operators.gcs_to_gcs.GoogleCloudStorageToGoogleCloudStorageOperator",
),
(
"airflow.operators.gcs_to_s3.GoogleCloudStorageToS3Operator",
"airflow.operators.gcs_to_s3.GCSToS3Operator",
"airflow.contrib.operators.gcs_to_s3.GoogleCloudStorageToS3Operator",
),
(
Expand Down