Task/set fixed dc version (#3)

* FIX set fixed dc version

.circleci/config.yml

@@ -6,7 +6,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:3.7
+      - image: circleci/python:3.7.4
 
     working_directory: ~/repo
 

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name='datacatalog-fileset-enricher',
-    version='1.1.1',
+    version='1.1.2',
     author='Marcelo Costa',
     author_email='mesmacosta@gmail.com',
     description='A package for enriching the content of a fileset Entry with Datacatalog Tags',
@@ -18,7 +18,7 @@
     install_requires=(
         'pandas',
         'google-cloud-storage',
-        'google-cloud-datacatalog',
+        'google-cloud-datacatalog>=1,<2',
     ),
     setup_requires=(
         'flake8',

src/datacatalog_fileset_enricher/datacatalog_fileset_enricher.py

@@ -34,7 +34,7 @@ def __init__(self, project_id):
         self.__project_id = project_id
 
     def create_template(self, location):
-        logging.info(f'===> Create Template started')
+        logging.info('===> Create Template started')
 
         tag_template_name = self.__dacatalog_helper.get_tag_template_name(location=location)
 
@@ -47,19 +47,19 @@ def create_template(self, location):
         logging.info('==== DONE ==================================================')
 
     def clean_up_all(self):
-        logging.info(f'===> Clean up started')
+        logging.info('===> Clean up started')
 
         self.__dacatalog_helper.delete_tag_template()
-        logging.info(f'Template and Tags deleted...')
+        logging.info('Template and Tags deleted...')
 
         self.__dacatalog_helper.delete_entries_and_entry_groups()
         logging.info('==== DONE ==================================================')
 
     def clean_up_fileset_template_and_tags(self):
-        logging.info(f'===> Clean up started')
+        logging.info('===> Clean up started')
 
         self.__dacatalog_helper.delete_tag_template()
-        logging.info(f'Template and Tags deleted...')
+        logging.info('Template and Tags deleted...')
 
     def run(self,
             entry_group_id=None,
@@ -96,7 +96,7 @@ def enrich_datacatalog_fileset_entry(self,
         logging.info(f'[LOCATION: {location}]')
         logging.info(f'[ENTRY_GROUP: {entry_group_id}]')
         logging.info(f'[ENTRY: {entry_id}]')
-        logging.info(f'===> Enrich Fileset Entry metadata with tags')
+        logging.info('===> Enrich Fileset Entry metadata with tags')
         logging.info('')
         logging.info('===> Get Entry from DataCatalog...')
         entry = self.__dacatalog_helper.get_entry(location, entry_group_id, entry_id)

src/datacatalog_fileset_enricher/datacatalog_helper.py

@@ -301,7 +301,7 @@ def synchronize_entry_tags(self, entry, updated_tags):
                 self.__datacatalog.update_tag(tag=tag_to_update, update_mask=None)
                 logging.info(f'Tag updated: {tag_to_update.name}')
             else:
-                logging.info(f'Tag is up to date')
+                logging.info('Tag is up to date')
 
     @classmethod
     def __tags_fields_are_equal(cls, tag_1, tag_2):

tests/datacatalog_fileset_enricher/datacatalog_fileset_enricher_test.py

@@ -182,7 +182,8 @@ def test_run_given_bucket_with_wildcard_should_call_retrieve_multiple_buckets(
     def test_run_given_bucket_with_wildcard_and_multiple_gcs_patterns_should_call_retrieve_multiple_buckets(  # noqa: E501
         self, get_manually_created_fileset_entries, get_entry, parse_gcs_file_patterns,
         create_filtered_data_for_single_bucket, create_filtered_data_for_multiple_buckets,
-        create_stats_from_dataframe, create_tag_from_stats):
+        create_stats_from_dataframe, create_tag_from_stats):  # noqa:E125
+
         entry = self.__make_fake_fileset_entry()
 
         entry.gcs_fileset_spec.file_patterns.append('gs://my_bucket*/*csv')

tests/datacatalog_fileset_enricher/datacatalog_helper_test.py

@@ -144,7 +144,7 @@ def test_create_tag_template_should_create_all_fields(self, create_tag_template)
            'create_fileset_enricher_tag_template')
     def test_create_tag_with_no_files_found_should_not_raise_error(
         self, create_fileset_enricher_tag_template, get_fileset_enricher_tag_template, list_tags,
-        create_tag):  # noqa: E501
+        create_tag):  # noqa
 
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
@@ -178,7 +178,7 @@ def test_create_tag_with_no_files_found_should_not_raise_error(
            'create_fileset_enricher_tag_template')
     def test_create_tag_with_files_found_should_not_raise_error(
         self, create_fileset_enricher_tag_template, get_fileset_enricher_tag_template, list_tags,
-        create_tag):  # noqa: E501
+        create_tag):  # noqa
 
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
@@ -209,7 +209,8 @@ def test_create_tag_with_bucket_prefix_should_not_raise_error(
         create_fileset_enricher_tag_template,
         get_fileset_enricher_tag_template,
         list_tags,  # noqa: E125
-        create_tag):
+        create_tag):  # noqa:E125
+
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
         entry.name = 'fileset_entry'
@@ -240,7 +241,8 @@ def test_create_tag_non_existent_template_should_not_raise_error(
         create_fileset_enricher_tag_template,
         get_fileset_enricher_tag_template,
         list_tags,  # noqa: E125
-        create_tag):
+        create_tag):  # noqa:E125
+
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
         entry.name = 'fileset_entry'
@@ -272,7 +274,8 @@ def test_create_tag_providing_tag_fields_should_filter_fields(
         create_fileset_enricher_tag_template,
         get_fileset_enricher_tag_template,
         list_tags,  # noqa: E125
-        create_tag):
+        create_tag):  # noqa:E125
+
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
         entry.name = 'fileset_entry'
@@ -324,7 +327,8 @@ def test_create_tag_providing_tag_fields_with_key_error_should_leak_the_error(
         create_fileset_enricher_tag_template,
         get_fileset_enricher_tag_template,
         list_tags,  # noqa: E125
-        create_tag):
+        create_tag):  # noqa:E125
+
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
         entry.name = 'fileset_entry'
@@ -375,7 +379,7 @@ def test_create_tag_providing_tag_fields_with_key_error_should_leak_the_error(
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.delete_entry')
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.delete_entry_group')
     def test_delete_entries_and_entry_groups_should_successfully_delete_them(
-        self, delete_entry_group, delete_entry, search_catalog):  # noqa: E501
+        self, delete_entry_group, delete_entry, search_catalog):  # noqa
 
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
@@ -402,7 +406,7 @@ def test_delete_entries_and_entry_groups_should_successfully_delete_them(
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.delete_entry')
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.delete_entry_group')
     def test_delete_entries_error_on_delete_entry_should_not_leak_error(
-        self, delete_entry_group, delete_entry, search_catalog):  # noqa: E501
+        self, delete_entry_group, delete_entry, search_catalog):  # noqa
 
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
@@ -431,7 +435,7 @@ def test_delete_entries_error_on_delete_entry_should_not_leak_error(
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.delete_entry')
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.delete_entry_group')
     def test_delete_entries_error_on_delete_entry_group_should_not_leak_error(
-        self, delete_entry_group, delete_entry, search_catalog):  # noqa: E501
+        self, delete_entry_group, delete_entry, search_catalog):  # noqa
 
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()
@@ -481,7 +485,7 @@ def test_synchronize_entries_tags_should_update_tag_on_changes(self, list_tags,
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.create_tag')
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.list_tags')
     def test_synchronize_entries_tags_should_not_update_tag_when_no_changes(
-        self, list_tags, create_tag, update_tag):  # noqa: E501
+        self, list_tags, create_tag, update_tag):  # noqa
 
         updated_tag = self.__make_fake_tag()
         current_tag = self.__make_fake_tag()
@@ -502,7 +506,7 @@ def test_synchronize_entries_tags_should_not_update_tag_when_no_changes(
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.create_tag')
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.list_tags')
     def test_synchronize_entries_tags_should_create_tag_when_tag_is_new(
-        self, list_tags, create_tag, update_tag):  # noqa: E501
+        self, list_tags, create_tag, update_tag):  # noqa
 
         updated_tag = self.__make_fake_tag()
 
@@ -522,7 +526,7 @@ def test_synchronize_entries_tags_should_create_tag_when_tag_is_new(
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.create_tag')
     @patch('google.cloud.datacatalog_v1.DataCatalogClient.list_tags')
     def test_synchronize_entries_tags_should_do_nothing_when_no_tags_are_provided(
-        self, list_tags, create_tag, update_tag):  # noqa: E501
+        self, list_tags, create_tag, update_tag):  # noqa
 
         datacatalog_helper = DataCatalogHelper('test_project')
         entry = MockedObject()

tests/datacatalog_fileset_enricher/gcs_storage_filter_test.py

@@ -15,7 +15,7 @@ class StorageFilterTestCase(TestCase):
     @patch('datacatalog_fileset_enricher.gcs_storage_client_helper.StorageClientHelper.list_blobs')
     @patch('datacatalog_fileset_enricher.gcs_storage_client_helper.StorageClientHelper.get_bucket')
     def test_create_filtered_data_for_multiple_buckets_with_a_matching_bucket_should_create_filtered_data(  # noqa: E501
-        self, get_bucket, list_blobs, list_buckets):
+        self, get_bucket, list_blobs, list_buckets):  # noqa:E125
         execution_time = pd.Timestamp.utcnow()
 
         bucket = MockedObject()
@@ -68,7 +68,7 @@ def test_create_filtered_data_for_multiple_buckets_with_a_matching_bucket_should
     @patch('datacatalog_fileset_enricher.gcs_storage_client_helper.StorageClientHelper.list_blobs')
     @patch('datacatalog_fileset_enricher.gcs_storage_client_helper.StorageClientHelper.get_bucket')
     def test_create_filtered_data_for_single_bucket_with_a_existent_bucket_should_create_filtered_data(  # noqa: E501
-        self, get_bucket, list_blobs, list_buckets):
+        self, get_bucket, list_blobs, list_buckets):  # noqa:E125
 
         execution_time = pd.Timestamp.utcnow()
 
@@ -124,7 +124,7 @@ def test_create_filtered_data_for_single_bucket_with_a_existent_bucket_should_cr
     @patch('datacatalog_fileset_enricher.gcs_storage_client_helper.StorageClientHelper.list_blobs')
     @patch('datacatalog_fileset_enricher.gcs_storage_client_helper.StorageClientHelper.get_bucket')
     def test_create_filtered_data_for_single_bucket_with_nonexistent_bucket_should_create_filtered_data(  # noqa: E501
-        self, get_bucket, list_blobs, list_buckets):
+        self, get_bucket, list_blobs, list_buckets):  # noqa:E125
 
         get_bucket.return_value = None
 

tests/datacatalog_fileset_enricher/gcs_storage_stats_summarizer_test.py

@@ -24,7 +24,8 @@ def test_create_stats_from_dataframe_with_no_dataframe_should_summarize_the_buck
         self.assertEqual(None, stats['bucket_prefix'])
 
     def test_create_stats_from_dataframe_with_no_dataframe_and_no_bucket_stats_should_summarize_the_bucket_stats(  # noqa: E501
-        self):
+        self):  # noqa:E125
+
         dataframe = None
         filtered_buckets_stats = []
         execution_time = pd.Timestamp.utcnow()
@@ -42,7 +43,8 @@ def test_create_stats_from_dataframe_with_no_dataframe_and_no_bucket_stats_shoul
         self.assertEqual(None, stats['bucket_prefix'])
 
     def test_create_stats_from_dataframe_with_no_dataframe_with_bucket_prefix_should_summarize_the_bucket_stats(  # noqa: E501
-        self):
+        self):  # noqa:E125
+
         dataframe = None
         filtered_buckets_stats = [{'bucket_name': 'my_bucket', 'files': 100}]
         execution_time = pd.Timestamp.utcnow()