upgrade from data-integration 2.7 to mara pipelines 3.0.0 (#8)

CHANGELOG.md

@@ -5,6 +5,10 @@
 - Catch more (connection, authentication,...) problems during loading of data and
   retry in that case (closes: #2, #4)
 
+## 0.2.0 (2020-06-17)
+
+- replace data_integration with mara_pipelines 3.0.0
+
 ## 0.1.0 (2020-03-25)
 
 - Move to Github

README.md

@@ -44,8 +44,8 @@ The spreadsheet contains a worksheet 'test' which contains the following colums:
 Note that it doesn't contain an id column at the start!
 
 ```python
-from data_integration.pipelines import Pipeline, Task
-from data_integration.commands.sql import ExecuteSQL
+from mara_pipelines.pipelines import Pipeline, Task
+from mara_pipelines.commands.sql import ExecuteSQL
 from mara_google_sheet_downloader.mara_integration import DownloadGoogleSpreadsheet
 
 pipeline = Pipeline(

mara_google_sheet_downloader/mara_integration.py

@@ -1,7 +1,7 @@
 import shlex
 import sys
-from data_integration import pipelines
-from data_integration.logging import logger
+from mara_pipelines import pipelines
+from mara_pipelines.logging import logger
 import mara_db.shell
 
 from .columns_definition import COLUMN_DEFINITION_TYPE

setup.py

@@ -10,7 +10,7 @@ def get_long_description():
 
 setup(
     name='mara-google-sheet-downloader',
-    version='0.1.0',
+    version='0.2.0',
 
     description='Opinionated lightweight ETL pipeline framework',
 
@@ -21,7 +21,7 @@ def get_long_description():
 
     install_requires=[
         'mara-db>=4.2.0',
-        'data-integration>=2.7.0',
+        'mara-pipelines>=3.0.0',
         'gspread>=3.1.0',
         'oauth2client>=1.5.0', # old, will be replaced soon
         'google_auth_oauthlib' # new, already used in the user credential helper

tests/test_mara_integration.py

@@ -3,8 +3,8 @@
 
 
 @pytest.fixture
-def mock_data_integration_config(monkeypatch):
-    """Setups the data_integration.config so that it's usable for us"""
+def mock_mara_pipelines_config(monkeypatch):
+    """Setups the mara_pipelines.config so that it's usable for us"""
     import mara_db.dbs
     import mara_db.config
 
@@ -30,7 +30,7 @@ def mock_mara_google_sheet_downloader_config(monkeypatch):
 skip_rows = 3
 
 
-def test_DownloadGoogleSpreadsheet_command_with_flask(mock_data_integration_config):
+def test_DownloadGoogleSpreadsheet_command_with_flask(mock_mara_pipelines_config):
     command = mi.DownloadGoogleSpreadsheet(spreadsheet_key=spreadsheet_key,
                                            worksheet_name=worksheet_name,
                                            columns_definition=columns_definition,
@@ -46,7 +46,7 @@ def test_DownloadGoogleSpreadsheet_command_with_flask(mock_data_integration_conf
     assert shell_command.startswith('flask')
 
 
-def test_DownloadGoogleSpreadsheet_command_with_python(mock_data_integration_config,
+def test_DownloadGoogleSpreadsheet_command_with_python(mock_mara_pipelines_config,
                                                        mock_mara_google_sheet_downloader_config):
     command = mi.DownloadGoogleSpreadsheet(spreadsheet_key=spreadsheet_key,
                                            worksheet_name=worksheet_name,
@@ -67,7 +67,7 @@ def test_DownloadGoogleSpreadsheet_command_with_python(mock_data_integration_con
     assert 'python' in shell_command
 
 
-def test_DownloadGoogleSpreadsheet_command_with_python_without_credentials(mock_data_integration_config):
+def test_DownloadGoogleSpreadsheet_command_with_python_without_credentials(mock_mara_pipelines_config):
     command = mi.DownloadGoogleSpreadsheet(spreadsheet_key=spreadsheet_key,
                                            worksheet_name=worksheet_name,
                                            columns_definition=columns_definition,