Merge pull request #10 from mitodl/py3_update

Upgraded dependencies and added support for Python 3

.travis.yml

@@ -1,8 +1,9 @@
 language: python
 python:
-  - "2.7"
+  - "3.6"
 install:
-  - pip install -r requirements.txt -r test_requirements.txt
+  - pip install poetry
+  - poetry install
 script: py.test
 after_success:
   coveralls

gitreload/__init__.py

@@ -3,4 +3,4 @@
 installation.
 """
 
-VERSION = '0.1.0'
+VERSION = '0.2.0'

gitreload/config.py

@@ -2,16 +2,17 @@
 Setup configuration from a json file with defaults
 """
 import logging
-from logging.handlers import RotatingFileHandler
 import os
 import platform
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
 
 log = logging.getLogger('gitreload')  # pylint: disable=C0103
 
 MINUTE = 60  # seconds
 
 
-class Config(object):
+class Config:
     """
     Configuration for the app
     """
@@ -61,7 +62,7 @@ def configure_logging(level_override=None, config=Config):
     address = None
     if config.LOG_FILE_PATH:
         address = config.LOG_FILE_PATH
-    elif os.path.exists('/dev/log'):
+    elif Path('/dev/log').is_file():
         address = '/dev/log'
 
     if address:

gitreload/processing.py

@@ -6,7 +6,7 @@
 import logging
 import multiprocessing
 import os
-import subprocess32
+import subprocess
 
 from git import Repo
 
@@ -35,15 +35,15 @@ def import_repo(action_call):
     log.info('Beginning import of course repo %s with command %s',
              action_call.repo_name, ' '.join(cmd))
     try:
-        import_process = subprocess32.check_output(
+        import_process = subprocess.check_output(
             cmd,
             cwd=config.Config.EDX_PLATFORM,
-            stderr=subprocess32.STDOUT,
+            stderr=subprocess.STDOUT,
             timeout=config.Config.SUBPROCESS_TIMEOUT,
         )
-    except subprocess32.CalledProcessError as exc:
+    except subprocess.CalledProcessError as exc:
         log.exception('Import command failed with: %s', exc.output)
-    except subprocess32.TimeoutExpired as exc:
+    except subprocess.TimeoutExpired as exc:
         log.exception('Import command timed out after %s seconds with: %s', exc.timeout, exc.output)
     except OSError as ex:
         log.exception('System or configuration error occurred: %s', str(ex))
@@ -68,8 +68,8 @@ def git_get_latest(action_call):
     repo.git.clean('-xdf')
     new_head = repo.head.commit.tree.hexsha
     if new_head == orig_head:
-        log.warn('Attempted update of %s at HEAD %s, but no updates',
-                 action_call.repo_name, orig_head)
+        log.warning('Attempted update of %s at HEAD %s, but no updates',
+                    action_call.repo_name, orig_head)
     else:
         log.info('Updated to latest revision of repo %s. '
                  'Original SHA: %s. Head SHA: %s',
@@ -81,10 +81,9 @@ class InvalidGitActionException(Exception):
     Catachable exception for when an invalid
     action is requested in init.
     """
-    pass
 
 
-class ActionCall(object):
+class ActionCall:
     """
     Class structure for passing to processing queue
     """
@@ -104,7 +103,7 @@ def __init__(
         self.repo_name = repo_name
         self.repo_url = repo_url
 
-        if action_type not in self.ACTION_TYPES.values():
+        if action_type not in list(self.ACTION_TYPES.values()):
             raise InvalidGitActionException(
                 'Action must be in ActionCall.ACTION_TYPES'
             )
@@ -116,7 +115,7 @@ def action_text(self):
         """
         Get the text representation of the action
         """
-        action_type = [key for key, value in self.ACTION_TYPES.items()
+        action_type = [key for key, value in list(self.ACTION_TYPES.items())
                        if value == self.action_type]
         return action_type[0]
 

gitreload/tests/base.py

@@ -1,6 +1,7 @@
 """
 Define a base classes for testing
 """
+# pylint: disable=import-outside-toplevel
 import os
 from multiprocessing import JoinableQueue
 import unittest

gitreload/tests/test_config.py

@@ -1,8 +1,9 @@
 """
 Unit tests to validate configuration loading
 """
+# pylint: disable=import-outside-toplevel
+
 import logging
-import os
 import unittest
 import mock
 
@@ -61,7 +62,7 @@ def test_bad_log_level(self,):
         self.assertEqual(logging.WARNING, log_level)
 
         from gitreload.config import configure_logging
-        with self.assertRaisesRegexp(ValueError, 'Invalid log level.+'):
+        with self.assertRaisesRegex(ValueError, 'Invalid log level.+'):
             log_level = configure_logging()
 
     @mock.patch('gitreload.config.Config.LOG_LEVEL', None)
@@ -76,47 +77,3 @@ def test_no_log_level(self):
         from gitreload.config import configure_logging
         log_level = configure_logging()
         self.assertEqual(logging.NOTSET, log_level)
-
-    def test_syslog_devices(self):
-        """
-        Test syslog address handling and handler
-        """
-
-        for log_device in ['/dev/log', '/var/run/syslog', '']:
-            root_logger = logging.getLogger()
-            # Nuke syslog handlers from init
-            syslog_handlers = []
-            for handler in root_logger.handlers:
-                if isinstance(handler, logging.handlers.SysLogHandler):
-                    syslog_handlers.append(handler)
-            for handler in syslog_handlers:
-                root_logger.removeHandler(handler)
-
-            real_exists = os.path.exists(log_device)
-
-            def mock_effect(*args):
-                """Contextual choice of log device."""
-                if args[0] == log_device:  # pylint: disable=cell-var-from-loop
-                    return True
-                return False
-
-            # Call so that it will think /dev/log exists
-            with mock.patch('os.path') as os_exists:
-                os_exists.exists.side_effect = mock_effect
-                from gitreload.config import configure_logging
-                if not real_exists and log_device != '':
-                    with self.assertRaises(Exception):
-                        configure_logging()
-                else:
-                    configure_logging()
-                    syslog_handler = None
-                    for handler in root_logger.handlers:
-                        if isinstance(handler, logging.handlers.SysLogHandler):
-                            syslog_handler = handler
-                    self.assertIsNotNone(syslog_handler)
-                    if log_device == '':
-                        self.assertEqual(
-                            syslog_handler.address, ('127.0.0.1', 514)
-                        )
-                    else:
-                        self.assertEqual(syslog_handler.address, log_device)

gitreload/tests/test_processing.py

@@ -1,9 +1,10 @@
 """
 Tests related to the processing module
 """
+# pylint: disable=import-outside-toplevel
 import os
 import shutil
-import subprocess32
+import subprocess
 import mock
 
 from git import Repo
@@ -56,7 +57,7 @@ def test_import_failure(self, mocked_logging):
             ))
         mocked_logging.exception.assert_called_with(
             'System or configuration error occurred: %s',
-            "[Errno 20] Not a directory"
+            "[Errno 20] Not a directory: '/dev/null'"
         )
 
     @mock.patch('gitreload.processing.log')
@@ -85,8 +86,8 @@ def test_command_error_and_input(self, mocked_logging):
                     'SUBPROCESS_TIMEOUT': 59,
                 }
             )
-            with mock.patch('subprocess32.check_output') as check_output:
-                check_output.side_effect = subprocess32.CalledProcessError(
+            with mock.patch('subprocess.check_output') as check_output:
+                check_output.side_effect = subprocess.CalledProcessError(
                     10, 'test_command', output='Test output'
                 )
                 import_repo(ActionCall(
@@ -124,7 +125,7 @@ def test_command_success(self, mocked_logging):
 
         # Have mock get called on import and check parameters and have
         # return raise the right Exception
-        with mock.patch('subprocess32.check_output') as check_output:
+        with mock.patch('subprocess.check_output') as check_output:
             check_output.return_value = "Test Success"
             import_repo(ActionCall(
                 'NOTREAL', 'NOTREAL',
@@ -146,8 +147,8 @@ def test_import_timeout(self, mocked_logging):
         from gitreload.processing import import_repo, ActionCall
 
         # Call with bad edx-platform path to prevent actual execution
-        with mock.patch('subprocess32.check_output') as check_output:
-            check_output.side_effect = subprocess32.TimeoutExpired(cmd='ls', output='foooo', timeout=39)
+        with mock.patch('subprocess.check_output') as check_output:
+            check_output.side_effect = subprocess.TimeoutExpired(cmd='ls', output='foooo', timeout=39)
             import_repo(ActionCall(
                 'NOTREAL', 'NOTREAL',
                 ActionCall.ACTION_TYPES['COURSE_IMPORT']
@@ -276,7 +277,7 @@ def test_git_get_latest(self, mocked_log):
         with mock.patch('gitreload.config.Config.REPODIR', TEST_ROOT):
             git_get_latest(action_call)
 
-        mocked_log.warn.assert_called_with(
+        mocked_log.warning.assert_called_with(
             'Attempted update of %s at HEAD %s, but no updates',
             repo_name, repo.head.commit.tree.hexsha
         )

gitreload/tests/test_web.py

@@ -1,6 +1,7 @@
 """
 Test out the flask Web application
 """
+# pylint: disable=import-outside-toplevel
 import json
 import os
 import shutil
@@ -10,7 +11,7 @@
 from git import Repo
 
 import gitreload.web
-from .base import GitreloadTestBase
+from gitreload.tests.base import GitreloadTestBase
 
 
 class TestWebApplication(GitreloadTestBase):
@@ -71,7 +72,7 @@ def test_json_response(self):
         """
         from gitreload.web import json_dump_msg
         json_dump = json_dump_msg('Test')
-        self.assertEquals(json.loads(json_dump), {'msg':  'Test'})
+        self.assertEqual(json.loads(json_dump), {'msg':  'Test'})
 
     def test_queue_status_page(self):
         """
@@ -97,9 +98,9 @@ def test_queue_status_page(self):
         self.assertEqual(
             json_data['queue'],
             [{
-                'repo_name': u'testing',
-                'repo_url': u'http://example.com/testing.git',
-                'action': u'COURSE_IMPORT'
+                'repo_name': 'testing',
+                'repo_url': 'http://example.com/testing.git',
+                'action': 'COURSE_IMPORT'
             }])
         # Clean up queue
         queued_jobs.pop()

gitreload/web.py

@@ -123,16 +123,14 @@ def hook_receive():
     return_value, repo_name = verify_hook()
     if not repo_name:
         return return_value
-    else:
-        repo = return_value
 
     log.debug('Local and remote branch match, scheduling action')
 
     # Go ahead and run the git import script. Use simple thread for now
     # to prevent timeouts.
     action = ActionCall(
         repo_name,
-        repo.remotes.origin.url,
+        return_value.remotes.origin.url,
         ActionCall.ACTION_TYPES['COURSE_IMPORT']
     )
     queued_jobs.append(action)
@@ -152,16 +150,14 @@ def update_repo():
     return_value, repo_name = verify_hook()
     if not repo_name:
         return return_value
-    else:
-        repo = return_value
 
     log.debug('Local and remote branch match, doing pull')
 
     # Go ahead and run the git import script. Use simple thread for now
     # to prevent timeouts.
     action = ActionCall(
         repo_name,
-        repo.remotes.origin.url,
+        return_value.remotes.origin.url,
         ActionCall.ACTION_TYPES['GET_LATEST']
     )
     queued_jobs.append(action)