From 3efaead9ca87635c3c72edd124cc7813731bb559 Mon Sep 17 00:00:00 2001 From: Martin Morgenstern Date: Sun, 4 Mar 2018 20:06:00 +0100 Subject: [PATCH] Remove the inloop.gh_import module --- .env.develop | 1 - docs/INSTALL.md | 4 - inloop/gh_import/__init__.py | 0 inloop/gh_import/git.py | 49 ----------- inloop/gh_import/github_known_hosts.txt | 1 - inloop/gh_import/tasks.py | 63 -------------- inloop/gh_import/urls.py | 8 -- inloop/gh_import/utils.py | 30 ------- inloop/gh_import/views.py | 30 ------- inloop/settings.py | 7 -- inloop/urls.py | 4 +- tests/.env | 1 - tests/unit/test_gitimport.py | 110 ------------------------ 13 files changed, 2 insertions(+), 306 deletions(-) delete mode 100644 inloop/gh_import/__init__.py delete mode 100644 inloop/gh_import/git.py delete mode 100644 inloop/gh_import/github_known_hosts.txt delete mode 100644 inloop/gh_import/tasks.py delete mode 100644 inloop/gh_import/urls.py delete mode 100644 inloop/gh_import/utils.py delete mode 100644 inloop/gh_import/views.py delete mode 100644 tests/unit/test_gitimport.py diff --git a/.env.develop b/.env.develop index 7e470a47..e4c1b951 100644 --- a/.env.develop +++ b/.env.develop @@ -6,7 +6,6 @@ DEBUG=True DJANGO_SETTINGS_MODULE=inloop.settings EMAIL_URL=consolemail:// FROM_EMAIL=inloop@example.com -GITHUB_SECRET=secret REDIS_URL=redis://127.0.0.1:6379/1 SECRET_KEY=secret SECURE_COOKIES=False diff --git a/docs/INSTALL.md b/docs/INSTALL.md index ea64b1d0..09caf08b 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -192,8 +192,6 @@ Installation ├── DATABASE_URL ├── DJANGO_SETTINGS_MODULE ├── FROM_EMAIL - ├── GITHUB_SECRET - ├── GIT_ROOT ├── LANG ├── MEDIA_ROOT ├── PATH @@ -309,8 +307,6 @@ Name | Description `DATABASE_URL` | 12factor style database URL, e.g. `postgres://user:pass@host:port/db` `DJANGO_SETTINGS_MODULE` | Set to `inloop.settings` unless you know what you are doing `FROM_EMAIL` | Address used for outgoing mail, e.g. `inloop@example.com` -`GITHUB_SECRET` | Github webhook endpoint secret -`GIT_ROOT` | Must be a subdirectory of `MEDIA_ROOT` `LANG` | Set to `en_US.UTF-8` unless you know what you are doing `MEDIA_ROOT` | Path to the directory for user uploads `PYTHONPATH` | Set it to the path of the INLOOP git clone diff --git a/inloop/gh_import/__init__.py b/inloop/gh_import/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/inloop/gh_import/git.py b/inloop/gh_import/git.py deleted file mode 100644 index 2c46f1c7..00000000 --- a/inloop/gh_import/git.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from subprocess import check_call - -from inloop.gh_import.utils import ssh_options - -GIT_COMMAND = 'git' -DEFAULT_BRANCH = 'master' - - -class GitRepository: - silent_subcmds = ['clone', 'checkout', 'pull'] - keep_envvars = ['PATH', 'LANG', 'LC_ALL', 'USER'] - - def __init__(self, git_dir, id_rsa=None, verbose=False): - self.git_dir = git_dir - self.id_rsa = id_rsa - self.verbose = verbose - - def pull(self, git_branch=DEFAULT_BRANCH): - self._run('checkout', git_branch) - self._run_with_ssh('pull') - - def clone(self, git_url, git_branch=DEFAULT_BRANCH): - self._run_with_ssh('clone', '--branch', git_branch, git_url, '.') - - def _clean_env(self): - env = {} - for key, val in os.environ.items(): - if key in self.keep_envvars: - env[key] = val - return env - - def _run_with_ssh(self, subcmd, *args): - env = self._clean_env() - env['GIT_SSH_COMMAND'] = 'ssh {}'.format( - ssh_options(self.id_rsa, shell=True, verbose=self.verbose) - ) - self._run(subcmd, *args, env=env) - - def _run(self, subcmd, *args, **kwargs): - cmd = [GIT_COMMAND, subcmd] - env = kwargs.get('env', self._clean_env()) - - if subcmd in self.silent_subcmds: - cmd.append('--quiet') - if args: - cmd.extend(args) - - return check_call(cmd, env=env, cwd=self.git_dir) diff --git a/inloop/gh_import/github_known_hosts.txt b/inloop/gh_import/github_known_hosts.txt deleted file mode 100644 index 1bae52b8..00000000 --- a/inloop/gh_import/github_known_hosts.txt +++ /dev/null @@ -1 +0,0 @@ -github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== diff --git a/inloop/gh_import/tasks.py b/inloop/gh_import/tasks.py deleted file mode 100644 index da0e6fce..00000000 --- a/inloop/gh_import/tasks.py +++ /dev/null @@ -1,63 +0,0 @@ -import json -import logging -from glob import glob -from os.path import basename, dirname, join -from subprocess import CalledProcessError, check_call - -from django.conf import settings -from django.db.transaction import atomic - -from huey.contrib.djhuey import db_task - -from inloop.gh_import.git import GitRepository -from inloop.tasks.models import Task - -logger = logging.getLogger(__name__) - -NEWS_FILE = "news.md" -META_FILE = "meta.json" -TASK_FILE = "task.md" - - -@db_task() -def update_tasks(): - # git refresh - git_root = settings.GIT_ROOT - if not settings.DEBUG: - logger.info("Pulling changes from git") - repo = GitRepository(git_root) - repo.pull() - - # find task directories - meta_files = glob(join(git_root, '*', META_FILE)) - task_dirs = [dirname(f) for f in meta_files] - logger.info("found %d task directories", len(task_dirs)) - for task_dir in task_dirs: - process_dir(task_dir) - - # ZIP the tests - logger.info("zipping the unit tests w/ gradle") - gradlew_exe = join(git_root, 'gradlew') - try: - check_call([gradlew_exe, '-q', 'zipTests'], cwd=git_root) - except CalledProcessError as e: - logger.error("gradlew FAILED, exception follows.") - logger.exception(e) - - -def process_dir(task_dir): - """Processes a single task directory.""" - task_name = basename(task_dir) - logger.info("Processing task %s", task_name) - try: - with atomic(): - data = {} - with open(join(task_dir, META_FILE), encoding="utf-8") as json_file: - data.update(json.load(json_file)) - with open(join(task_dir, TASK_FILE), encoding="utf-8") as markdown_file: - data["description"] = markdown_file.read() - Task.objects.update_or_create_related(system_name=task_name, data=data) - logger.info("Successfully imported %s", task_name) - except Exception as e: - logger.error("Importing task %s FAILED, exception follows.", task_name) - logger.exception(e) diff --git a/inloop/gh_import/urls.py b/inloop/gh_import/urls.py deleted file mode 100644 index e999971d..00000000 --- a/inloop/gh_import/urls.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.conf.urls import url - -from inloop.gh_import.views import PayloadView - -app_name = "github" -urlpatterns = [ - url(r'^payload$', PayloadView.as_view(), name='payload') -] diff --git a/inloop/gh_import/utils.py b/inloop/gh_import/utils.py deleted file mode 100644 index 3a068492..00000000 --- a/inloop/gh_import/utils.py +++ /dev/null @@ -1,30 +0,0 @@ -import hashlib -import hmac -from os.path import dirname, join -from shlex import quote - -# known_hosts file prepopulated with GitHub's pubkeys -HOSTS_FILE = join(dirname(__file__), 'github_known_hosts.txt') - - -def ssh_options(id_rsa, shell=False, verbose=False): - opts = [ - '-F/dev/null', - '-oUserKnownHostsFile=%s' % HOSTS_FILE, - '-oCheckHostIP=no', - '-oBatchMode=yes' - ] - if id_rsa: - opts.append('-oIdentityFile=%s' % id_rsa) - if verbose: - opts.append('-v') - if shell: - return ' '.join([quote(opt) for opt in opts]) - return opts - - -def compute_signature(secret, data): - mac = hmac.new(secret, digestmod=hashlib.sha1) - for chunk in iter(data): - mac.update(chunk) - return 'sha1=%s' % mac.hexdigest() diff --git a/inloop/gh_import/views.py b/inloop/gh_import/views.py deleted file mode 100644 index 09597f8d..00000000 --- a/inloop/gh_import/views.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.conf import settings -from django.http import HttpResponse, HttpResponseBadRequest -from django.utils.crypto import constant_time_compare -from django.utils.decorators import method_decorator -from django.utils.encoding import smart_bytes -from django.views.decorators.csrf import csrf_exempt -from django.views.generic import View - -from inloop.gh_import.tasks import update_tasks -from inloop.gh_import.utils import compute_signature - -SIGNATURE_HEADER = 'HTTP_X_HUB_SIGNATURE' -GITHUB_SECRET = smart_bytes(settings.GITHUB_SECRET) - - -class PayloadView(View): - @method_decorator(csrf_exempt) - def dispatch(self, request, *args, **kwargs): - return super().dispatch(request, *args, **kwargs) - - def post(self, request): - try: - signature = request.META[SIGNATURE_HEADER] - except KeyError: - return HttpResponseBadRequest('Missing signature') - my_signature = compute_signature(GITHUB_SECRET, request) - if not constant_time_compare(signature, my_signature): - return HttpResponseBadRequest('Invalid signature') - update_tasks() - return HttpResponse('Success') diff --git a/inloop/settings.py b/inloop/settings.py index d27a9a6e..8bf7291a 100644 --- a/inloop/settings.py +++ b/inloop/settings.py @@ -55,7 +55,6 @@ "inloop.solutions", "inloop.tasks", "inloop.testrunner", - "inloop.gh_import", "inloop.gitload", ] @@ -183,12 +182,6 @@ } DOCKER_IMAGE = "inloop-java-checker" -GITHUB_SECRET = env("GITHUB_SECRET") -GIT_ROOT = str(env("GIT_ROOT", cast=Path, default=BASE_DIR.parent / "inloop-tasks")) -GIT_SSH_URL = env("GIT_URL", default="") -GIT_SSH_KEY = None -GIT_BRANCH = env("GIT_BRANCH", default="master") - REPOSITORY_ROOT = str(Path(MEDIA_ROOT).joinpath("repository")) ACCOUNT_ACTIVATION_DAYS = 7 diff --git a/inloop/urls.py b/inloop/urls.py index d64886ac..6af09b3f 100644 --- a/inloop/urls.py +++ b/inloop/urls.py @@ -4,7 +4,7 @@ from django.contrib.flatpages.views import flatpage from inloop.accounts import urls as account_urls -from inloop.gh_import import urls as github_urls +from inloop.gitload import urls as gitload_urls from inloop.solutions import urls as solution_urls from inloop.tasks import urls as task_urls from inloop.views import home, logout @@ -17,7 +17,7 @@ url(r'^logout/$', logout, name="logout"), url(r'^account/', include(account_urls)), - url(r'^github/', include(github_urls)), + url(r'^gitload/', include(gitload_urls)), url(r'^solutions/', include(solution_urls)), url(r'^tasks/', include(task_urls)), diff --git a/tests/.env b/tests/.env index 8638602a..750fab64 100644 --- a/tests/.env +++ b/tests/.env @@ -6,7 +6,6 @@ DEBUG=True DJANGO_SETTINGS_MODULE=tests.settings EMAIL_URL=consolemail:// FROM_EMAIL=inloop@example.com -GITHUB_SECRET=secret INTERNAL_IPS=127.0.0.1 REDIS_URL=redis://127.0.0.1:9/ SECRET_KEY=testsecret diff --git a/tests/unit/test_gitimport.py b/tests/unit/test_gitimport.py deleted file mode 100644 index 13bfd397..00000000 --- a/tests/unit/test_gitimport.py +++ /dev/null @@ -1,110 +0,0 @@ -from io import BytesIO -from os import path -from unittest import TestCase -from unittest.mock import patch - -from django.test.client import RequestFactory -from django.urls import resolve - -from inloop.gh_import import __name__ as PACKAGE, views -from inloop.gh_import.git import GitRepository -from inloop.gh_import.utils import HOSTS_FILE, compute_signature, ssh_options - - -class UtilsTest(TestCase): - def test_signature_computation(self): - self.assertEqual( - compute_signature(b'secret', BytesIO(b'foo')), - 'sha1=9baed91be7f58b57c824b60da7cb262b2ecafbd2' - ) - - def test_hosts_file_exists(self): - self.assertTrue(path.exists(HOSTS_FILE)) - - def test_ssh_options_shell_quoting(self): - opts_space = ssh_options('path with spaces', shell=True) - self.assertTrue(isinstance(opts_space, str)) - self.assertTrue("'-oIdentityFile=path with spaces'" in opts_space) - self.assertFalse('-v' in opts_space) - - def test_ssh_options_args(self): - opts = ssh_options(None, shell=True, verbose=True) - self.assertTrue(isinstance(opts, str)) - self.assertTrue('IdentityFile' not in opts) - self.assertTrue('-v' in opts) - - def test_ssh_options_noshell(self): - opts = ssh_options(None, shell=False) - self.assertTrue(isinstance(opts, list)) - - -@patch('%s.git.check_call' % PACKAGE) -class GitRepositoryTest(TestCase): - def setUp(self): - self.repo = GitRepository('path/to/repo') - - def test_clone(self, mock): - self.repo.clone('user@host:repo.git', git_branch='test_branch') - args, kwargs = mock.call_args - - self.assertEqual(kwargs['cwd'], 'path/to/repo') - self.assertTrue('GIT_SSH_COMMAND' in kwargs['env'].keys()) - - cmd_list = args[0] - self.assertEqual(cmd_list[:2], ['git', 'clone']) - self.assertEqual(cmd_list[-1], '.') - self.assertTrue('user@host:repo.git' in cmd_list) - self.assertEqual(cmd_list.index('test_branch') - cmd_list.index('--branch'), 1) - - def test_pull(self, mock): - self.repo.pull('test_branch') - self.assertEqual(mock.call_count, 2) - - args1, kwargs1 = mock.call_args_list[0] - cmd_list1 = args1[0] - self.assertEqual(cmd_list1[:2], ['git', 'checkout']) - self.assertTrue('test_branch' in cmd_list1) - self.assertFalse('GIT_SSH_COMMAND' in kwargs1['env'].keys()) - - args2, kwargs2 = mock.call_args_list[1] - cmd_list2 = args2[0] - self.assertEqual(cmd_list2[:2], ['git', 'pull']) - self.assertTrue('GIT_SSH_COMMAND' in kwargs2['env'].keys()) - - -@patch('%s.views.update_tasks' % PACKAGE) -class PayloadHandlerTest(TestCase): - def setUp(self): - self.factory = RequestFactory() - self.payload = views.PayloadView.as_view() - - def test_url_resolves_to_view(self, mock): - resolve('/github/payload') - - def test_get_not_allowed(self, mock): - request = self.factory.get('/payload') - response = self.payload(request) - self.assertEqual(response.status_code, 405) - self.assertEqual(mock.call_count, 0) - - def test_post_without_signature(self, mock): - request = self.factory.post('/payload') - response = self.payload(request) - self.assertEqual(response.status_code, 400) - self.assertTrue(b'missing' in response.content.lower()) - self.assertEqual(mock.call_count, 0) - - def test_post_with_invalid_signature(self, mock): - request = self.factory.post('/payload') - request.META['HTTP_X_HUB_SIGNATURE'] = 'invalid' - response = self.payload(request) - self.assertEqual(response.status_code, 400) - self.assertTrue(b'invalid' in response.content.lower()) - self.assertEqual(mock.call_count, 0) - - def test_post_with_valid_signature(self, mock): - request = self.factory.post('/payload', data=b'foo', content_type='text/plain') - request.META['HTTP_X_HUB_SIGNATURE'] = 'sha1=9baed91be7f58b57c824b60da7cb262b2ecafbd2' - response = self.payload(request) - self.assertEqual(response.status_code, 200) - self.assertEqual(mock.call_count, 1)