Big refactor for better file structure

.envs/.local/.flask

@@ -0,0 +1,2 @@
+GITHUB_USER=supmanbot
+GITHUB_PSX="ha! wouldn't you like to know"

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "~/.virtualenvs/autognome/bin/python"
+}

compose/local/flask/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.6-alpine
+
+ENV PYTHONBUFFERED 1
+
+# Requirements are installed here to ensure they will be cached.
+COPY ./requirements /requirements
+RUN pip install -r /requirements/local.txt
+
+COPY ./compose/remote/flask/entrypoint /entrypoint
+RUN sed -i 's/\r//' /entrypoint
+RUN chmod +x /entrypoint
+
+COPY ./compose/local/flask/start /start
+RUN sed -i 's/\r//' /start
+RUN chmod +x /start
+
+WORKDIR /app
+
+ENTRYPOINT ["/entrypoint"]

compose/local/flask/start

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+/usr/local/bin/gunicorn --reload --bind 0.0.0.0:8000 --chdir=/app/gnome app:app --log-level debug

compose/remote/flask/entrypoint

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+exec "$@"

config/__init__.py

@@ -0,0 +1,16 @@
+# from os import environ
+from envparse import env
+from .kms import string_or_b64kms
+
+
+class Env(object):
+    def __call__(self, var, default=None, cast=None):
+        value = env(var, default=default, cast=cast)
+        return string_or_b64kms(value)
+
+# class Env(object):
+#     """Wrapper around os.getenv with added AWS KMS encryption support."""
+
+#     def __call__(self, var, default=None):
+#         value = environ.get(var, default)
+#         return string_or_b64kms(value)

config/kms.py

@@ -0,0 +1,46 @@
+import os
+import base64
+import logging
+
+import boto3
+
+BASE64_PREFIX = 'base64:'
+AWS_REGION = os.environ.get('AWS_REGION', None)
+
+
+def decrypt_kms_data(encrypted_data):
+    """Decrypt KMS encoded data."""
+    if not AWS_REGION:
+        return
+
+    kms = boto3.client('kms', region_name=AWS_REGION)
+
+    decrypted = kms.decrypt(CiphertextBlob=encrypted_data)
+
+    if decrypted.get('KeyId'):
+        # Decryption succeed
+        decrypted_value = decrypted.get('Plaintext', '')
+        if isinstance(decrypted_value, bytes):
+            decrypted_value = decrypted_value.decode('utf-8')
+        return decrypted_value
+
+
+def string_or_b64kms(value):
+    """Check if value is base64 encoded - if yes, decode it using KMS."""
+    if not value:
+        return value
+
+    try:
+        # Check if environment value base64 encoded
+        if isinstance(value, (str, bytes)) and value.startswith(BASE64_PREFIX):
+            value = value[len(BASE64_PREFIX):]
+            # If yes, decode it using AWS KMS
+            data = base64.b64decode(value)
+            decrypted_value = decrypt_kms_data(data)
+
+            # If decryption succeed, use it
+            if decrypted_value:
+                value = decrypted_value
+    except Exception as e:
+        logging.exception(e)
+    return value

config/settings.py

@@ -0,0 +1,8 @@
+# Load our conf
+from . import Env
+
+env = Env()
+
+DEBUG = env('DEBUG', default=False, cast=bool)
+GITHUB_USER = env('GITHUB_USER')
+GITHUB_PSX = env('GITHUB_PSX')

docs/conf.py

@@ -1,7 +1,7 @@
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../src'))
-sys.path.insert(0, os.path.abspath('../src/tests'))
+sys.path.insert(0, os.path.abspath('../gnome'))
+sys.path.insert(0, os.path.abspath('../gnome/tests'))
 
 extensions = ['sphinx.ext.autodoc',
               'sphinx.ext.viewcode',

docs/gnome_development.rst

@@ -33,9 +33,9 @@ Hacking the gnome itself
 Codebase
 --------
 
-The callback service is provided by a Flask app, and the code for this is in `src/app.py`. It's only job is to receive callbacks from GitHub and process them.
+The callback service is provided by a Flask app, and the code for this is in `gnome/app.py`. It's only job is to receive callbacks from GitHub and process them.
 
-app.py delegates the interesting stuff to code in `src/utils.py`. This does two things, interacts with GitHub to obtain configuration, then delegates configured tasks to the plugins (via the plugin register, `src/policies/__init__.py`)
+app.py delegates the interesting stuff to code in `gnome/utils.py`. This does two things, interacts with GitHub to obtain configuration, then delegates configured tasks to the plugins (via the plugin register, `gnome/policies/__init__.py`)
 
 Ultimately, interesting stuff is delegated to plugins. All plugins must provide a `dispatch_gnome()` method. If configured, this is called with data from the originating callback event (and config).
 
@@ -49,23 +49,23 @@ app.py
    :undoc-members:
 
 
-src/utils.py
+gnome/utils.py
 ^^^^^^^^^^^^
 
 .. automodule:: util
    :members:
    :undoc-members:
 
 
-src/gh.py
+gnome/gh.py
 ^^^^^^^^^
 
 .. automodule:: gh
    :members:
    :undoc-members:
 
 
-src/policies/__init__.py
+gnome/policies/__init__.py
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 The util module instantiates the `policy` module. This is a very simple thing, all it does is import the relevant classes (from modules in the plugins directory).
@@ -89,4 +89,3 @@ Tests
 .. automodule:: tests.test_callback
    :members:
    :undoc-members:
-

docs/pavement.py

@@ -1,6 +1,6 @@
 import os
 import sys
-sys.path.insert(0, os.path.abspath('../src'))
+sys.path.insert(0, os.path.abspath('../gnome'))
 
 from paver.easy import *
 import paver.doctools

docs/requirements.txt

@@ -1,4 +1,4 @@
 sphinx
 paver
 cogapp
--r ../src/requirements.txt
+-r ../requirements/base.txt

src/app.py → gnome/app.py

@@ -1,18 +1,16 @@
 from flask import Flask, request, abort
-from github import Github
 import json
-import requests
 import sys
-import yaml
-import config
-from util import (Config, CallbackEvent)
-from gh import EventSourceValidator
+from config import settings
+from gnome.util import (Config, CallbackEvent)
+from gnome.gh import EventSourceValidator
 
 # verbose logging
-DEBUG = config.DEBUG
+DEBUG = settings.DEBUG
 
 app = Flask(__name__)
 
+
 @app.route('/', methods=['GET', 'POST'])
 def index():
     if request.method == 'GET':
@@ -47,4 +45,3 @@ def index():
 
 if __name__ == '__main__':
     app.run()
-

src/gh.py → gnome/gh.py

@@ -4,10 +4,10 @@
 import base64
 from github import Github
 
-import config
+from config import settings
 
-GITHUB_USER = config.GITHUB_USER
-GITHUB_PSX = config.GITHUB_PSX
+GITHUB_USER = settings.GITHUB_USER
+GITHUB_PSX = settings.GITHUB_PSX
 
 gh = Github(GITHUB_USER, GITHUB_PSX)
 
@@ -245,4 +245,3 @@ def ip_str_is_valid(self, ip_str):
             if ipaddress.ip_address(request_ip) in ipaddress.ip_network(block):
                 return True
         return False
-

src/plugins/propagate_milestones.py → gnome/plugins/propagate_milestones.py

@@ -1,8 +1,8 @@
 import yaml
 import logging
 import dateutil.parser
-from policies import Policy
-from gh import Repo
+from gnome.policies import Policy
+from gnome.gh import Repo
 
 
 logger = logging.getLogger(__file__)
@@ -58,6 +58,3 @@ def dispatch_gnome(self):
                 milestone =  repo.milestones.get(title)
                 if milestone:
                     milestone._milestone.edit(title, state='closed')
-
-
-

src/plugins/sorting_hat.py → gnome/plugins/sorting_hat.py

@@ -2,13 +2,13 @@
 import os.path
 import json
 # FIXME: shouldn't have to mess with sys.path to import my stuff
-sys.path.append(
-    os.path.abspath(
-        os.path.join(
-            os.path.dirname(__file__),
-            os.path.pardir)))
-from gh import (Milestone, Issue, Repo)
-from policies import Policy
+# sys.path.append(
+#     os.path.abspath(
+#         os.path.join(
+#             os.path.dirname(__file__),
+#             os.path.pardir)))
+from gnome.gh import (Milestone, Issue, Repo)
+from gnome.policies import Policy
 
 # TODO: this should optionally come from config in .gnome.yml
 SORTING_HAT_MILESTONE = "The Sorting Hat"

gnome/plugins/sync_project_milestones.py

@@ -0,0 +1,6 @@
+from gnome.policies import Policy
+
+
+class SyncProjectMilestones(Policy):
+    def dispatch_gnome(self):
+        pass

src/plugins/verbose_callback_logging.py → gnome/plugins/verbose_callback_logging.py

@@ -1,6 +1,6 @@
 import sys
 import json
-from policies import Policy
+from gnome.policies import Policy
 
 class VerboseCallbackLogging(Policy):
     """
@@ -29,6 +29,5 @@ def dispatch_gnome(self):
         # the inherited constructor creates self.callback
         callback_payload = self.callback.payload()
         # all this policy does is log the callback payload to stdout
-        print(json.dumps(callback_payload, sort_keys=True, indent=4), 
+        print(json.dumps(callback_payload, sort_keys=True, indent=4),
               file=sys.stdout)
-

src/policies/__init__.py → gnome/policies/__init__.py

@@ -25,13 +25,13 @@ class AbstractBaseGnomePolicyCanNotBeDispatchedError(Exception): pass
 
 
 # this is where policies are registered
-from plugins.verbose_callback_logging import VerboseCallbackLogging
-from plugins.sorting_hat import SortingHat
-from plugins.propagate_milestones import PropagateMilestones
+from gnome.plugins.verbose_callback_logging import VerboseCallbackLogging
+from gnome.plugins.sorting_hat import SortingHat
+from gnome.plugins.propagate_milestones import PropagateMilestones
 
 # TODO: make a decision about case for config
 MANIFEST = {
     'SortingHat': SortingHat,
     'VerboseCallbackLogging': VerboseCallbackLogging,
     'propagate_milestones': PropagateMilestones
-}
+}

src/requirements.txt → gnome/requirements.txt

@@ -3,5 +3,8 @@ zappa
 PyGithub
 requests
 ipaddress
-PyYAML
+PyYAML==3.12
 pytest
+pylint
+python-dotenv
+envparse