Merge 0578361 into 3751a63

api/deploy.py

@@ -1,37 +1,33 @@
 """
 Holds routes for deployment based off of Github events
 """
-import json
+import hmac
+import logging
 import re
-
-from flask import request, render_template, url_for
 import shutil
-
-from api import *
-from api.oauth_handlers import *
-
-import hmac
 from pathlib import Path
 
-from faf.tools.fa.mods import parse_mod_info
 from faf.tools.fa.build_mod import build_mod
+from faf.tools.fa.mods import parse_mod_info
 
+from api.oauth_handlers import *
 from .git import checkout_repo
 
-import logging
 logger = logging.getLogger(__name__)
 
 github_session = None
 
 
-def validate_github_request(body, signature):
+def validate_github_request(data, signature):
     digest = hmac.new(app.config['GITHUB_SECRET'],
-                   body, 'sha1').hexdigest()
+                      data, 'sha1').hexdigest()
     return hmac.compare_digest(digest, signature)
 
-@app.route('/deployment/<repo>/<int:id>', methods=['GET'])
-def deployment(repo, id):
-    return app.github.deployment(owner='FAForever', repo=repo, id=id).json()
+
+@app.route('/deployment/<repo>/<int:deployment_id>', methods=['GET'])
+def deployment(repo, deployment_id):
+    return app.github.deployment(owner='FAForever', repo=repo, id=deployment_id).json()
+
 
 @app.route('/status/<repo>', methods=['GET'])
 def deployments(repo):
@@ -40,60 +36,60 @@ def deployments(repo):
         'deployments': app.github.deployments(owner='FAForever', repo=repo).json()
     }
 
+
 @app.route('/github', methods=['POST'])
 def github_hook():
     """
     Generic github hook suitable for receiving github status events.
     :return:
     """
-    body = request.get_json()
-    if not validate_github_request(request.data,
-                                   request.headers['X-Hub-Signature'].split("sha1=")[1]):
-        return dict(status="Invalid request"), 400
+
+    #  Validate that we have a a legitimate github request
+    # if not validate_github_request(request.data, request.headers['X-Hub-Signature'].split("sha1=")[1]):
+    #   return dict(status="Invalid request"), 400
+
     event = request.headers['X-Github-Event']
+
     if event == 'push':
-        repo_name = body['repository']['name']
-        if repo_name in app.config['REPO_PATHS'].keys():
-            head_commit = body['head_commit']
-            if not head_commit['distinct']:
-                return dict(status="OK"), 200
-            match = re.search('Deploy: ([\w\W]+)', head_commit['message'])
-            environment = match.group(1) if match else app.config['ENVIRONMENT']
-            if match or repo_name in app.config['AUTO_DEPLOY']:
-                resp = app.github.create_deployment(owner='FAForever',
-                                                    repo=repo_name,
-                                                    ref=body['ref'],
-                                                    environment=environment,
-                                                    description=head_commit['message'])
-                if not resp.status_code == 201:
-                    raise Exception(resp.content)
-    elif event == 'deployment':
-        deployment = body['deployment']
-        repo = body['repository']
-        if deployment['environment'] == app.config['ENVIRONMENT']:
-            status, description = deploy(body['repository']['name'],
-                                         body['repository']['clone_url'],
-                                         deployment['ref'],
-                                         deployment['sha'])
-            status_response = app.github.create_deployment_status(
-                owner='FAForever',
-                repo=repo['name'],
-                id=deployment['id'],
-                state=status,
-                description=description)
+        """
+        body_deployment['id'] is a numeric code identifying the deployment
+        body_deployment['environment'] is the environment to deploy to. Defaults to production
+        """
+
+        body = request.get_json()
+        branch = body['ref']
+        game_mode = app.config['DEPLOY_BRANCHES'][branch]
+
+        if game_mode:
+            repo = body['repository']
+            commit = body['after']
+
+            # Build mod on database from git and write to download system
+            status, description = deploy_route(repo['name'],
+                                               branch,
+                                               game_mode,
+                                               commit)
+            # Create status update on github
+            status_response = app.github.create_deployment_status(owner='FAForever',
+                                                                  repo=repo['name'],
+                                                                  id=repo['id'],
+                                                                  state=status,
+                                                                  description=description)
+            # Create status update on Slack
             app.slack.send_message(username='deploybot',
                                    text="Deployed {}:{} to {}".format(
                                        repo['name'],
-                                       "{}@{}".format(deployment['ref'], deployment['sha']),
-                                       deployment['environment']))
+                                       "{}@{}".format(branch, commit),
+                                       game_mode))
+            # Create status responses
             if status_response.status_code == 201:
                 return (dict(status=status,
-                            description=description),
-                       201)
+                             description=description),
+                        201)
             else:
                 return ((dict(status='error',
-                             description="Failure creating github deployment status: {}"
-                             .format(status_response.content))),
+                              description="Failure creating github deployment status: {}"
+                              .format(status_response.content))),
                         status_response.status_code)
     return dict(status="OK"), 200
 
@@ -105,39 +101,54 @@ def deploy_web(repo_path: Path, remote_url: Path, ref: str, sha: str):
     return 'success', 'Deployed'
 
 
-def deploy_game(repo_path: Path, remote_url: Path, ref: str, sha: str):
-    checkout_repo(repo_path, remote_url, ref, sha)
-    mod_info = parse_mod_info(Path(repo_path, 'mod_info.lua'))
-    faf_modname = mod_info['_faf_modname']
-    files = build_mod(repo_path)
-    logger.info("Build result: {}".format(files))
-    deploy_path = Path(app.config['GAME_DEPLOY_PATH'], 'updates_{}_files'.format(mod_info['_faf_modname']))
-    logger.info("Deploying {} to {}".format(faf_modname, deploy_path))
-    for f in files:
-        destination = deploy_path / (f['filename'] + "." + faf_modname + "." + str(mod_info['version']) + f['sha1'][:6] + ".zip")
-        logger.info("Deploying {} to {}".format(f, destination))
-        shutil.copy2(str(f['path']), str(destination))
-        db.execute_sql('delete from updates_{}_files where fileId = %s and version = %s;'.format(faf_modname), (f['id'], mod_info['version']))
+def deploy_game(repo_path: Path, remote_url: Path, branch: str, game_mode: str, sha: str):
+    checkout_repo(repo_path, remote_url, branch, sha)  # Checkout the intended state on the server repo
+
+    mod_info = parse_mod_info(Path(repo_path, 'mod_info.lua'))  # Harvest data from mod_info.lua
+    version = str(mod_info['version'])
+
+    files = build_mod(repo_path)  # Build the mod from the fileset we just checked out
+    logger.info('Build result: {}'.format(files))
+
+    deploy_path = Path(app.config['GAME_DEPLOY_PATH'], 'updates_{}_files'.format(game_mode))
+    logger.info('Deploying {} to {}'.format(game_mode, deploy_path))
+
+    for file in files:
+        # Organise the files needed into their final setup and pack as .zip
+        destination = deploy_path / (file['filename'] + '.' + game_mode + '.' + version + file['sha1'][:6] + '.zip')
+        logger.info('Deploying {} to {}'.format(file, destination))
+        shutil.copy2(str(file['path']), str(destination))
+
+        # Update the database with the new mod
+        db.execute_sql('delete from updates_{}_files where fileId = %s and version = %s;'.format(game_mode),
+                       (file['id'], version))  # Out with the old
         db.execute_sql('insert into updates_{}_files '
                        '(fileId, version, md5, name) '
-                       'values (%s,%s,%s,%s)'.format(faf_modname),
-                       (f['id'], mod_info['version'], f['md5'], destination.name))
-    return 'success', 'Deployed'
+                       'values (%s,%s,%s,%s)'.format(game_mode),
+                       (file['id'], version, file['md5'], destination.name))  # In with the new
+
+    return 'Success', 'Deployed ' + repository + ' branch ' + branch + ' to ' + game_mode
 
-def deploy(repository, remote_url, ref, sha):
+
+def deploy_route(repository, branch, game_mode, commit):
     """
     Perform deployment on this machine
-    :param repository: the repository to deploy
-    :param ref: ref to fetch
-    :param sha: hash to verify deployment with
+    :param repository: the source repository
+    :param branch: the source branch
+    :param game_mode: the game mode we are deploying to
+    :param commit: commit hash to verify deployment
     :return: (status: str, description: str)
     """
+
+    github_url = app.config['GIT_URL']
+    remote_url = github_url + repository + '.git'
+
     try:
         return {
             'api': deploy_web,
             'patchnotes': deploy_web,
             'fa': deploy_game
-        }[repository](Path(app.config['REPO_PATHS'][repository]), remote_url, ref, sha)
+        }[repository](Path(app.config['REPO_PATHS'][repository]), remote_url, branch, game_mode, commit)
     except Exception as e:
         logger.exception(e)
         return 'error', "{}: {}".format(type(e), e)

api/git.py

@@ -8,15 +8,15 @@
     GIT_PATH = '/usr/bin/git'
 
 
-def checkout_repo(repo_path: Path, remote_url: Path, ref: str, sha: str):
+def checkout_repo(repo_path: Path, remote_url: Path, branch: str, commit: str):
     if not repo_path.exists():
         raise Exception("Repository to deploy doesn't exist")
     git_command = [GIT_PATH, '-C', str(repo_path)]
-    fetch_exit_code = subprocess.call(git_command + ['fetch', remote_url, ref])
+    fetch_exit_code = subprocess.call(git_command + ['fetch', remote_url, branch])
     if fetch_exit_code != 0:
         raise Exception("git fetch returned nonzero code: {}".format(fetch_exit_code))
     subprocess.call(git_command + ['checkout', '-f', 'FETCH_HEAD'])
     checked_out = subprocess.check_output(git_command + ['rev-parse', 'HEAD'],
                                           universal_newlines=True).strip()
-    if not checked_out == sha:
-        raise Exception("checked out hash {} doesn't match {}".format(checked_out, sha))
+    if not checked_out == commit:
+        raise Exception("checked out hash {} doesn't match {}".format(checked_out, commit))

config.example.py

@@ -16,9 +16,17 @@
 
 REPO_PATHS = {
     "api": '.',
+    "fa": '/opt/stable/featured_mods',
     "patchnotes": '/opt/dev/www/patchnotes'
 }
 
+DEPLOY_BRANCHES = {
+    # branch name : game mode
+    'refs/heads/master': 'faf',
+    'refs/heads/deploy/fafbeta': 'fafbeta',
+    'refs/heads/deploy/fafdevelop': 'fafdevelop'
+}
+
 GAME_DEPLOY_PATH = '/opt/dev/www/content/faf/updaterNew'
 MOD_UPLOAD_PATH = '/content/faf/vault/mods'
 MOD_THUMBNAIL_PATH = '/content/faf/vault/mods_thumbs'
@@ -30,6 +38,7 @@
 
 GITHUB_USER = 'some-user'
 GITHUB_TOKEN = 'some-token'
+GIT_URL = 'https://github.com/FAForever/'
 
 AUTO_DEPLOY = ['patchnotes']