Merged in dev/gideon/sync_to_stack_2021.02.12 (pull request elastic#55)

* Initial implementation of sync_to_stack

Script to update stack from local code, e.g. to sync to HEAD or master or other
branch.

The script will:
* check and update repos per command line options (currently support one branch
  name for all repos)
* find and classify stack instances
* sync code to engageli and dequeue (recorder/merger) nodes

Approved-by: Matan Yemini

aws/misc/sync_to_stack.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import subprocess
+import sys
+
+import boto3
+import botocore
+
+from collections import defaultdict
+from pprint import pprint
+
+GIT_ROOT = os.path.dirname(subprocess.check_output(('git', 'rev-parse', '--show-toplevel'))).decode(encoding='UTF-8')
+REPOS = (
+    'admin-portal',
+    'backend1',
+    'dequeue',
+    'devops',
+    'engageli-media-server',
+    'media-manager',
+    'pet',
+    )
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-r', '--region', type=str, default=None,
+                        help='Override default region')
+    parser.add_argument('-s', '--stack-name', type=str, required=True,
+                        help='Destination stack name')
+    parser.add_argument('-b', '--sandbox', type=str, default='staging',
+                        help='Name of sandbox to update')
+    parser.add_argument('--pull', action="store_true",
+                        help='call git pull to get HEAD')
+    parser.add_argument('--clean', action="store_true",
+                        help='check clean git status')
+    parser.add_argument('--branch', type=str,
+                        help='switch git branch')
+    parser.add_argument('--skip-build', action='store_true',
+                        help='Skip building web apps')
+    return parser.parse_args()
+
+def find_stack_instances(ec2, stack_name):
+    ''' Find stack's instances and return object with role as key and values are
+        lists of public IPs
+    '''
+    instances = defaultdict(list)
+    stack_instances = ec2.describe_instances(Filters=[{'Name': 'tag:StackName', 'Values': [stack_name]}])
+    for res in stack_instances['Reservations']:
+        for instance in res['Instances']:
+            for tag in instance['Tags']:
+                if tag['Key'] == 'Role':
+                    role = tag['Value']
+                    instances[role].append(instance['PublicIpAddress'])
+    pprint(instances)
+    return instances
+
+def isGitClean():
+    status = subprocess.check_output(('git', 'status', '--porcelain', '--untracked-files=no'))
+    if status:
+        print('git not clean - %s:\n%s' % (os.getcwd(), status.decode(encoding='UTF-8')))
+        return True
+    return False
+
+def check_update_repos(args):
+    for repo in REPOS:
+        os.chdir(os.path.join(GIT_ROOT, repo))
+        if args.clean and not isGitClean():
+            return False
+        if args.branch:
+            # call fetch to ensure branch is available locally
+            subprocess.check_call(('git', 'fetch'))
+            subprocess.check_call(('git', 'checkout', args.branch))
+        if args.pull:
+            subprocess.check_call(('git', 'pull'))
+            if repo not in ('admin-portal', 'pet'):
+                subprocess.check_call(('git', 'submodule', 'update', '--init'))
+    return True
+
+def sync_to_engageli_nodes(instances, sandbox, skip_build):
+    for instance in instances:
+        print('*' * 78)
+        print(f'* Syncing engageli to {instance}')
+        print('*' * 78)
+        # backend1
+        os.chdir(os.path.join(GIT_ROOT, 'backend1'))
+        subprocess.check_call(('./utils/sync_to_host.sh', '-H', instance, '-n', sandbox))
+        # student app
+        os.chdir(os.path.join(GIT_ROOT, 'pet'))
+        # TODO: add option to specify student app subdir
+        sync_cmd = ('./utils/sync_to_host.sh', '-H', instance, '-n', sandbox)
+        if skip_build:
+            sync_cmd += ('-s',)
+        subprocess.check_call(sync_cmd)
+        # media-manager backend
+        os.chdir(os.path.join(GIT_ROOT, 'media-manager/fm'))
+        subprocess.check_call(('./utils/sync_to_host.sh', '-H', instance, '-n', sandbox))
+        # TODO:
+        # * media-manager frontend
+        # * admin-portal
+        # * admin.js
+
+def sync_to_dequeue_nodes(instances):
+    os.chdir(os.path.join(GIT_ROOT, 'dequeue'))
+    for instance in instances:
+        print('*' * 78)
+        print(f'* Syncing dequeue to {instance}')
+        print('*' * 78)
+        subprocess.check_call(('./utils/sync_to_host.sh', '-H', instance))
+
+def main():
+    args = parse_args()
+    if not check_update_repos(args):
+        return
+    ec2 = boto3.client('ec2', args.region)
+    instances = find_stack_instances(ec2, args.stack_name)
+    sync_to_engageli_nodes(instances['engageli-api'], args.sandbox, args.skip_build)
+    sync_to_dequeue_nodes(instances['merger'] + instances['recorder'])
+
+if __name__ == '__main__':
+    main()