Flatten MedaWiki deploy into a single git repo

Tags: #release-engineering-team

Maniphest Tasks: T147478

Summary:
After the local compilation of wikiversions on the deployment master,
add the entirey of /srv/mediawiki to a unified git repository.

This doesn't yet attempt to change the transport mechanism for sync, nor
does it use much of the scap3 code, but it gets us to a single version
number for deployment.

This also overwrites the mediawiki-config repo as all php-* would be
ignored otherwise.

Fixes T147478

Reviewers: bd808, dduvall, demon, mmodell, #release-engineering-team

Reviewed By: mmodell, #release-engineering-team

Subscribers: jenkins

Differential Revision: https://phabricator.wikimedia.org/D429

scap/deploy.py

@@ -13,6 +13,8 @@
 import time
 import yaml
 import errno
+import subprocess
+
 from datetime import datetime
 
 from . import checks
@@ -850,3 +852,37 @@ def latest_log_file():
 
     def _setup_loggers(self):
         pass
+
+
+@cli.command('deploy-mediawiki', help=argparse.SUPPRESS)
+class DeployMediaWiki(cli.Application):
+    """
+    Deploy mediawiki via scap3.
+
+    Ideally, this class and command will be fully merged with
+    the `scap deploy` command by the end of the transition; however, there
+    may also be unique reasons, particularly on the deployment masters, to
+    keep this command
+    """
+
+    @cli.argument('message', nargs='*', help='Log message for git')
+    def main(self, *extra_args):
+        """Run deploy-mediawiki."""
+        # Flatten local into git repo
+        self.get_logger().info('scap deploy-mediawiki')
+        git.default_ignore(self.config['deploy_dir'])
+
+        git.add_all(self.config['deploy_dir'], message=self.arguments.message)
+
+        scap = self.get_script_path()
+        options = {
+            'git_repo': self.config['deploy_dir'],
+        }
+
+        option_list = ['-D{}:{}'.format(x, y) for x, y in options.iteritems()]
+        cmd = [scap, 'deploy', '-v']
+        cmd += option_list
+        cmd += ['--init']
+
+        with utils.cd(self.config['deploy_dir']):
+            subprocess.check_call(cmd)

scap/git.py

@@ -9,13 +9,23 @@
 from datetime import datetime
 import errno
 import os
+import socket
 import subprocess
 
 from . import utils
 
 import yaml
 
 
+# Key is the pattern for .gitignore, value is a test for that pattern.
+DEFAULT_IGNORE = {
+    '*~',
+    '*.swp',
+    '*/cache/l10n/*.cdb',
+    'scap/log/*',
+}
+
+
 def info_filename(directory, install_path, cache_path):
     """Compute the path for a git_info cache file related to a given
     directory.
@@ -139,6 +149,44 @@ def info(directory):
     }
 
 
+def default_ignore(location):
+    """Create a default .gitignore file."""
+    ignore = '\n'.join(DEFAULT_IGNORE)
+    with utils.cd(location):
+        with open('.gitignore', 'w+') as f:
+            f.write(ignore)
+
+
+def add_all(location, message='Update'):
+    """Add everything to repo at location as user."""
+    git = '/usr/bin/git'
+    with utils.cd(location):
+        # Initialize repo if it isn't already
+        if not is_dir(location):
+            cmd = [git, 'init']
+            subprocess.check_call(cmd)
+
+        cmd = [git, 'add', '--all']
+        subprocess.check_call(cmd)
+
+        host = socket.getfqdn()
+        euid = utils.get_username()
+        ruid = utils.get_real_username()
+        ename = utils.get_user_fullname()
+        rname = utils.get_real_user_fullname()
+
+        os.environ['GIT_COMMITTER_EMAIL'] = '{}@{}'.format(euid, host)
+        os.environ['GIT_AUTHOR_EMAIL'] = '{}@{}'.format(ruid, host)
+
+        os.environ['GIT_COMMITTER_NAME'] = ename
+        os.environ['GIT_AUTHOR_NAME'] = rname
+
+        cmd = [git, 'commit', '-m', message]
+
+        # Soft errors if nothing new to commit
+        subprocess.call(cmd)
+
+
 def next_deploy_tag(location):
     """Calculates the scap/sync/{date}/{n} tag to use for this deployment"""
     ensure_dir(location)

scap/main.py

@@ -35,9 +35,12 @@ def main(self, *extra_args):
         print utils.logo()
         self._assert_auth_sock()
 
+        self.include = None
+
         with utils.lock(self.config['lock_file'], self.arguments.message):
             self._check_sync_flag()
             self._before_cluster_sync()
+            self._git_repo()
             self._sync_masters()
 
             # Run canary checks
@@ -197,6 +200,33 @@ def _apache_sync_command(self, proxies):
         """
         return self._proxy_sync_command() + proxies
 
+    def _git_repo(self):
+        """Flatten deploy directory into shared git repo."""
+        includes = None
+
+        if self.include is not None:
+            includes = []
+
+            parts = self.include.split('/')
+            for i in range(1, len(parts)):
+                # Include parent directories in sync command or the default
+                # exclude will block them and by extension block the target
+                # file.
+                includes.append('/'.join(parts[:i]))
+
+            includes.append(self.include)
+
+        tasks.sync_common(
+            self.config,
+            include=includes,
+            verbose=self.verbose
+        )
+
+        self.get_logger().info('Setting up deploy git directory')
+        cmd = '{} deploy-mediawiki -v "{}"'.format(
+            self.get_script_path(), self.arguments.message)
+        utils.sudo_check_call('mwdeploy', cmd)
+
     def _after_cluster_sync(self):
         pass
 
@@ -329,6 +359,7 @@ class Scap(AbstractSync):
     #. Compile wikiversions.json to php in deploy directory
     #. Update l10n files in staging area
     #. Compute git version information
+    #. Commit all changes to local git repo in deploy directory
     #. Ask scap masters to sync with current master
     #. Ask scap proxies to sync with master server
     #. Ask apaches to sync with fastest rsync server
@@ -353,9 +384,6 @@ def _before_cluster_sync(self):
             '%(stage_dir)s/wmf-config' % self.config,
             '%(stage_dir)s/multiversion' % self.config)
 
-        # Sync deploy directory on localhost with staging area
-        tasks.sync_common(self.config)
-
         # Bug 63659: Compile deploy_dir/wikiversions.json to cdb
         cmd = '{} wikiversions-compile'.format(self.get_script_path())
         utils.sudo_check_call('mwdeploy', cmd)
@@ -691,8 +719,11 @@ def main(self, *extra_args):
             err_msg = 'l10n cache missing for %s' % version
             utils.check_file_exists(cache_file, err_msg)
 
+        # this is here for git_repo
+        self.include = '/wikiversions*.{json,php}'
         with utils.lock(self.config['lock_file'], self.arguments.message):
             self._check_sync_flag()
+            self._git_repo()
             self._sync_masters()
             mw_install_hosts = self._get_target_list()
             tasks.sync_wikiversions(mw_install_hosts, self.config)

scap/utils.py

@@ -231,6 +231,19 @@ def get_real_username():
         return get_username()
 
 
+def get_real_user_fullname():
+    """Return the first entry in GECOS field for real user."""
+    return get_user_fullname(get_real_username())
+
+
+def get_user_fullname(name=None):
+    """Return the first entry in GECOS field for name."""
+    if name is None:
+        name = get_username()
+
+    return pwd.getpwnam(name).pw_gecos.split(',')[0]
+
+
 def get_env_specific_filename(path, env=None):
     """Find a file specific to the environment in which scap is running."""
     if env is None: