Add nightly deployment

Automates deploying nightly builds to the dropbox shared folder and to
each of Wrye Bash's nexus pages.

Dropbox:
 - Uses dropbox's cli oauth to allow access to the shared folder;
 - The oauth process above can be overridden by providing your own
   access token;
 - Will deploy to the root of the shared folder - if previous builds are
   present it will delete them before uploading the new ones;
 - Test/Branch builds are supported by deploying to a subfolder via the
   '--branch' flag.

Nexus:
 - Requires Google Chrome installed and an active nexus session in
   either Chrome, Firefox or Safari;
 - The appropriate chromedriver for the Chrome version is automatically
   downloaded;
 - The required nexus auth cookies ('member_id', 'pass_hash', and 'sid')
   are automatically retrieved for one of the above browsers;
 - Both chromedriver version and auth cookies can be overridden;
 - Selenium will run Chrome headless while deploying to prevent
   accidental user input;
 - If previous builds are present in the Updates section that match the
   current regex they will be deleted and replaced by the newer files.

A log is kept of all verbose output (sensitive information is not
logged). The path to this log can be changed via '--logfile'.

The authentication credentials are stored in plain text to ease
multiple deployments - this can be disabled via '--no-config'.

For testing purposes, a dry run can be executed via '--dry-run'.

The folder containing distributables to deploy can be changed via
'--dist-folder'.

It is recommended to always use 'deploy.py' to deploy - disabling
deployment to either target can be via '--no-dropbox' or '--no-nexus'.

.gitignore

@@ -30,6 +30,9 @@ scripts/dist/*.7z
 # Local NSIS
 scripts/build/nsis/
 
+# Deploy
+scripts/deploy/
+
 # translation files
 Mopy/bash/l10n/*.mo
 Mopy/bash/l10n/*NEW.txt

requirements.txt

@@ -5,4 +5,7 @@ https://github.com/wrye-bash/dev-tools/raw/master/wheels/wxPython-3.0.2.0-cp27-c
 scandir>=1.9.0
 # Compile/Build-time
 pygit2>=0.28.0
+dropbox>=9.3.0
+selenium>=3.141.0
+browsercookie>=0.6.2
 https://github.com/wrye-bash/dev-tools/raw/master/wheels/py2exe-0.6.9-cp27-cp27m-win32.whl

scripts/deploy.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# GPL License and Copyright Notice ============================================
+#  This file is part of Wrye Bash.
+#
+#  Wrye Bash is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  Wrye Bash is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Wrye Bash; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  Wrye Bash copyright (C) 2005-2009 Wrye, 2010-2019 Wrye Bash Team
+#  https://github.com/wrye-bash
+#
+# =============================================================================
+
+"""
+Deploy nightly builds.
+
+To deploy to dropbox you need a Dropbox account and access to the wrye bash
+shared folder in your dropbox. Optionally, you can override the regular oauth
+process with your own API access token via '--access-token'. Deploying test
+builds to a subfolder is also supported with '--branch'.
+
+To deploy to nexus you need to have Google Chrome/Chromium installed and be
+logged in to nexus in a browser [Supported: Chrome, Firefox, Safari]. The
+appropriate selenium driver and the needed auth cookies are automatically
+retrieved. If there are issues with this process, you can override the
+chromedriver version with '--driver-version' and the auth cookie values with
+'--member-id', '--pass-hash' and '--sid'.
+
+Unless '--no-config' is supplied, all values are saved to a
+configuration file at './deploy_config.json'. Values are
+stored as a dictionary with the format (keys in lowercase):
+    '%ARGUMENT%': '%VALUE%'
+"""
+
+import argparse
+
+import deploy_dropbox
+import deploy_nexus
+import utils
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    utils.setup_deploy_parser(parser)
+    parser.add_argument(
+        "--no-dropbox",
+        action="store_false",
+        dest="dropbox",
+        help="Do not deploy to dropbox.",
+    )
+    parser.add_argument(
+        "--no-nexus",
+        action="store_false",
+        dest="nexus",
+        help="Do not deploy to nexusmods.",
+    )
+    dropbox_parser = parser.add_argument_group("dropbox arguments")
+    deploy_dropbox.setup_parser(dropbox_parser)
+    nexus_parser = parser.add_argument_group("nexus arguments")
+    deploy_nexus.setup_parser(nexus_parser)
+    args = parser.parse_args()
+    open(args.logfile, "w").close()
+    if args.dropbox:
+        deploy_dropbox.main(args)
+        print
+    if args.nexus:
+        deploy_nexus.main(args)

scripts/deploy_dropbox.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# GPL License and Copyright Notice ============================================
+#  This file is part of Wrye Bash.
+#
+#  Wrye Bash is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  Wrye Bash is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Wrye Bash; if not, write to the Free Software Foundation,
+#  Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+#  Wrye Bash copyright (C) 2005-2009 Wrye, 2010-2019 Wrye Bash Team
+#  https://github.com/wrye-bash
+#
+# =============================================================================
+
+import argparse
+import json
+import logging
+import os
+import re
+
+import dropbox
+
+import utils
+
+LOGGER = logging.getLogger(__name__)
+
+# constants
+SHARED_FOLDER_ID = "4796182912"
+FILE_REGEX = r"Wrye Bash \d{3,}\.\d{12,12} - (Installer.exe|Python Source.7z|Standalone Executable.7z)"
+COMPILED = re.compile(FILE_REGEX)
+
+
+def setup_parser(parser):
+    parser.add_argument(
+        "-t",
+        "--access-token",
+        default=argparse.SUPPRESS,
+        help="The dropbox API access token.\n"
+        "  To get your own access token\n"
+        "  go to https://www.dropbox.com/developers/apps\n"
+        "  register an app and generate your token.",
+    )
+    parser.add_argument(
+        "-b",
+        "--branch",
+        default="",
+        help="Upload a specific branch.\n"
+        "  Will upload to a separate folder\n"
+        "  within the shared folder.",
+    )
+
+
+def remove_files(dbx, path, dry_run=False):
+    # get all files in folder
+    files = []
+    for entry in dbx.files_list_folder(path).entries:
+        if isinstance(entry, dropbox.files.FileMetadata):
+            files.append(entry.name)
+            LOGGER.debug("Found '{}' under shared folder.".format(entry.name))
+    # delete the previous nightly files
+    filtered = filter(COMPILED.match, files)
+    for fname in filtered:
+        fpath = path + "/" + fname
+        if dry_run:
+            LOGGER.info("Would remove '{}'.".format(fpath))
+            continue
+        LOGGER.info("Removing '{}'...".format(fpath))
+        dbx.files_delete_v2(fpath)
+
+
+def upload_file(dbx, fpath, folder_path, dry_run=False):
+    fname = os.path.basename(fpath)
+    LOGGER.debug("Found '{}' under distributable folder.".format(fname))
+    if dry_run:
+        LOGGER.info("Would upload '{}'.".format(os.path.relpath(fpath, os.getcwd())))
+        return
+    LOGGER.info("Uploading '{}'...".format(os.path.relpath(fpath, os.getcwd())))
+    with open(fpath, "rb") as fopen:
+        upload_path = folder_path + "/" + fname
+        dbx.files_upload(fopen.read(), upload_path)
+
+
+def main(args):
+    utils.setup_log(LOGGER, verbosity=args.verbosity, logfile=args.logfile)
+    creds = utils.parse_deploy_credentials(args, ["access_token"], args.save_config)
+    # setup dropbox instance
+    dbx = dropbox.Dropbox(creds["access_token"])
+    shared_folder_path = dbx.sharing_get_folder_metadata(SHARED_FOLDER_ID).path_lower
+    LOGGER.debug("Found shared folder path at '{}'.".format(shared_folder_path))
+    # create folder inside shared folder if needed for branch nightly
+    if args.branch:
+        shared_folder_path += "/" + args.branch
+        try:
+            dbx.files_create_folder_v2(shared_folder_path)
+        except dropbox.exceptions.ApiError:
+            pass
+        LOGGER.debug(
+            "Using branch folder '{}' at '{}'.".format(args.branch, shared_folder_path)
+        )
+    remove_files(dbx, shared_folder_path, args.dry_run)
+    for fname in os.listdir(args.dist_folder):
+        fpath = os.path.join(args.dist_folder, fname)
+        if not os.path.isfile(fpath):
+            continue
+        upload_file(dbx, fpath, shared_folder_path, args.dry_run)
+
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser()
+    utils.setup_deploy_parser(argparser)
+    setup_parser(argparser)
+    parsed_args = argparser.parse_args()
+    open(parsed_args.logfile, "w").close()
+    main(parsed_args)