Skip to content

Commit

Permalink
[#938] remote_release.py helper script
Browse files Browse the repository at this point in the history 
Summary:
- Added remote_release.py script to run yb_release on a remote server
- Modified remote_build.py to extract common stuff into the python/yb/remote.py
- Modified yb_release script to pass "skip-java" flag no matter what
- Changed remote.py to replace usage of an unspecified "args" map with specific args

Test Plan:
Ran both scriptis manually

Jenkins: skip

Reviewers: mikhail

Reviewed By: mikhail

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D6631
  • Loading branch information
@frozenspider
frozenspider committed May 17, 2019
1 parent 29a4a8a commit 8b6de48
Show file tree
Hide file tree
Showing 4 changed files with 362 additions and 194 deletions.
224 changes: 31 additions & 193 deletions bin/remote_build.py
View file
Expand Up @@ -16,82 +16,12 @@

import argparse
import os
import shlex
import subprocess
import sys
import time
import json

REMOTE_BUILD_HOST_ENV_VAR = 'YB_REMOTE_BUILD_HOST'
DEFAULT_BASE_BRANCH = 'origin/master'
sys.path.insert(0, os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'python')) # noqa


def check_output(args):
bytes = subprocess.check_output(args)
return bytes.decode('utf-8')


def check_output_line(args):
return check_output(args).strip()


def check_output_lines(args):
return [file.strip() for file in check_output(args).split('\n')]


def parse_name_status(lines):
files = ([], [])
for line in lines:
tokens = [x.strip() for x in line.split('\t')]
if len(tokens) == 0 or tokens[0] == '':
continue
if tokens[0] == 'D':
files[1].append(tokens[1])
continue
if tokens[0].startswith('R'):
name = tokens[2]
else:
name = tokens[1]
files[0].append(name)
return files


def remote_communicate(args, remote_command, error_ok=False):
args = ['ssh', args.host, remote_command]
proc = subprocess.Popen(args, shell=False)
proc.communicate()
if proc.returncode != 0:
if error_ok:
return False
else:
sys.exit(proc.returncode)
return True


def check_remote_files(escaped_remote_path, args, files):
remote_command = "cd {0} && git diff --name-status".format(escaped_remote_path)
remote_changed, remote_deleted = \
parse_name_status(check_output_lines(['ssh', args.host, remote_command]))
unexpected = []
for changed in remote_changed:
if changed not in files:
unexpected.append(changed)
if unexpected:
command = 'cd {0}'.format(args.remote_path)
message = 'Reverting:\n'
for file_path in unexpected:
message += ' {0}\n'.format(file_path)
command += ' && git checkout -- {0}'.format(shlex.quote(file_path))
print(message)
remote_communicate(args, command)


def remote_output_line(args, command):
return check_output_line(['ssh', args.host, 'cd {0} && {1}'.format(args.remote_path, command)])


def fetch_remote_commit(args):
return remote_output_line(args, 'git rev-parse HEAD')
from yb import remote


def add_extra_ybd_args(ybd_args, extra_args):
Expand All @@ -111,175 +41,83 @@ def add_extra_ybd_args(ybd_args, extra_args):
return ybd_args + extra_args


def read_config_file():
conf_file_path = os.path.expanduser('~/.yb_remote_build.json')
if not os.path.exists(conf_file_path):
return None
with open(conf_file_path) as conf_file:
return json.load(conf_file)


def main():
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument('--host', type=str, default=None,
help=('Host to build on. Can also be specified using the {} environment ' +
'variable.').format(REMOTE_BUILD_HOST_ENV_VAR))
'variable.').format(remote.REMOTE_BUILD_HOST_ENV_VAR))
home = os.path.expanduser('~')
cwd = os.getcwd()
default_path = '~/{0}'.format(cwd[len(home) + 1:] if cwd.startswith(home) else 'code/yugabyte')

# Note: don't specify default arguments here, because they may come from the "profile".
parser.add_argument('--remote-path', type=str, help='path used for build')
parser.add_argument('--branch', type=str, help='base branch for build')
parser.add_argument('--build-type', type=str, default=None, help='build type')
parser.add_argument('--remote-path', type=str,
help='path used for build')
parser.add_argument('--branch', type=str, default=None,
help='base branch for build')
parser.add_argument('--build-type', type=str, default=None,
help='build type')
parser.add_argument('--skip-build', action='store_true',
help='skip build, only sync files')
parser.add_argument('--wait-for-ssh', action='store_true',
help='Wait for the remote server to be ssh-able')
parser.add_argument('--profile',
help='Use a "profile" specified in the ~/.yb_remote_build.json file')
parser.add_argument('args', nargs=argparse.REMAINDER, help='arguments for yb_build.sh')
help='Use a "profile" specified in the {} file'.format(
remote.CONFIG_FILE_PATH))
parser.add_argument('build_args', nargs=argparse.REMAINDER,
help='arguments for yb_build.sh')

if len(sys.argv) >= 2 and sys.argv[1] in ['ybd', 'yb_build.sh']:
# Allow the first argument to be 'ybd' so we can copy and paste a ybd command line directly
# after remote_build.py.
sys.argv[1:2] = ['--']
args = parser.parse_args()

conf = read_config_file()

if conf and not args.profile:
args.profile = conf.get("default_profile", args.profile)

if args.profile:
profiles = conf['profiles']
profile = profiles.get(args.profile)
if profile is None:
# Match profile using the remote host.
for profile_name_to_try in profiles:
if profiles[profile_name_to_try].get('host') == args.profile:
profile = profiles[profile_name_to_try]
break
if profile is None:
raise ValueError("Unknown profile '%s'" % args.profile)
for arg_name in ['host', 'remote_path', 'branch']:
if getattr(args, arg_name) is None:
setattr(args, arg_name, profile.get(arg_name))
args.args += profile.get('extra_args', [])
remote.load_profile(['host', 'remote_path', 'branch'], args, args.profile)

# ---------------------------------------------------------------------------------------------
# Default arguments go here.

if args.host is None and REMOTE_BUILD_HOST_ENV_VAR in os.environ:
args.host = os.environ[REMOTE_BUILD_HOST_ENV_VAR]
args.host = remote.apply_default_host_value(args.host)

if args.branch is None:
args.branch = DEFAULT_BASE_BRANCH
args.branch = remote.DEFAULT_BASE_BRANCH

if args.remote_path is None:
args.remote_path = default_path

# End of default arguments.
# ---------------------------------------------------------------------------------------------

if args.host is None:
sys.stderr.write(
"Please specify host with --host option or {} variable\n".format(
REMOTE_BUILD_HOST_ENV_VAR))
sys.exit(1)

os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

print("Host: {0}, build type: {1}, remote path: {2}".format(args.host,
args.build_type or 'N/A',
args.remote_path))
print("Arguments to remote yb_build.sh: {}".format(args.args))
commit = check_output_line(['git', 'merge-base', args.branch, 'HEAD'])
print("Base commit: {0}".format(commit))

if args.wait_for_ssh:
while not remote_communicate(args, 'true', error_ok=True):
print("Remote host is unavailabe, re-trying")
time.sleep(1)

remote_commit = fetch_remote_commit(args)

if args.remote_path.startswith('~/'):
escaped_remote_path = '$HOME/' + shlex.quote(args.remote_path[2:])
else:
escaped_remote_path = shlex.quote(args.remote_path)

if remote_commit != commit:
print("Remote commit mismatch, syncing")
remote_command = 'cd {0} && '.format(escaped_remote_path)
remote_command += 'git checkout -- . && '
remote_command += 'git clean -f . && '
remote_command += 'git checkout master && '
remote_command += 'git pull && '
remote_command += 'git checkout {0}'.format(commit)
remote_communicate(args, remote_command)
remote_commit = fetch_remote_commit(args)
if remote_commit != commit:
sys.stderr.write("Failed to sync remote commit to: {0}, it is still: {1}".format(
commit, remote_commit))
sys.exit(1)

files, del_files = \
parse_name_status(check_output_lines(['git', 'diff', commit, '--name-status']))
print("Total files: {0}, deleted files: {1}".format(len(files), len(del_files)))

if files:
# From this StackOverflow thread: https://goo.gl/xzhBUC
# The -a option is equivalent to -rlptgoD. You need to remove the -t. -t tells rsync to
# transfer modification times along with the files and update them on the remote system.
#
# Another relevant one -- how to make rsync preserve timestamps of unchanged files:
# https://goo.gl/czD96F
#
# We are using "rlpcgoD" instead of "rlptgoD" (with "t" replaced with "c").
# The goal is to use checksums for deciding what files to skip.
rsync_args = ['rsync', '-rlpcgoDvR']
rsync_args += files
rsync_args += ["{0}:{1}".format(args.host, args.remote_path)]
proc = subprocess.Popen(rsync_args, shell=False)
proc.communicate()
if proc.returncode != 0:
sys.exit(proc.returncode)

if del_files:
remote_command = 'cd {0} && rm -f '.format(escaped_remote_path)
for file in del_files:
remote_command += shlex.quote(file)
remote_command += ' '
remote_communicate(args, remote_command)
print("Arguments to remote build: {}".format(args.build_args))

check_remote_files(escaped_remote_path, args, files)
escaped_remote_path = \
remote.sync_changes(args.host, args.branch, args.remote_path, args.wait_for_ssh)

if args.skip_build:
sys.exit(0)

ybd_args = []
remote_args = []
if args.build_type:
ybd_args.append(args.build_type)
remote_args.append(args.build_type)

if len(args.args) != 0 and args.args[0] == '--':
ybd_args += args.args[1:]
if len(args.build_args) != 0 and args.build_args[0] == '--':
remote_args += args.build_args[1:]
else:
ybd_args += args.args
remote_args += args.build_args

if '--host-for-tests' not in ybd_args and 'YB_HOST_FOR_RUNNING_TESTS' in os.environ:
ybd_args = add_extra_ybd_args(ybd_args,
['--host-for-tests', os.environ['YB_HOST_FOR_RUNNING_TESTS']])
if '--host-for-tests' not in remote_args and 'YB_HOST_FOR_RUNNING_TESTS' in os.environ:
remote_args = add_extra_ybd_args(remote_args,
['--host-for-tests',
os.environ['YB_HOST_FOR_RUNNING_TESTS']])

remote_command = "cd {0} && ./yb_build.sh".format(escaped_remote_path)
for arg in ybd_args:
remote_command += " {0}".format(shlex.quote(arg))
print("Remote command: {0}".format(remote_command))
# Let's not use subprocess if the output is potentially large:
# https://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/
ssh_path = subprocess.check_output(['which', 'ssh']).strip()
ssh_args = [ssh_path, args.host, remote_command]
os.execv(ssh_path, ssh_args)
remote.exec_command(args.host, escaped_remote_path, 'yb_build.sh', remote_args,
do_quote_args=True)


if __name__ == '__main__':
Expand Down
103 changes: 103 additions & 0 deletions bin/remote_release.py
View file
@@ -0,0 +1,103 @@
#!/usr/bin/env python3

#
# Copyright (c) YugaByte, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations
# under the License.
#

import argparse
import os
import sys

sys.path.insert(0, os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'python')) # noqa

from yb import remote


def main():
parser = argparse.ArgumentParser(prog=sys.argv[0])
parser.add_argument('--host', type=str, default=None,
help=('Host to build on. Can also be specified using the {} environment ' +
'variable.').format(remote.REMOTE_BUILD_HOST_ENV_VAR))
home = os.path.expanduser('~')
cwd = os.getcwd()
default_path = '~/{0}'.format(cwd[len(home) + 1:] if cwd.startswith(home) else 'code/yugabyte')

# Note: don't specify default arguments here, because they may come from the "profile".
parser.add_argument('--remote-path', type=str, default=None,
help='path used for build')
parser.add_argument('--branch', type=str, default=None,
help='base branch for build')
parser.add_argument('--build-type', type=str, default=None,
help='build type, defaults to release')
parser.add_argument('--skip-build', action='store_true',
help='skip build, only sync files')
parser.add_argument('--build-args', type=str, default=None,
help='build arguments to pass')
parser.add_argument('--edition', type=str, default=None,
help='use ee or (default) ce edition')
parser.add_argument('--wait-for-ssh', action='store_true',
help='Wait for the remote server to be ssh-able')
parser.add_argument('--profile',
help='Use a "profile" specified in the {} file'.format(
remote.CONFIG_FILE_PATH))

args = parser.parse_args()

remote.load_profile(['host', 'remote_path', 'branch'], args, args.profile)

# ---------------------------------------------------------------------------------------------
# Default arguments go here.

args.host = remote.apply_default_host_value(args.host)

if args.branch is None:
args.branch = remote.DEFAULT_BASE_BRANCH

if args.remote_path is None:
args.remote_path = default_path

if args.build_type is None:
args.build_type = "release"

if args.edition is None:
args.edition = "ce"

# End of default arguments.
# ---------------------------------------------------------------------------------------------

os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

print("Host: {0}, build type: {1}, remote path: {2}".format(args.host,
args.build_type,
args.remote_path))
print("Arguments to remote build: {}".format(args.build_args))

escaped_remote_path = remote.sync_changes(args.host, args.branch, args.remote_path,
args.wait_for_ssh)

if args.skip_build:
sys.exit(0)

remote_args = []
remote_args.append("--build {}".format(args.build_type))
remote_args.append("--edition {}".format(args.edition))
remote_args.append("--force")
if args.build_args is not None:
remote_args.append("--build_args=\"{}\"".format(args.build_args))

remote.exec_command(args.host, escaped_remote_path, 'yb_release', remote_args, False)


if __name__ == '__main__':
main()

0 comments on commit 8b6de48

Please sign in to comment.