Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix branch updates and make Linux compatible #5224

Merged
merged 2 commits into from Oct 13, 2017
Merged

Fix branch updates and make Linux compatible #5224

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d2d2b68
Fix the prepare_branch function
adbridge Sep 27, 2017
31407a1
Move to using command strings rather than command lists.
adbridge Sep 29, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
158 changes: 80 additions & 78 deletions tools/test/examples/update.py
View file
Open in desktop
Expand Up @@ -42,9 +42,6 @@
# -c <config file> - Optional path to an examples file.
# If not proved the default is 'examples.json'
# -T <github_token> - GitHub token for secure access (required)
# -l <logging level> - Optional Level for providing logging output. Can be one of,
# CRITICAL, ERROR, WARNING, INFO, DEBUG
# If not provided the default is 'INFO'
# -f - Update forked repos. This will use the 'github-user' parameter in
# the 'via-fork' section.
# -b - Update branched repos. This will use the "src-branch" and
Expand Down Expand Up @@ -75,66 +72,76 @@
import examples_lib as lib
from examples_lib import SUPPORTED_TOOLCHAINS

userlog = logging.getLogger("Update")

# Set logging level
userlog.setLevel(logging.DEBUG)

# Everything is output to the log file
logfile = os.path.join(os.getcwd(), 'update.log')
fh = logging.FileHandler(logfile)
fh.setLevel(logging.DEBUG)

# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)

formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s')
ch.setFormatter(formatter)
fh.setFormatter(formatter)

# add the handlers to the logger
userlog.addHandler(fh)
userlog.addHandler(ch)

def run_cmd(command, exit_on_failure=False):
""" Run a system command and return the result status
""" Run a system command returning a status result

Description:
This is just a wrapper for the run_cmd_with_output() function, but
only returns the status of the call.

Passes a command to the system and returns a True/False result, once the
command has been executed, indicating success/failure. Commands are passed
as a list of tokens.
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']

Args:
command - system command as a list of tokens
exit_on_failure - If True exit the program on failure (default = False)

Returns:
return_code - True/False indicating the success/failure of the command
"""
update_log.debug('[Exec] %s', ' '.join(command))
return_code = subprocess.call(command, shell=True)

if return_code:
update_log.warning("Command '%s' failed with return code: %s",
' '.join(command), return_code)
if exit_on_failure:
sys.exit(1)

return_code, _ = run_cmd_with_output(command, exit_on_failure)
return return_code

def run_cmd_with_output(command, exit_on_failure=False):
""" Run a system command and return the result status plus output
""" Run a system command returning a status result and any command output

Description:

Passes a command to the system and returns a True/False result once the
command has been executed, indicating success/failure. If the command was
successful then the output from the command is returned to the caller.
Commands are passed as a list of tokens.
E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
Commands are passed as a string.
E.g. The command 'git remote -v' would be passed in as "git remote -v"

Args:
command - system command as a list of tokens
command - system command as a string
exit_on_failure - If True exit the program on failure (default = False)

Returns:
returncode - True/False indicating the success/failure of the command
return_code - True/False indicating the success/failure of the command
output - The output of the command if it was successful, else empty string
"""
update_log.debug('[Exec] %s', ' '.join(command))
text = '[Exec] ' + command
userlog.debug(text)
returncode = 0
output = ""
try:
output = subprocess.check_output(command, shell=True)
except subprocess.CalledProcessError as e:
update_log.warning("Command '%s' failed with return code: %s",
' '.join(command), e.returncode)
text = "The command " + str(command) + "failed with return code: " + str(e.returncode)
userlog.warning(text)
returncode = e.returncode
if exit_on_failure:
sys.exit(1)
return returncode, output


def rmtree_readonly(directory):
""" Deletes a readonly directory tree.

Expand Down Expand Up @@ -198,7 +205,7 @@ def upgrade_single_example(example, tag, directory, ref):

os.rename("mbed-os.lib", "mbed-os.lib_bak")
else:
update_log.error("Failed to backup mbed-os.lib prior to updating.")
userlog.error("Failed to backup mbed-os.lib prior to updating.")
return False

# mbed-os.lib file contains one line with the following format
Expand All @@ -221,7 +228,7 @@ def upgrade_single_example(example, tag, directory, ref):

if updated:
# Setup and run the git add command
cmd = ['git', 'add', 'mbed-os.lib']
cmd = "git add mbed-os.lib"
return_code = run_cmd(cmd)

os.chdir(cwd)
Expand All @@ -242,12 +249,12 @@ def prepare_fork(arm_example):
"""

logstr = "In: " + os.getcwd()
update_log.debug(logstr)
userlog.debug(logstr)

for cmd in [['git', 'remote', 'add', 'armmbed', arm_example],
['git', 'fetch', 'armmbed'],
['git', 'reset', '--hard', 'armmbed/master'],
['git', 'push', '-f', 'origin']]:
for cmd in ["git remote add armmbed " + str(arm_example),
"git fetch armmbed",
"git reset --hard armmbed/master",
"git push -f origin"]:
run_cmd(cmd, exit_on_failure=True)

def prepare_branch(src, dst):
Expand All @@ -265,25 +272,34 @@ def prepare_branch(src, dst):

"""

update_log.debug("Preparing branch: %s", dst)
userlog.debug("Preparing branch: %s", dst)

# Check if branch already exists or not.
cmd = ['git', 'branch']
# We can use the 'git branch -r' command. This returns all the remote branches for
# the current repo.
# The output consists of a list of lines of the form:
# origin/<branch>
# From these we need to extract just the branch names to a list and then check if
# the specified dst exists in that list
branches = []
cmd = "git branch -r"
_, output = run_cmd_with_output(cmd, exit_on_failure=True)

if not dst in output:
branches = [line.split('/')[1] for line in output.split('\n') if 'origin' in line and not '->' in line]

if not dst in branches:

# OOB branch does not exist thus create it, first ensuring we are on
# the src branch and then check it out

for cmd in [['git', 'checkout', src],
['git', 'checkout', '-b', dst],
['git', 'push', '-u', 'origin', dst]]:
for cmd in ["git checkout " + str(src),
"git checkout -b " + str(dst),
"git push -u origin " + str(dst)]:

run_cmd(cmd, exit_on_failure=True)

else:
cmd = ['git', 'checkout', dst]
cmd = "git checkout " + str(dst)
run_cmd(cmd, exit_on_failure=True)

def upgrade_example(github, example, tag, ref, user, src, dst, template):
Expand Down Expand Up @@ -321,18 +337,18 @@ def upgrade_example(github, example, tag, ref, user, src, dst, template):
user = 'ARMmbed'

ret = False
update_log.info("Updating example '%s'", example['name'])
update_log.debug("User: %s", user)
update_log.debug("Src branch: %s", (src or "None"))
update_log.debug("Dst branch: %s", (dst or "None"))
userlog.info("Updating example '%s'", example['name'])
userlog.debug("User: %s", user)
userlog.debug("Src branch: %s", (src or "None"))
userlog.debug("Dst branch: %s", (dst or "None"))

cwd = os.getcwd()

update_repo = "https://github.com/" + user + '/' + example['name']
update_log.debug("Update repository: %s", update_repo)
userlog.debug("Update repository: %s", update_repo)

# Clone the example repo
clone_cmd = ['git', 'clone', update_repo]
clone_cmd = "git clone " + str(update_repo)
return_code = run_cmd(clone_cmd)

if not return_code:
Expand All @@ -353,30 +369,27 @@ def upgrade_example(github, example, tag, ref, user, src, dst, template):
os.chdir(cwd)
return False

# Setup the default commit message
commit_message = 'Updating mbed-os to ' + tag

# Setup and run the commit command
commit_cmd = ['git', 'commit', '-m', commit_message]
commit_cmd = "git commit -m \"Updating mbed-os to " + tag + "\""
return_code = run_cmd(commit_cmd)
if not return_code:

# Setup and run the push command
push_cmd = ['git', 'push', 'origin']
push_cmd = "git push origin"
return_code = run_cmd(push_cmd)

if not return_code:
# If the user is not ARMmbed then a fork is being used
if user != 'ARMmbed':

upstream_repo = 'ARMmbed/'+ example['name']
update_log.debug("Upstream repository: %s", upstream_repo)
userlog.debug("Upstream repository: %s", upstream_repo)
# Check access to mbed-os repo
try:
repo = github.get_repo(upstream_repo, False)

except:
update_log.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
userlog.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
return False

jinja_loader = FileSystemLoader(template)
Expand All @@ -391,15 +404,15 @@ def upgrade_example(github, example, tag, ref, user, src, dst, template):
ret = True
except GithubException as e:
# Default to False
update_log.error("Pull request creation failed with error: %s", e)
userlog.error("Pull request creation failed with error: %s", e)
else:
ret = True
else:
update_log.error("Git push command failed.")
userlog.error("Git push command failed.")
else:
update_log.error("Git commit command failed.")
userlog.error("Git commit command failed.")
else:
update_log.error("Git clone %s failed", update_repo)
userlog.error("Git clone %s failed", update_repo)

os.chdir(cwd)
return ret
Expand All @@ -413,7 +426,7 @@ def create_work_directory(path):

"""
if os.path.exists(path):
update_log.info("'%s' directory already exists. Deleting...", path)
userlog.info("'%s' directory already exists. Deleting...", path)
rmtree_readonly(path)

os.makedirs(path)
Expand All @@ -424,28 +437,17 @@ def create_work_directory(path):
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
parser.add_argument('-l', '--log-level',
help="Level for providing logging output",
default='INFO')

exclusive = parser.add_mutually_exclusive_group(required=True)
exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true')
exclusive.add_argument('-b', '--branch', help="Update a branch", action='store_true')

args = parser.parse_args()

default = getattr(logging, 'INFO')
level = getattr(logging, args.log_level.upper(), default)

# Set logging level
logging.basicConfig(level=level)

update_log = logging.getLogger("Update")

# Load the config file
with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
if not config:
update_log.error("Failed to load config file '%s'", args.config_file)
userlog.error("Failed to load config file '%s'", args.config_file)
sys.exit(1)
json_data = json.load(config)

Expand All @@ -470,11 +472,11 @@ def create_work_directory(path):
exit(1)

# Get the github sha corresponding to the specified mbed-os tag
cmd = ['git', 'rev-list', '-1', tag]
cmd = "git rev-list -1 " + tag
return_code, ref = run_cmd_with_output(cmd)

if return_code:
update_log.error("Could not obtain SHA for tag: %s", tag)
userlog.error("Could not obtain SHA for tag: %s", tag)
sys.exit(1)

# Loop through the examples
Expand All @@ -499,11 +501,11 @@ def create_work_directory(path):
os.chdir('../')

# Finish the script and report the results
update_log.info("Finished updating examples")
userlog.info("Finished updating examples")
if successes:
for success in successes:
update_log.info(" SUCCEEDED: %s", success)
userlog.info(" SUCCEEDED: %s", success)

if failures:
for fail in failures:
update_log.info(" FAILED: %s", fail)
userlog.info(" FAILED: %s", fail)