Skip to content

Commit

Permalink
Merge pull request #25673 from gottesmm/pr-346412d7746bf95e1c890610c9…
Browse files Browse the repository at this point in the history
…5d7401bc4490c3

[update-checkout] Make SWIFT_SOURCE_ROOT customizable and add a basic clone test
  • Loading branch information
gottesmm committed Jun 22, 2019
2 parents 5c5de29 + a9d384d commit 17d65dc
Show file tree
Hide file tree
Showing 5 changed files with 227 additions and 29 deletions.
20 changes: 20 additions & 0 deletions utils/update_checkout/test_update_checkout.sh
@@ -0,0 +1,20 @@
#!/bin/bash

set -e

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# If the user did not specify a workspace dir, create one in /tmp/workspace and
# export it for the tests.
if [[ -n "${1}" ]]; then
export UPDATECHECKOUT_TEST_WORKSPACE_DIR="${1}"
echo "Using ${UPDATECHECKOUT_TEST_WORKSPACE_DIR}"
else
export UPDATECHECKOUT_TEST_WORKSPACE_DIR=/tmp/workspace
echo "No preset workspace dir! Using ${UPDATECHECKOUT_TEST_WORKSPACE_DIR}"
fi

python -m unittest discover -s $DIR

set +e

153 changes: 153 additions & 0 deletions utils/update_checkout/tests/scheme_mock.py
@@ -0,0 +1,153 @@
# ===--- SchemeMock.py ----------------------------------------------------===#
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https:#swift.org/LICENSE.txt for license information
# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ===----------------------------------------------------------------------===#
"""This file defines objects for mocking an update-checkout scheme. It creates
a json .config file and a series of .git repos with "fake commits".
"""

import json
import os
import subprocess
import unittest

# For now we only use a config with a single scheme. We should add support for
# handling multiple schemes.
MOCK_REMOTE = {
'repo1': [
# This is a series of changes to repo1. (File, NewContents)
('A.txt', 'A'),
('B.txt', 'B'),
('A.txt', 'a'),
],
'repo2': [
# This is a series of changes to repo1. (File, NewContents)
('X.txt', 'X'),
('Y.txt', 'Y'),
('X.txt', 'z'),
],
}

MOCK_CONFIG = {
# We reset this value with our remote path when we process
'https-clone-pattern': '',
'repos': {
'repo1': {
'remote': {'id': 'repo1'},
},
'repo2': {
'remote': {'id': 'repo2'},
},
},
'default-branch-scheme': 'master',
'branch-schemes': {
'master': {
'aliases': ['master'],
'repos': {
'repo1': 'master',
'repo2': 'master',
}
}
}
}


def call_quietly(*args, **kwargs):
with open(os.devnull, 'w') as f:
kwargs['stdout'] = f
kwargs['stderr'] = f
subprocess.check_call(*args, **kwargs)


def create_dir(d):
if not os.path.isdir(d):
os.makedirs(d)


def teardown_mock_remote(base_dir):
call_quietly(['rm', '-rf', base_dir])


def get_config_path(base_dir):
return os.path.join(base_dir, 'test-config.json')


def setup_mock_remote(base_dir):
create_dir(base_dir)

# We use local as a workspace for creating commits.
LOCAL_PATH = os.path.join(base_dir, 'local')
# We use remote as a directory that simulates our remote unchecked out
# repo.
REMOTE_PATH = os.path.join(base_dir, 'remote')

create_dir(REMOTE_PATH)
create_dir(LOCAL_PATH)

for (k, v) in MOCK_REMOTE.items():
local_repo_path = os.path.join(LOCAL_PATH, k)
remote_repo_path = os.path.join(REMOTE_PATH, k)
create_dir(remote_repo_path)
create_dir(local_repo_path)
call_quietly(['git', 'init', '--bare', remote_repo_path])
call_quietly(['git', 'clone', '-l', remote_repo_path, local_repo_path])
for (i, (filename, contents)) in enumerate(v):
filename_path = os.path.join(local_repo_path, filename)
with open(filename_path, 'w') as f:
f.write(contents)
call_quietly(['git', 'add', filename], cwd=local_repo_path)
call_quietly(['git', 'commit', '-m', 'Commit %d' % i],
cwd=local_repo_path)
call_quietly(['git', 'push', 'origin', 'master'],
cwd=local_repo_path)

base_config = MOCK_CONFIG
https_clone_pattern = os.path.join('file://%s' % REMOTE_PATH, '%s')
base_config['https-clone-pattern'] = https_clone_pattern

with open(get_config_path(base_dir), 'w') as f:
json.dump(base_config, f)


BASEDIR_ENV_VAR = 'UPDATECHECKOUT_TEST_WORKSPACE_DIR'
CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__))
UPDATE_CHECKOUT_PATH = os.path.abspath(os.path.join(CURRENT_FILE_DIR,
os.path.pardir,
os.path.pardir,
'update-checkout'))


class SchemeMockTestCase(unittest.TestCase):

def __init__(self, *args, **kwargs):
super(SchemeMockTestCase, self).__init__(*args, **kwargs)

self.workspace = os.getenv(BASEDIR_ENV_VAR)
if self.workspace is None:
raise RuntimeError('Misconfigured test suite! Environment '
'variable %s must be set!' % BASEDIR_ENV_VAR)
self.config_path = get_config_path(self.workspace)
self.update_checkout_path = UPDATE_CHECKOUT_PATH
if not os.access(self.update_checkout_path, os.X_OK):
raise RuntimeError('Error! Could not find executable '
'update-checkout at path: %s'
% self.update_checkout_path)
self.source_root = os.path.join(self.workspace, 'source_root')

def setUp(self):
create_dir(self.source_root)
setup_mock_remote(self.workspace)

def tearDown(self):
teardown_mock_remote(self.workspace)

def call(self, *args, **kwargs):
kwargs['cwd'] = self.source_root
call_quietly(*args, **kwargs)
25 changes: 25 additions & 0 deletions utils/update_checkout/tests/test_clone.py
@@ -0,0 +1,25 @@
# ===--- test_clone.py ----------------------------------------------------===#
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https:#swift.org/LICENSE.txt for license information
# See https:#swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ===----------------------------------------------------------------------===#

import scheme_mock


class CloneTestCase(scheme_mock.SchemeMockTestCase):

def __init__(self, *args, **kwargs):
super(CloneTestCase, self).__init__(*args, **kwargs)

def test_simple_clone(self):
self.call([self.update_checkout_path,
'--config', self.config_path,
'--source-root', self.source_root,
'--clone'])
45 changes: 25 additions & 20 deletions utils/update_checkout/update_checkout/update_checkout.py
Expand Up @@ -81,10 +81,10 @@ def get_branch_for_repo(config, repo_name, scheme_name, scheme_map,
return repo_branch, cross_repo


def update_single_repository(args):
config, repo_name, scheme_name, scheme_map, tag, timestamp, \
reset_to_remote, should_clean, cross_repos_pr = args
repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
def update_single_repository(pool_args):
source_root, config, repo_name, scheme_name, scheme_map, tag, timestamp, \
reset_to_remote, should_clean, cross_repos_pr = pool_args
repo_path = os.path.join(source_root, repo_name)
if not os.path.isdir(repo_path):
return

Expand Down Expand Up @@ -175,7 +175,7 @@ def update_single_repository(args):
def get_timestamp_to_match(args):
if not args.match_timestamp:
return None
with shell.pushd(os.path.join(SWIFT_SOURCE_ROOT, "swift"),
with shell.pushd(os.path.join(args.source_root, "swift"),
dry_run=False, echo=False):
return shell.capture(["git", "log", "-1", "--format=%cI"],
echo=False).strip()
Expand All @@ -198,7 +198,7 @@ def update_all_repositories(args, config, scheme_name, cross_repos_pr):
if repo_name in args.skip_repository_list:
print("Skipping update of '" + repo_name + "', requested by user")
continue
my_args = [config,
my_args = [args.source_root, config,
repo_name,
scheme_name,
scheme_map,
Expand All @@ -220,7 +220,7 @@ def obtain_additional_swift_sources(pool_args):
env = dict(os.environ)
env.update({'GIT_TERMINAL_PROMPT': 0})

with shell.pushd(SWIFT_SOURCE_ROOT, dry_run=False, echo=False):
with shell.pushd(args.source_root, dry_run=False, echo=False):

print("Cloning '" + repo_name + "'")

Expand All @@ -236,14 +236,14 @@ def obtain_additional_swift_sources(pool_args):
env=env,
echo=True)
if scheme_name:
src_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name, ".git")
src_path = os.path.join(args.source_root, repo_name, ".git")
shell.run(['git', '--git-dir',
src_path, '--work-tree',
os.path.join(SWIFT_SOURCE_ROOT, repo_name),
os.path.join(args.source_root, repo_name),
'checkout', repo_branch],
env=env,
echo=False)
with shell.pushd(os.path.join(SWIFT_SOURCE_ROOT, repo_name),
with shell.pushd(os.path.join(args.source_root, repo_name),
dry_run=False, echo=False):
shell.run(["git", "submodule",
"update", "--recursive"],
Expand All @@ -255,7 +255,7 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
skip_history, skip_repository_list):

pool_args = []
with shell.pushd(SWIFT_SOURCE_ROOT, dry_run=False, echo=False):
with shell.pushd(args.source_root, dry_run=False, echo=False):
for repo_name, repo_info in config['repos'].items():
if repo_name in skip_repository_list:
print("Skipping clone of '" + repo_name + "', requested by "
Expand Down Expand Up @@ -308,7 +308,7 @@ def obtain_all_additional_swift_sources(args, config, with_ssh, scheme_name,
args.n_processes)


def dump_repo_hashes(config, branch_scheme_name='repro'):
def dump_repo_hashes(args, config, branch_scheme_name='repro'):
"""
Dumps the current state of the repo into a new config file that contains a
master branch scheme with the relevant branches set to the appropriate
Expand All @@ -319,17 +319,17 @@ def dump_repo_hashes(config, branch_scheme_name='repro'):
for config_copy_key in config_copy_keys:
new_config[config_copy_key] = config[config_copy_key]
repos = {}
repos = repo_hashes(config)
repos = repo_hashes(args, config)
branch_scheme = {'aliases': [branch_scheme_name], 'repos': repos}
new_config['branch-schemes'] = {branch_scheme_name: branch_scheme}
json.dump(new_config, sys.stdout, indent=4)


def repo_hashes(config):
def repo_hashes(args, config):
repos = {}
for repo_name, repo_info in sorted(config['repos'].items(),
key=lambda x: x[0]):
repo_path = os.path.join(SWIFT_SOURCE_ROOT, repo_name)
repo_path = os.path.join(args.source_root, repo_name)
if os.path.exists(repo_path):
with shell.pushd(repo_path, dry_run=False, echo=False):
h = shell.capture(["git", "rev-parse", "HEAD"],
Expand All @@ -340,8 +340,8 @@ def repo_hashes(config):
return repos


def print_repo_hashes(config):
repos = repo_hashes(config)
def print_repo_hashes(args, config):
repos = repo_hashes(args, config)
for repo_name, repo_hash in sorted(repos.items(),
key=lambda x: x[0]):
print("{:<35}: {:<35}".format(repo_name, repo_hash))
Expand Down Expand Up @@ -484,6 +484,11 @@ def main():
help="Number of threads to run at once",
default=0,
dest="n_processes")
parser.add_argument(
"--source-root",
help="The root directory to checkout repositories",
default=SWIFT_SOURCE_ROOT,
dest='source_root')
args = parser.parse_args()

if not args.scheme:
Expand Down Expand Up @@ -513,7 +518,7 @@ def main():
return (None, None)

if args.dump_hashes_config:
dump_repo_hashes(config, args.dump_hashes_config)
dump_repo_hashes(args, config, args.dump_hashes_config)
return (None, None)

cross_repos_pr = {}
Expand All @@ -540,7 +545,7 @@ def main():
skip_repo_list)

# Quick check whether somebody is calling update in an empty directory
directory_contents = os.listdir(SWIFT_SOURCE_ROOT)
directory_contents = os.listdir(args.source_root)
if not ('cmark' in directory_contents or
'llvm' in directory_contents or
'clang' in directory_contents):
Expand All @@ -556,5 +561,5 @@ def main():
print("update-checkout failed, fix errors and try again")
else:
print("update-checkout succeeded")
print_repo_hashes(config)
print_repo_hashes(args, config)
sys.exit(fail_count)
13 changes: 4 additions & 9 deletions validation-test/Python/update_checkout.test-sh
@@ -1,10 +1,5 @@
# RUN: %{python} -m unittest discover -s %swift_src_root/utils/update_checkout
# RUN: %empty-directory(%t)
# RUN: %swift_src_root/utils/update_checkout/test_update_checkout.sh %t

# Do not run this when doing simulator or device testing on darwin.
#
# Otherwise, we are running these unittests multiple times without providing
# value. We want to still run this on linux though.
#
# UNSUPPORTED: OS=ios
# UNSUPPORTED: OS=watchos
# UNSUPPORTED: OS=tvos
# For now, only run this on macosx.
# REQUIRES: OS=macosx

0 comments on commit 17d65dc

Please sign in to comment.