From 5c7a81b6be66b70d0c5dcef47c0ca8b241e03ae4 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 2 Apr 2018 11:45:08 -0700 Subject: [PATCH 1/7] Add envs to SwaggerToSdk --- swaggertosdk/SwaggerToSdkNewCLI.py | 7 +++++-- swaggertosdk/autorest_tools.py | 5 +++-- swaggertosdk/generate_sdk.py | 1 + tests/test_autorest_tools.py | 5 +++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/swaggertosdk/SwaggerToSdkNewCLI.py b/swaggertosdk/SwaggerToSdkNewCLI.py index 6d4f687..014b5ee 100644 --- a/swaggertosdk/SwaggerToSdkNewCLI.py +++ b/swaggertosdk/SwaggerToSdkNewCLI.py @@ -119,9 +119,12 @@ def write_build_file(sdk_root, local_conf): def execute_after_script(sdk_root, global_conf, local_conf): after_scripts = merge_options(global_conf, local_conf, "after_scripts", keep_list_order=True) or [] + local_envs = dict(os.environ) + local_envs.update(global_conf.get("envs", {})) + for script in after_scripts: _LOGGER.info("Execute after script: %s", script) - execute_simple_command(script, cwd=sdk_root, shell=True) + execute_simple_command(script, cwd=sdk_root, shell=True, env=local_envs) def get_local_path_dir(root, relative_path): @@ -155,7 +158,7 @@ def build_libraries(config, skip_callback, restapi_git_folder, sdk_repo, temp_di global_conf = config["meta"] global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_repo.working_tree_dir) - + global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_repo.working_tree_dir) for project, local_conf in config.get("projects", {}).items(): if skip_callback(project, local_conf): diff --git a/swaggertosdk/autorest_tools.py b/swaggertosdk/autorest_tools.py index ec3c0f8..c51325d 100644 --- a/swaggertosdk/autorest_tools.py +++ b/swaggertosdk/autorest_tools.py @@ -121,14 +121,15 @@ def generate_code(input_file, global_conf, local_conf, output_dir=None, autorest raise ValueError("Autorest call ended with 0, but no files were generated") -def execute_simple_command(cmd_line, cwd=None, shell=False): +def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: process = subprocess.Popen(cmd_line, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True, cwd=cwd, - shell=shell) + shell=shell, + env=env) output_buffer = [] for line in process.stdout: output_buffer.append(line.rstrip()) diff --git a/swaggertosdk/generate_sdk.py b/swaggertosdk/generate_sdk.py index 81228b4..74f3c44 100644 --- a/swaggertosdk/generate_sdk.py +++ b/swaggertosdk/generate_sdk.py @@ -26,6 +26,7 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde global_conf = config["meta"] repotag = get_repo_tag_meta(global_conf) global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_folder) + global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_folder) if restapi_git_folder: restapi_git_folder = Path(restapi_git_folder).expanduser() diff --git a/tests/test_autorest_tools.py b/tests/test_autorest_tools.py index 14e660b..212b3be 100644 --- a/tests/test_autorest_tools.py +++ b/tests/test_autorest_tools.py @@ -23,6 +23,11 @@ def test_execute_simple_command(): output = execute_simple_command(["python", "--version"]) assert "Python" in output + env = dict(os.environ) + env.update({"GOPATH": "something"}) + output = execute_simple_command(["python", "-c", "import os; print(os.environ['GOPATH'])"], env=env) + assert "something" in output + try: execute_simple_command(["python", "--oiuyertuyerituy"]) pytest.fail("This should raise an exception") From 7a488ac40579918338e21b12b7deab45bfa9af9f Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 2 Apr 2018 14:56:45 -0700 Subject: [PATCH 2/7] Add tools to get swaggertosdk conf link --- swaggertosdk/SwaggerToSdkCore.py | 4 ++++ tests/test_swaggertosdk.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/swaggertosdk/SwaggerToSdkCore.py b/swaggertosdk/SwaggerToSdkCore.py index eee3af1..eddf96a 100644 --- a/swaggertosdk/SwaggerToSdkCore.py +++ b/swaggertosdk/SwaggerToSdkCore.py @@ -204,3 +204,7 @@ def solve_relative_path(autorest_options, sdk_root): else: solved_autorest_options[key] = value return solved_autorest_options + +def get_configuration_github_path(sdk_id, branch="master"): + gh_token = os.environ.get("GH_TOKEN", None) # Token here is just for private + return GithubLink(sdk_id, "raw", branch, "swagger_to_sdk_config.json", gh_token) diff --git a/tests/test_swaggertosdk.py b/tests/test_swaggertosdk.py index 32959b3..972b1f0 100644 --- a/tests/test_swaggertosdk.py +++ b/tests/test_swaggertosdk.py @@ -10,6 +10,7 @@ extract_conf_from_readmes, get_context_tag_from_git_object, get_readme_files_from_git_object, + get_configuration_github_path ) from swaggertosdk.SwaggerToSdkNewCLI import ( solve_relative_path, @@ -281,3 +282,8 @@ def side_effect(*args, **kwargs): extract_conf_from_readmes(swagger_files_in_pr, Path(CWD, "files"), sdk_git_id, config) assert len(config["projects"]) == 2 + +def test_get_configuration_github_path(github_token): + raw_link = str(get_configuration_github_path("Azure/azure-sdk-for-python", "dev")) + raw_link = raw_link.replace(github_token, "TOKEN") + assert raw_link == "https://TOKEN@raw.githubusercontent.com/Azure/azure-sdk-for-python/dev/swagger_to_sdk_config.json" From d267dcb3bed57a93d69bf5e9aed9928a6ec16428 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 2 Apr 2018 15:39:44 -0700 Subject: [PATCH 3/7] Download the conf --- swaggertosdk/SwaggerToSdkCore.py | 8 +++++++- tests/test_swaggertosdk.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/swaggertosdk/SwaggerToSdkCore.py b/swaggertosdk/SwaggerToSdkCore.py index eddf96a..0c4cf7b 100644 --- a/swaggertosdk/SwaggerToSdkCore.py +++ b/swaggertosdk/SwaggerToSdkCore.py @@ -7,6 +7,8 @@ import tempfile from pathlib import Path +import requests + from github import Github, UnknownObjectException from .autorest_tools import ( @@ -117,6 +119,10 @@ def read_config(sdk_git_folder, config_file): with open(config_path, 'r') as config_fd: return json.loads(config_fd.read()) +def read_config_from_github(sdk_id, branch="master"): + raw_link = str(get_configuration_github_path(sdk_id, branch)) + content = requests.get(raw_link).text + return json.loads(content) def extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config): readme_files_in_pr = {readme for readme in swagger_files_in_pr if getattr(readme, "name", readme).lower().endswith("readme.md")} @@ -207,4 +213,4 @@ def solve_relative_path(autorest_options, sdk_root): def get_configuration_github_path(sdk_id, branch="master"): gh_token = os.environ.get("GH_TOKEN", None) # Token here is just for private - return GithubLink(sdk_id, "raw", branch, "swagger_to_sdk_config.json", gh_token) + return GithubLink(sdk_id, "raw", branch, CONFIG_FILE, gh_token) diff --git a/tests/test_swaggertosdk.py b/tests/test_swaggertosdk.py index 972b1f0..d74203c 100644 --- a/tests/test_swaggertosdk.py +++ b/tests/test_swaggertosdk.py @@ -10,7 +10,8 @@ extract_conf_from_readmes, get_context_tag_from_git_object, get_readme_files_from_git_object, - get_configuration_github_path + get_configuration_github_path, + read_config_from_github ) from swaggertosdk.SwaggerToSdkNewCLI import ( solve_relative_path, @@ -287,3 +288,8 @@ def test_get_configuration_github_path(github_token): raw_link = str(get_configuration_github_path("Azure/azure-sdk-for-python", "dev")) raw_link = raw_link.replace(github_token, "TOKEN") assert raw_link == "https://TOKEN@raw.githubusercontent.com/Azure/azure-sdk-for-python/dev/swagger_to_sdk_config.json" + +def test_read_config_from_github(github_token): + conf = read_config_from_github("Azure/azure-sdk-for-python") + # Don't do too much + assert "meta" in conf From 48803bc435222700616019490fd718436b622b7d Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 2 Apr 2018 16:07:58 -0700 Subject: [PATCH 4/7] Download conf from github before the clone --- swaggertosdk/SwaggerToSdkMain.py | 6 +++--- swaggertosdk/SwaggerToSdkNewCLI.py | 15 +++++++++++++-- swaggertosdk/restapi/sdkbot.py | 5 +++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/swaggertosdk/SwaggerToSdkMain.py b/swaggertosdk/SwaggerToSdkMain.py index 9253aab..f489040 100644 --- a/swaggertosdk/SwaggerToSdkMain.py +++ b/swaggertosdk/SwaggerToSdkMain.py @@ -9,7 +9,7 @@ from .SwaggerToSdkCore import ( CONFIG_FILE, - read_config, + read_config_from_github, extract_conf_from_readmes, get_input_paths, get_readme_files_from_file_list @@ -32,14 +32,14 @@ def generate_sdk(config_path, # On Travis, local folder is restapi git folder restapi_git_folder = '.' + config = read_config_from_github(sdk_git_id, base_branch_name) + # No token is provided to clone SDK. Do NOT try to clone a private it will fail. with tempfile.TemporaryDirectory() as temp_dir, \ manage_git_folder(None, Path(temp_dir) / Path("sdk"), sdk_git_id+'@'+base_branch_name) as sdk_folder: sdk_repo = Repo(str(sdk_folder)) - config = read_config(sdk_repo.working_tree_dir, config_path) - global_conf = config["meta"] swagger_files_in_pr = get_files_in_commit(restapi_git_folder) diff --git a/swaggertosdk/SwaggerToSdkNewCLI.py b/swaggertosdk/SwaggerToSdkNewCLI.py index 014b5ee..4ccb143 100644 --- a/swaggertosdk/SwaggerToSdkNewCLI.py +++ b/swaggertosdk/SwaggerToSdkNewCLI.py @@ -10,7 +10,7 @@ from .SwaggerToSdkCore import ( CONFIG_FILE, - read_config, + read_config_from_github, DEFAULT_COMMIT_MESSAGE, get_input_paths, extract_conf_from_readmes, @@ -224,6 +224,18 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi # Always clone SDK from fallback branch that is required to exist branched_sdk_git_id = sdk_git_id+'@'+fallback_base_branch_name + # I don't know if the destination branch exists, try until it works + config = None + for branch in base_branch_names + branch_name + fallback_base_branch_name: + try: + config = read_config_from_github(sdk_git_id, branch) + except Exception: + pass + else: + break + if config is None: + raise ValueError("Unable to locate configuration in {}".format(base_branch_names + branch_name + fallback_base_branch_name)) + with tempfile.TemporaryDirectory() as temp_dir, \ manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \ manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_git_id) as sdk_folder: @@ -251,7 +263,6 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi configure_user(gh_token, sdk_repo) - config = read_config(sdk_repo.working_tree_dir, config_path) global_conf = config["meta"] # Look for configuration in Readme diff --git a/swaggertosdk/restapi/sdkbot.py b/swaggertosdk/restapi/sdkbot.py index 9efe89f..dd82f83 100644 --- a/swaggertosdk/restapi/sdkbot.py +++ b/swaggertosdk/restapi/sdkbot.py @@ -7,7 +7,7 @@ from swaggertosdk.SwaggerToSdkCore import ( CONFIG_FILE, - read_config, + read_config_from_github, build_swaggertosdk_conf_from_json_readme, ) from swaggertosdk.git_tools import ( @@ -122,6 +122,8 @@ def rebuild(self, issue, project_pattern): path = None # Not such notion of path here, since it's inside SwaggerToSdk conf branched_rest_api_id = rest_api_id + "@" + rest_api_branch + config = read_config_from_github(pr.head.repo.full_name, branch_name) + with tempfile.TemporaryDirectory() as temp_dir, \ manage_git_folder(token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \ manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder: @@ -129,7 +131,6 @@ def rebuild(self, issue, project_pattern): sdk_repo = Repo(str(sdk_folder)) configure_user(self.gh_token, sdk_repo) - config = read_config(sdk_repo.working_tree_dir, config_path) if path: # Assume this is a Readme path config["projects"] = {} # Wipe out everything build_swaggertosdk_conf_from_json_readme(path, sdkid, config, base_folder=restapi_git_folder) From 941401fcda52f2d4dfb0a4f44f1006509a19f311 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 2 Apr 2018 16:11:16 -0700 Subject: [PATCH 5/7] Solve relative path in advanced_conf --- swaggertosdk/SwaggerToSdkNewCLI.py | 1 + swaggertosdk/generate_sdk.py | 1 + 2 files changed, 2 insertions(+) diff --git a/swaggertosdk/SwaggerToSdkNewCLI.py b/swaggertosdk/SwaggerToSdkNewCLI.py index 4ccb143..ab8d800 100644 --- a/swaggertosdk/SwaggerToSdkNewCLI.py +++ b/swaggertosdk/SwaggerToSdkNewCLI.py @@ -159,6 +159,7 @@ def build_libraries(config, skip_callback, restapi_git_folder, sdk_repo, temp_di global_conf = config["meta"] global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_repo.working_tree_dir) global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_repo.working_tree_dir) + global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_repo.working_tree_dir) for project, local_conf in config.get("projects", {}).items(): if skip_callback(project, local_conf): diff --git a/swaggertosdk/generate_sdk.py b/swaggertosdk/generate_sdk.py index 74f3c44..e7ec858 100644 --- a/swaggertosdk/generate_sdk.py +++ b/swaggertosdk/generate_sdk.py @@ -27,6 +27,7 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde repotag = get_repo_tag_meta(global_conf) global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_folder) global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_folder) + global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_folder) if restapi_git_folder: restapi_git_folder = Path(restapi_git_folder).expanduser() From e6a7c85f68922b56d01c236d50405f77ce09ce63 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 2 Apr 2018 16:20:57 -0700 Subject: [PATCH 6/7] Implement clone_dir --- swaggertosdk/SwaggerToSdkMain.py | 63 ++++++++-------- swaggertosdk/SwaggerToSdkNewCLI.py | 114 +++++++++++++++-------------- 2 files changed, 89 insertions(+), 88 deletions(-) diff --git a/swaggertosdk/SwaggerToSdkMain.py b/swaggertosdk/SwaggerToSdkMain.py index f489040..061a0d4 100644 --- a/swaggertosdk/SwaggerToSdkMain.py +++ b/swaggertosdk/SwaggerToSdkMain.py @@ -8,11 +8,11 @@ from git import Repo from .SwaggerToSdkCore import ( - CONFIG_FILE, read_config_from_github, extract_conf_from_readmes, get_input_paths, - get_readme_files_from_file_list + get_readme_files_from_file_list, + solve_relative_path ) from .github_tools import ( manage_git_folder, @@ -24,8 +24,7 @@ _LOGGER = logging.getLogger(__name__) -def generate_sdk(config_path, - sdk_git_id, base_branch_name, +def generate_sdk(sdk_git_id, base_branch_name, autorest_bin=None): """Main method of the the file""" @@ -33,39 +32,43 @@ def generate_sdk(config_path, restapi_git_folder = '.' config = read_config_from_github(sdk_git_id, base_branch_name) + global_conf = config["meta"] # No token is provided to clone SDK. Do NOT try to clone a private it will fail. - with tempfile.TemporaryDirectory() as temp_dir, \ - manage_git_folder(None, Path(temp_dir) / Path("sdk"), sdk_git_id+'@'+base_branch_name) as sdk_folder: + with tempfile.TemporaryDirectory() as temp_dir: - sdk_repo = Repo(str(sdk_folder)) + # Solve advanced_options now + global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), temp_dir) + clone_dir = Path(global_conf["advanced_options"].get("clone_dir", Path(temp_dir) / Path("sdk"))) - global_conf = config["meta"] + with manage_git_folder(None, clone_dir, sdk_git_id+'@'+base_branch_name) as sdk_folder: - swagger_files_in_pr = get_files_in_commit(restapi_git_folder) - _LOGGER.info("Files in PR: %s ", swagger_files_in_pr) - swagger_files_in_pr = get_readme_files_from_file_list(swagger_files_in_pr, restapi_git_folder) - _LOGGER.info("Readmes in PR: %s ", swagger_files_in_pr) + sdk_repo = Repo(str(sdk_folder)) - # Look for configuration in Readme - extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config) + swagger_files_in_pr = get_files_in_commit(restapi_git_folder) + _LOGGER.info("Files in PR: %s ", swagger_files_in_pr) + swagger_files_in_pr = get_readme_files_from_file_list(swagger_files_in_pr, restapi_git_folder) + _LOGGER.info("Readmes in PR: %s ", swagger_files_in_pr) - def skip_callback(project, local_conf): - if not swagger_files_in_pr: - return True # Travis with no files found, always skip + # Look for configuration in Readme + extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config) - markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf) + def skip_callback(project, local_conf): + if not swagger_files_in_pr: + return True # Travis with no files found, always skip - if swagger_files_in_pr and not ( - markdown_relative_path in swagger_files_in_pr or - any(input_file in swagger_files_in_pr for input_file in optional_relative_paths)): - _LOGGER.info(f"In project {project} no files involved in this PR") - return True - return False + markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf) - from . import SwaggerToSdkNewCLI - SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder, - sdk_repo, temp_dir, autorest_bin) + if swagger_files_in_pr and not ( + markdown_relative_path in swagger_files_in_pr or + any(input_file in swagger_files_in_pr for input_file in optional_relative_paths)): + _LOGGER.info(f"In project {project} no files involved in this PR") + return True + return False + + from . import SwaggerToSdkNewCLI + SwaggerToSdkNewCLI.build_libraries(config, skip_callback, restapi_git_folder, + sdk_repo, temp_dir, autorest_bin) _LOGGER.info("Build SDK finished and cleaned") @@ -94,9 +97,6 @@ def main(argv): parser.add_argument('--base-branch', '-o', dest='base_branch', default='master', help='The base branch to checkout. [default: %(default)s]') - parser.add_argument('--config', '-c', - dest='config_path', default=CONFIG_FILE, - help='The JSON configuration format path [default: %(default)s]') parser.add_argument('--autorest', dest='autorest_bin', help='Force the Autorest to be executed. Must be a executable command.') @@ -117,7 +117,6 @@ def main(argv): logging.basicConfig() main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO) - generate_sdk(args.config_path, - args.sdk_git_id, + generate_sdk(args.sdk_git_id, args.base_branch, args.autorest_bin) diff --git a/swaggertosdk/SwaggerToSdkNewCLI.py b/swaggertosdk/SwaggerToSdkNewCLI.py index ab8d800..0fb81af 100644 --- a/swaggertosdk/SwaggerToSdkNewCLI.py +++ b/swaggertosdk/SwaggerToSdkNewCLI.py @@ -9,7 +9,6 @@ from git import Repo, GitCommandError from .SwaggerToSdkCore import ( - CONFIG_FILE, read_config_from_github, DEFAULT_COMMIT_MESSAGE, get_input_paths, @@ -209,7 +208,6 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi This method might push to "branch_name" and "base_branch_name". No push will be made to "fallback_base_branch_name" """ gh_token = os.environ["GH_TOKEN"] - config_path = CONFIG_FILE message_template = DEFAULT_COMMIT_MESSAGE autorest_bin = None if sdk_tag is None: @@ -236,65 +234,69 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi break if config is None: raise ValueError("Unable to locate configuration in {}".format(base_branch_names + branch_name + fallback_base_branch_name)) + global_conf = config["meta"] - with tempfile.TemporaryDirectory() as temp_dir, \ - manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \ - manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_git_id) as sdk_folder: - - swagger_files_in_commit = get_readme_files_from_git_object(git_object, restapi_git_folder) - _LOGGER.info("Files in PR: %s ", swagger_files_in_commit) - if not swagger_files_in_commit: - _LOGGER.info("No Readme in PR, quit") - return - - # SDK part - sdk_repo = Repo(str(sdk_folder)) - - for base_branch in base_branch_names: - _LOGGER.info('Checkout and create %s', base_branch) - checkout_and_create_branch(sdk_repo, base_branch) + with tempfile.TemporaryDirectory() as temp_dir: - _LOGGER.info('Try to checkout destination branch %s', branch_name) - try: - sdk_repo.git.checkout(branch_name) - _LOGGER.info('The branch exists.') - except GitCommandError: - _LOGGER.info('Destination branch does not exists') - # Will be created by do_commit + # Solve advanced_options now + global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), temp_dir) + clone_dir = Path(global_conf["advanced_options"].get("clone_dir", Path(temp_dir) / Path("sdk"))) - configure_user(gh_token, sdk_repo) + with manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \ + manage_git_folder(gh_token, clone_dir, branched_sdk_git_id) as sdk_folder: - global_conf = config["meta"] + swagger_files_in_commit = get_readme_files_from_git_object(git_object, restapi_git_folder) + _LOGGER.info("Files in PR: %s ", swagger_files_in_commit) + if not swagger_files_in_commit: + _LOGGER.info("No Readme in PR, quit") + return - # Look for configuration in Readme - _LOGGER.info('Extract conf from Readmes for target: %s', sdk_git_id) - extract_conf_from_readmes(swagger_files_in_commit, restapi_git_folder, sdk_tag, config) - _LOGGER.info('End of extraction') + # SDK part + sdk_repo = Repo(str(sdk_folder)) - def skip_callback(project, local_conf): - # We know "project" is based on Path in "swagger_files_in_commit" - if Path(project) in swagger_files_in_commit: + for base_branch in base_branch_names: + _LOGGER.info('Checkout and create %s', base_branch) + checkout_and_create_branch(sdk_repo, base_branch) + + _LOGGER.info('Try to checkout destination branch %s', branch_name) + try: + sdk_repo.git.checkout(branch_name) + _LOGGER.info('The branch exists.') + except GitCommandError: + _LOGGER.info('Destination branch does not exists') + # Will be created by do_commit + + configure_user(gh_token, sdk_repo) + + # Look for configuration in Readme + _LOGGER.info('Extract conf from Readmes for target: %s', sdk_git_id) + extract_conf_from_readmes(swagger_files_in_commit, restapi_git_folder, sdk_tag, config) + _LOGGER.info('End of extraction') + + def skip_callback(project, local_conf): + # We know "project" is based on Path in "swagger_files_in_commit" + if Path(project) in swagger_files_in_commit: + return False + # Might be a regular project + markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf) + if not ( + markdown_relative_path in swagger_files_in_commit or + any(input_file in swagger_files_in_commit for input_file in optional_relative_paths)): + _LOGGER.info(f"In project {project} no files involved in this commit") + return True return False - # Might be a regular project - markdown_relative_path, optional_relative_paths = get_input_paths(global_conf, local_conf) - if not ( - markdown_relative_path in swagger_files_in_commit or - any(input_file in swagger_files_in_commit for input_file in optional_relative_paths)): - _LOGGER.info(f"In project {project} no files involved in this commit") - return True - return False - - build_libraries(config, skip_callback, restapi_git_folder, - sdk_repo, temp_dir, autorest_bin) - try: - commit_for_sha = git_object.commit # Commit - except AttributeError: - commit_for_sha = list(git_object.get_commits())[-1].commit # PR - message = message_template + "\n\n" + commit_for_sha.message - commit_sha = do_commit(sdk_repo, message, branch_name, commit_for_sha.sha) - if commit_sha: - for base_branch in base_branch_names: - sdk_repo.git.push('origin', base_branch, set_upstream=True) - sdk_repo.git.push('origin', branch_name, set_upstream=True) - return "https://github.com/{}/commit/{}".format(sdk_git_id, commit_sha) + build_libraries(config, skip_callback, restapi_git_folder, + sdk_repo, temp_dir, autorest_bin) + + try: + commit_for_sha = git_object.commit # Commit + except AttributeError: + commit_for_sha = list(git_object.get_commits())[-1].commit # PR + message = message_template + "\n\n" + commit_for_sha.message + commit_sha = do_commit(sdk_repo, message, branch_name, commit_for_sha.sha) + if commit_sha: + for base_branch in base_branch_names: + sdk_repo.git.push('origin', base_branch, set_upstream=True) + sdk_repo.git.push('origin', branch_name, set_upstream=True) + return "https://github.com/{}/commit/{}".format(sdk_git_id, commit_sha) From fda4ff371be77de5a44c886b59467e103d2a436e Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Mon, 2 Apr 2018 16:42:58 -0700 Subject: [PATCH 7/7] Add log --- swaggertosdk/SwaggerToSdkMain.py | 1 + swaggertosdk/SwaggerToSdkNewCLI.py | 1 + 2 files changed, 2 insertions(+) diff --git a/swaggertosdk/SwaggerToSdkMain.py b/swaggertosdk/SwaggerToSdkMain.py index 061a0d4..1638af6 100644 --- a/swaggertosdk/SwaggerToSdkMain.py +++ b/swaggertosdk/SwaggerToSdkMain.py @@ -40,6 +40,7 @@ def generate_sdk(sdk_git_id, base_branch_name, # Solve advanced_options now global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), temp_dir) clone_dir = Path(global_conf["advanced_options"].get("clone_dir", Path(temp_dir) / Path("sdk"))) + _LOGGER.info("Clone dir will be: %s", clone_dir) with manage_git_folder(None, clone_dir, sdk_git_id+'@'+base_branch_name) as sdk_folder: diff --git a/swaggertosdk/SwaggerToSdkNewCLI.py b/swaggertosdk/SwaggerToSdkNewCLI.py index 0fb81af..4f86340 100644 --- a/swaggertosdk/SwaggerToSdkNewCLI.py +++ b/swaggertosdk/SwaggerToSdkNewCLI.py @@ -241,6 +241,7 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi # Solve advanced_options now global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), temp_dir) clone_dir = Path(global_conf["advanced_options"].get("clone_dir", Path(temp_dir) / Path("sdk"))) + _LOGGER.info("Clone dir will be: %s", clone_dir) with manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \ manage_git_folder(gh_token, clone_dir, branched_sdk_git_id) as sdk_folder: