Merge 941401f into bc35f09

swaggertosdk/SwaggerToSdkCore.py

@@ -7,6 +7,8 @@
 import tempfile
 from pathlib import Path
 
+import requests
+
 from github import Github, UnknownObjectException
 
 from .autorest_tools import (
@@ -117,6 +119,10 @@ def read_config(sdk_git_folder, config_file):
     with open(config_path, 'r') as config_fd:
         return json.loads(config_fd.read())
 
+def read_config_from_github(sdk_id, branch="master"):
+    raw_link = str(get_configuration_github_path(sdk_id, branch))
+    content = requests.get(raw_link).text
+    return json.loads(content)
 
 def extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config):
     readme_files_in_pr = {readme for readme in swagger_files_in_pr if getattr(readme, "name", readme).lower().endswith("readme.md")}
@@ -204,3 +210,7 @@ def solve_relative_path(autorest_options, sdk_root):
         else:
             solved_autorest_options[key] = value
     return solved_autorest_options
+
+def get_configuration_github_path(sdk_id, branch="master"):
+    gh_token = os.environ.get("GH_TOKEN", None)  # Token here is just for private
+    return GithubLink(sdk_id, "raw", branch, CONFIG_FILE, gh_token)

swaggertosdk/SwaggerToSdkMain.py

@@ -9,7 +9,7 @@
 
 from .SwaggerToSdkCore import (
     CONFIG_FILE,
-    read_config,
+    read_config_from_github,
     extract_conf_from_readmes,
     get_input_paths,
     get_readme_files_from_file_list
@@ -32,14 +32,14 @@ def generate_sdk(config_path,
     # On Travis, local folder is restapi git folder
     restapi_git_folder = '.'
 
+    config = read_config_from_github(sdk_git_id, base_branch_name)
+
     # No token is provided to clone SDK. Do NOT try to clone a private it will fail.
     with tempfile.TemporaryDirectory() as temp_dir, \
             manage_git_folder(None, Path(temp_dir) / Path("sdk"), sdk_git_id+'@'+base_branch_name) as sdk_folder:
 
         sdk_repo = Repo(str(sdk_folder))
 
-        config = read_config(sdk_repo.working_tree_dir, config_path)
-
         global_conf = config["meta"]
 
         swagger_files_in_pr = get_files_in_commit(restapi_git_folder)

swaggertosdk/SwaggerToSdkNewCLI.py

@@ -10,7 +10,7 @@
 
 from .SwaggerToSdkCore import (
     CONFIG_FILE,
-    read_config,
+    read_config_from_github,
     DEFAULT_COMMIT_MESSAGE,
     get_input_paths,
     extract_conf_from_readmes,
@@ -119,9 +119,12 @@ def write_build_file(sdk_root, local_conf):
 
 def execute_after_script(sdk_root, global_conf, local_conf):
     after_scripts = merge_options(global_conf, local_conf, "after_scripts", keep_list_order=True) or []
+    local_envs = dict(os.environ)
+    local_envs.update(global_conf.get("envs", {}))
+
     for script in after_scripts:
         _LOGGER.info("Execute after script: %s", script)
-        execute_simple_command(script, cwd=sdk_root, shell=True)
+        execute_simple_command(script, cwd=sdk_root, shell=True, env=local_envs)
 
 
 def get_local_path_dir(root, relative_path):
@@ -155,7 +158,8 @@ def build_libraries(config, skip_callback, restapi_git_folder, sdk_repo, temp_di
 
     global_conf = config["meta"]
     global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_repo.working_tree_dir)
-
+    global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_repo.working_tree_dir)
+    global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_repo.working_tree_dir)
 
     for project, local_conf in config.get("projects", {}).items():
         if skip_callback(project, local_conf):
@@ -221,6 +225,18 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi
     # Always clone SDK from fallback branch that is required to exist
     branched_sdk_git_id = sdk_git_id+'@'+fallback_base_branch_name
 
+    # I don't know if the destination branch exists, try until it works
+    config = None
+    for branch in base_branch_names + branch_name + fallback_base_branch_name:
+        try:
+            config = read_config_from_github(sdk_git_id, branch)
+        except Exception:
+            pass
+        else:
+            break
+    if config is None:
+        raise ValueError("Unable to locate configuration in {}".format(base_branch_names + branch_name + fallback_base_branch_name))
+
     with tempfile.TemporaryDirectory() as temp_dir, \
             manage_git_folder(gh_token, Path(temp_dir) / Path("rest"), branched_rest_api_id, pr_number=pr_number) as restapi_git_folder, \
             manage_git_folder(gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_git_id) as sdk_folder:
@@ -248,7 +264,6 @@ def generate_sdk_from_git_object(git_object, branch_name, restapi_git_id, sdk_gi
 
         configure_user(gh_token, sdk_repo)
 
-        config = read_config(sdk_repo.working_tree_dir, config_path)
         global_conf = config["meta"]
 
         # Look for configuration in Readme

swaggertosdk/autorest_tools.py

@@ -121,14 +121,15 @@ def generate_code(input_file, global_conf, local_conf, output_dir=None, autorest
         raise ValueError("Autorest call ended with 0, but no files were generated")
 
 
-def execute_simple_command(cmd_line, cwd=None, shell=False):
+def execute_simple_command(cmd_line, cwd=None, shell=False, env=None):
     try:
         process = subprocess.Popen(cmd_line,
                                    stderr=subprocess.STDOUT,
                                    stdout=subprocess.PIPE,
                                    universal_newlines=True,
                                    cwd=cwd,
-                                   shell=shell)
+                                   shell=shell,
+                                   env=env)
         output_buffer = []
         for line in process.stdout:
             output_buffer.append(line.rstrip())

swaggertosdk/generate_sdk.py

@@ -26,6 +26,8 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde
     global_conf = config["meta"]
     repotag = get_repo_tag_meta(global_conf)
     global_conf["autorest_options"] = solve_relative_path(global_conf.get("autorest_options", {}), sdk_folder)
+    global_conf["envs"] = solve_relative_path(global_conf.get("envs", {}), sdk_folder)
+    global_conf["advanced_options"] = solve_relative_path(global_conf.get("advanced_options", {}), sdk_folder)
     if restapi_git_folder:
         restapi_git_folder = Path(restapi_git_folder).expanduser()
 

swaggertosdk/restapi/sdkbot.py

@@ -7,7 +7,7 @@
 
 from swaggertosdk.SwaggerToSdkCore import (
     CONFIG_FILE,
-    read_config,
+    read_config_from_github,
     build_swaggertosdk_conf_from_json_readme,
 )
 from swaggertosdk.git_tools import (
@@ -122,14 +122,15 @@ def rebuild(self, issue, project_pattern):
             path = None  # Not such notion of path here, since it's inside SwaggerToSdk conf
         branched_rest_api_id = rest_api_id + "@" + rest_api_branch
 
+        config = read_config_from_github(pr.head.repo.full_name, branch_name)
+
         with tempfile.TemporaryDirectory() as temp_dir, \
                 manage_git_folder(token, Path(temp_dir) / Path("rest"), branched_rest_api_id) as restapi_git_folder, \
                 manage_git_folder(self.gh_token, Path(temp_dir) / Path("sdk"), branched_sdk_id) as sdk_folder:
 
             sdk_repo = Repo(str(sdk_folder))
             configure_user(self.gh_token, sdk_repo)
 
-            config = read_config(sdk_repo.working_tree_dir, config_path)
             if path: # Assume this is a Readme path
                 config["projects"] = {} # Wipe out everything
                 build_swaggertosdk_conf_from_json_readme(path, sdkid, config, base_folder=restapi_git_folder)

tests/test_autorest_tools.py

@@ -23,6 +23,11 @@ def test_execute_simple_command():
     output = execute_simple_command(["python", "--version"])
     assert "Python" in output
 
+    env = dict(os.environ)
+    env.update({"GOPATH": "something"})
+    output = execute_simple_command(["python", "-c", "import os; print(os.environ['GOPATH'])"], env=env)
+    assert "something" in output
+
     try:
         execute_simple_command(["python", "--oiuyertuyerituy"])
         pytest.fail("This should raise an exception")

tests/test_swaggertosdk.py

@@ -10,6 +10,8 @@
     extract_conf_from_readmes,
     get_context_tag_from_git_object,
     get_readme_files_from_git_object,
+    get_configuration_github_path,
+    read_config_from_github
 )
 from swaggertosdk.SwaggerToSdkNewCLI import (
     solve_relative_path,
@@ -281,3 +283,13 @@ def side_effect(*args, **kwargs):
     extract_conf_from_readmes(swagger_files_in_pr, Path(CWD, "files"), sdk_git_id, config)
 
     assert len(config["projects"]) == 2
+
+def test_get_configuration_github_path(github_token):
+    raw_link = str(get_configuration_github_path("Azure/azure-sdk-for-python", "dev"))
+    raw_link = raw_link.replace(github_token, "TOKEN")
+    assert raw_link == "https://TOKEN@raw.githubusercontent.com/Azure/azure-sdk-for-python/dev/swagger_to_sdk_config.json"
+
+def test_read_config_from_github(github_token):
+    conf = read_config_from_github("Azure/azure-sdk-for-python")
+    # Don't do too much
+    assert "meta" in conf