Dashboard + context branch

swaggertosdk/SwaggerToSdkMain.py

@@ -22,6 +22,7 @@
     get_commit_object_from_travis,
     extract_conf_from_readmes,
     get_input_paths,
+    get_context_tag_from_files_list
 )
 from .git_tools import (
     do_commit,
@@ -76,6 +77,9 @@ def generate_sdk(gh_token, config_path, project_pattern, restapi_git_id,
         swagger_files_in_pr = get_swagger_project_files_in_git_object(initial_git_trigger, restapi_git_folder) if initial_git_trigger else set()
         _LOGGER.info("Files in PR: %s ", swagger_files_in_pr)
 
+        context_tags = get_context_tag_from_files_list(swagger_files_in_pr)
+        _LOGGER.info("Context tags: %s", context_tags)
+
         # Look for configuration in Readme
         extract_conf_from_readmes(swagger_files_in_pr, restapi_git_folder, sdk_git_id, config)
 

swaggertosdk/git_tools.py

@@ -2,7 +2,7 @@
 """
 import logging
 
-from git import Repo
+from git import Repo, GitCommandError
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -20,6 +20,18 @@ def checkout_and_create_branch(repo, name):
         local_branch = repo.create_head(name)
     local_branch.checkout()
 
+def checkout_create_push_branch(repo, name):
+    """Checkout this branch. Create it if necessary, and push it to origin.
+    """
+    try:
+        repo.git.checkout(name)
+        _LOGGER.info("Checkout %s success", name)
+    except GitCommandError:
+        _LOGGER.info("Checkout %s was impossible (branch does not exist)", name)
+        checkout_and_create_branch(repo, name)
+        repo.git.push('origin', name, set_upstream=True)
+
+
 def do_commit(repo, message_template, branch_name, hexsha):
     "Do a commit if modified/untracked files"
     repo.git.add(repo.working_tree_dir)

swaggertosdk/github_tools.py

@@ -115,6 +115,35 @@ def sync_fork(gh_token, github_repo_id, repo, push=True):
         msg = repo.git.push()
         _LOGGER.debug(msg)
 
+def get_or_create_pull(github_repo, title, body, head, base):
+    """Try to create the PR. If the PR exists, try to find it instead. Raises otherwise.
+
+    You should always use the complete head syntax "org:branch", since the syntax is required
+    in case of listing.
+    """
+    try: # Try to create or get a PR
+        return github_repo.create_pull(
+            title=title,
+            body=body,
+            head=head,
+            base=base
+        )
+    except GithubException as err:
+        if err.status == 422 and err.data['errors'][0].get('message', '').startswith('A pull request already exists'):
+            _LOGGER.info('PR already exists, get this PR')
+            return list(github_repo.get_pulls(
+                head=head,
+                base=base
+            ))[0]
+        else:
+            _LOGGER.warning("Unable to create PR:\n%s", err.data)
+            raise
+    except Exception as err:
+        response = traceback.format_exc()
+        _LOGGER.warning("Unable to create PR:\n%s", response)
+        raise
+
+
 def clone_to_path(gh_token, folder, sdk_git_id, branch=None):
     """Clone the given repo_id to the folder.
 

swaggertosdk/restapi/github.py

@@ -19,7 +19,14 @@
     GithubHandler as LocalHandler,
     generate_sdk_from_git_object
 )
-from ..github_tools import exception_to_github
+from ..github_tools import (
+    exception_to_github,
+    DashboardCommentableObject,
+    get_or_create_pull,
+)
+from ..SwaggerToSdkCore import (
+    get_context_tag_from_git_object
+)
 from . import app
 
 _LOGGER = logging.getLogger(__name__)
@@ -147,7 +154,7 @@ def push(body):
         sdk_pr_model.head_branch_name,
         restapi_git_id,
         sdkid,
-        None, # I don't know if the origin branch comes from "master", assume it.
+        [], # I don't know if the origin branch comes from "master", assume it.
         fallback_base_branch_name=sdkbase,
         sdk_tag=sdk_tag
     )
@@ -177,69 +184,72 @@ def rest_handle_action(body, sdkid, sdkbase, sdk_tag):
 
     restapi_git_id = body['repository']['full_name']
     restapi_repo = github_con.get_repo(restapi_git_id)
-    rest_pr_as_issue = restapi_repo.get_issue(body["number"])
+    rest_pr = restapi_repo.get_pull(body["number"])
+    dashboard = DashboardCommentableObject(rest_pr, "# Automation for {}".format(sdk_tag))
+
+    context_tags = list(get_context_tag_from_git_object(rest_pr))
+    if len(context_tags) == 0:
+        dashboard.create_comment("Unable to detect any generation context from this PR.")
+        return
 
     _LOGGER.info("Received PR action %s", body["action"])
-    with exception_to_github(rest_pr_as_issue, sdkid):
+    with exception_to_github(dashboard, sdk_tag):
         if body["action"] in ["opened", "reopened"]:
-            return rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
+            return rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdkbase, sdk_tag)
         if body["action"] == "closed":
-            return rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
+            return rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdkbase, sdk_tag)
         if body["action"] == "synchronize": # push to a PR from a fork
-            return rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
+            return rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdkbase, sdk_tag)
 
-def rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
+def rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdk_default_base="master", sdk_tag=None):
 
     rest_basebranch = body["pull_request"]["base"]["ref"]
     origin_repo = body["pull_request"]["head"]["repo"]["full_name"]
+    pr_title = body["pull_request"]["title"]
     pr_number = body["number"]
 
     sdk_pr_model = SdkPRModel.from_pr_webhook(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base)
     sdk_base = sdk_pr_model.base_branch_name
     sdk_dest_branch = sdk_pr_model.head_branch_name
 
-    sdk_checkout_base = None if rest_basebranch == "master" else sdk_base
+    sdk_checkout_bases = [] if rest_basebranch == "master" else [sdk_base]
+    context_branch = None
+    if len(context_tags) == 1:
+        context_branch = "restapi_auto_"+context_tags[0]
+        sdk_checkout_bases.insert(0, context_branch)
 
     rest_pr = restapi_repo.get_pull(pr_number)
+    dashboard = DashboardCommentableObject(rest_pr, "# Automation for {}".format(sdk_tag))
 
     if origin_repo != restapi_repo.full_name:
         # Let's always take the PR files list, and not the one from the commit.
         # If the PR contains a merge commit, we might generate the entire world
         # even if the global PR itself is not impacted by this merge commit.
         _LOGGER.info("This comes from a fork, I need generation first, since targetted branch does not exist")
-        generate_sdk_from_git_object(
+        msg = generate_sdk_from_git_object(
             rest_pr,
             sdk_dest_branch,
             origin_repo,
             sdk_pr_target_repo.full_name,
-            sdk_checkout_base,
+            sdk_checkout_bases,
             fallback_base_branch_name=sdk_default_base,
             sdk_tag=sdk_tag
         )
+        dashboard.create_comment(msg)
+    else:
+        context_branch = None
+
+    # Let it raise at worst
+    github_pr = get_or_create_pull(
+        sdk_pr_target_repo,
+        title='[AutoPR {}] {}'.format("/".join(context_tags), pr_title),
+        body="Created to sync {}".format(rest_pr.html_url),
+        head=sdk_pr_target_repo.owner.login+":"sdk_dest_branch,
+        base=context_branch or sdk_base
+    )
+    dashboard.create_comment("A PR has been created for you:\n{}".format(github_pr.html_url))
 
-    try: # Try to create or get a PR
-        github_pr = sdk_pr_target_repo.create_pull(
-            title='Automatic PR of {} into {}'.format(sdk_dest_branch, sdk_base),
-            body="Created to sync {}".format(rest_pr.html_url),
-            head=sdk_dest_branch,
-            base=sdk_base
-        )
-    except GithubException as err:
-        if err.status == 422 and err.data['errors'][0].get('message', '').startswith('A pull request already exists'):
-            _LOGGER.info('PR already exists, it was a commit on an open PR')
-            github_pr = list(sdk_pr_target_repo.get_pulls(
-                head=sdk_pr_target_repo.owner.login+":"+sdk_dest_branch,
-                base=sdk_base
-            ))[0]
-        else:
-            _LOGGER.warning("Unable to create PR:\n%s", err.data)
-            return {'message': err.data}
-    except Exception as err:
-        response = traceback.format_exc()
-        _LOGGER.warning("Unable to create PR:\n%s", response)
-        return {'message': response}
-
-    try: # Try to label it
+    try: # Try to label it. Catch, consider failing not critical.
         sdk_pr_as_issue = sdk_pr_target_repo.get_issue(github_pr.number)
         sdk_pr_as_issue.add_to_labels(get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.in_progress))
         safe_remove_label(sdk_pr_as_issue, get_or_create_label(sdk_pr_target_repo, SwaggerToSdkLabels.refused))
@@ -248,7 +258,15 @@ def rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_defau
         response = traceback.format_exc()
         _LOGGER.info("Unable to label PR %s:\n%s", github_pr.number, response)
         return {'message': response}
-
+
+    if context_branch:
+        get_or_create_pull(
+            sdk_pr_target_repo,
+            title='[AutoPR] {}'.format("/".join(context_tags)),
+            body="Created to accumulate context: {}".format(context_tags[0]),
+            head=sdk_pr_target_repo.owner.login+":"+context_branch,
+            base=sdk_base
+        )
 
 class SwaggerToSdkLabels(Enum):
     merged = "RestPRMerged", "0e8a16"
@@ -300,29 +318,34 @@ def from_pr_webhook(cls, webhook_body, github_con, restapi_repo, sdk_pr_target_r
             base_branch_name=sdk_base
         )
 
-def rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
+def rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdk_default_base="master", sdk_tag=None):
     _LOGGER.info("Received a PR closed event")
     sdkid = sdk_pr_target_repo.full_name
     rest_pr = restapi_repo.get_pull(body["number"])
 
     sdk_pr_model = SdkPRModel.from_pr_webhook(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base)
 
+    context_branch = None
+    if len(context_tags) == 1:
+        context_branch = "restapi_auto_"+context_tags[0]
+
     sdk_prs = list(sdk_pr_target_repo.get_pulls(
         head=sdk_pr_target_repo.owner.login+":"+sdk_pr_model.head_branch_name,
-        base=sdk_pr_model.base_branch_name
+        base=context_branch or sdk_pr_model.base_branch_name
     ))
     if not sdk_prs:
         # Didn't find it, it's probably because the bot wasn't there when it was created. Let's be smart and do it now.
-        rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base, sdk_tag)
+        rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdk_default_base, sdk_tag)
         # Look for it again now
         sdk_prs = list(sdk_pr_target_repo.get_pulls(
             head=sdk_pr_target_repo.owner.login+":"+sdk_pr_model.head_branch_name,
-            base=sdk_pr_model.base_branch_name
+            base=context_branch or sdk_pr_model.base_branch_name
         ))
 
     if not sdk_prs:
         # Not possible in theory, but let's be sad in the PR comment
-        rest_pr.create_issue_comment("Was unable to create SDK {} PR for this closed PR.".format(sdkid))
+        dashboard = DashboardCommentableObject(rest_pr, "# Automation for {}".format(sdk_tag))
+        dashboard.create_comment("Was unable to create SDK {} PR for this closed PR.".format(sdk_tag))
     elif len(sdk_prs) == 1:
         sdk_pr = sdk_prs[0]
         sdk_pr_as_issue = sdk_pr_target_repo.get_issue(sdk_pr.number)
@@ -337,9 +360,9 @@ def rest_pull_close(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_defa
     else:
         # Should be impossible, create_pull would have sent a 422
         pr_list = "\n".join(["- {}".format(pr.html_url) for pr in sdk_prs])
-        rest_pr.create_issue_comment("We found several SDK {} PRs and didn't notify closing event.\n{}".format(sdkid, pr_list))
+        _LOGGER.info("We found several SDK {} PRs and didn't notify closing event.\n{}".format(sdkid, pr_list))
 
-def rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
+def rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdk_default_base="master", sdk_tag=None):
 
     if body["before"] == body["after"]:
         return {'message': 'No commit id change'}
@@ -357,23 +380,25 @@ def rest_pull_sync(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_defau
     ))
     if not sdk_prs:
         # Didn't find it, let's consider this event as opening
-        return rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, sdk_default_base, sdk_tag)
+        return rest_pull_open(body, github_con, restapi_repo, sdk_pr_target_repo, context_tags, sdk_default_base, sdk_tag)
 
     pr_number = body["number"]
     rest_pr = restapi_repo.get_pull(pr_number)
     dest_branch = body["pull_request"]["head"]["ref"]
     fork_repo = github_con.get_repo(origin_repo)
     fork_owner = fork_repo.owner.login
     subbranch_name_part = fork_owner+"_"+dest_branch
-    generate_sdk_from_git_object(
+    msg = generate_sdk_from_git_object(
         rest_pr,
         "restapi_auto_"+subbranch_name_part,
         origin_repo,
         sdk_pr_target_repo.full_name,
-        None, # I don't know if the origin branch comes from "master", assume it.
+        [], # I don't know if the origin branch comes from "master", assume it.
         fallback_base_branch_name=sdk_default_base,
         sdk_tag=sdk_tag
     )
+    dashboard = DashboardCommentableObject(rest_pr, "# Automation for {}".format(sdk_tag))
+    dashboard.create_comment(msg)
 
     return {'message': 'No return for this endpoint'}