Merge 3eaea5e into 0737728

lmazuel · Aug 11, 2018 · f1ad829 · f1ad829
2 parents 0737728 + 3eaea5e
commit f1ad829
Show file tree

Hide file tree

Showing 12 changed files with 258 additions and 105 deletions.
diff --git a/setup.py b/setup.py
@@ -21,13 +21,15 @@
     ],
     install_requires=[
         "azure-devtools[ci_tools]>=1.1.1",
+        "gevent",
         "requests",
         "cookiecutter",
         "wheel"
     ],
     extras_require={
         'rest': [
             'flask',
+            'flask-request-id-middleware',
             'json-rpc'
         ]
     },

diff --git a/swaggertosdk/SwaggerToSdkMain.py b/swaggertosdk/SwaggerToSdkMain.py
@@ -76,18 +76,33 @@ def main(argv):
     """Main method"""
 
     if "--rest-server" in argv:
+        from gevent.pywsgi import WSGIServer
         from .restapi import app
+        from flask import request
+
+        old_factory = logging.getLogRecordFactory()
+        def request_record_factory(*args, **kwargs):
+            record = old_factory(*args, **kwargs)
+            if request:
+                record.requestid = request.environ.get("FLASK_REQUEST_ID")
+            else:
+                record.requestid = "(none)"
+            return record
+        logging.setLogRecordFactory(request_record_factory)
+
         log_level = logging.WARNING
         if "-v" in argv or "--verbose" in argv:
             log_level = logging.INFO
         if "--debug" in argv:
             log_level = logging.DEBUG
 
         main_logger = logging.getLogger()
-        logging.basicConfig()
+        logging.basicConfig(format="%(requestid)s:%(levelname)s:%(name)s:%(message)s")
         main_logger.setLevel(log_level)
 
-        app.run(debug=log_level == logging.DEBUG, host='0.0.0.0')
+        http_server = WSGIServer(('', 5000), app, log=logging.getLogger("WSGIServer"))
+        main_logger.info("REST server is listening on: http://%s:%s", http_server.server_host, http_server.server_port)
+        http_server.serve_forever()
         sys.exit(0)
 
     parser = argparse.ArgumentParser(

diff --git a/swaggertosdk/__main__.py b/swaggertosdk/__main__.py
@@ -1,3 +1,6 @@
+from gevent import monkey
+monkey.patch_all()
+
 import sys
 from swaggertosdk.SwaggerToSdkMain import main
 main(sys.argv)
diff --git a/swaggertosdk/restapi/__init__.py b/swaggertosdk/restapi/__init__.py
@@ -1,10 +1,12 @@
 from flask import Flask
+from flask_request_id import RequestID
 from jsonrpc.backend.flask import api
 
 from ..SwaggerToSdkMain import generate_sdk
 from ..SwaggerToSdkCore import CONFIG_FILE, DEFAULT_COMMIT_MESSAGE
 
 app = Flask(__name__)
+RequestID(app)
 app.add_url_rule('/', 'api', api.as_view(), methods=['POST'])
 
 @api.dispatcher.add_method
@@ -31,3 +33,4 @@ def generate_project(*args, **kwargs):
     )
 
 from .github import *
+from .travis import *
diff --git a/swaggertosdk/restapi/__main__.py b/swaggertosdk/restapi/__main__.py
@@ -1,2 +1,2 @@
 from swaggertosdk.restapi import app
-app.run(debug=True)
+app.run(debug=True)
diff --git a/swaggertosdk/restapi/github.py b/swaggertosdk/restapi/github.py
@@ -1,44 +1,29 @@
-import os
-from enum import Enum
-import hmac
 import hashlib
+import hmac
 import logging
-from queue import Queue
+import os
 import traceback
+from queue import Queue
 from threading import Thread
 
-from flask import request, jsonify
-
+from azure_devtools.ci_tools.bot_framework import BotHandler
+from azure_devtools.ci_tools.github_tools import (DashboardCommentableObject,
+                                                  exception_to_github)
+from flask import jsonify, request
 from github import Github
 
-from azure_devtools.ci_tools.bot_framework import (
-    BotHandler
-)
-from .sdkbot import (
-    GithubHandler
-)
-from .restbot import (
-    RestAPIRepoHandler
-)
-from .github_handler import (
-    rest_pr_management,
-    generate_sdk_from_git_object
-)
-from azure_devtools.ci_tools.github_tools import (
-    exception_to_github,
-    DashboardCommentableObject,
-)
 from . import app
+from .github_handler import generate_sdk_from_git_object, rest_pr_management
+from .restbot import RestAPIRepoHandler
+from .sdkbot import GithubHandler
 
+_HMAC_CHECK = False
 _LOGGER = logging.getLogger("swaggertosdk.restapi.github")
 _QUEUE = Queue(64)
 
-
 # Webhook secreet to authenticate message (bytes)
 SECRET = b'mydeepsecret'
 
-_HMAC_CHECK = False
-
 def check_hmac(local_request, secret):
     data = local_request.get_data()
     hmac_tester = hmac.HMAC(secret, data, hashlib.sha1)
@@ -68,12 +53,12 @@ def notify():
 def rest_notify():
     """Github rest endpoint."""
     sdkid = request.args.get("sdkid")
-    sdkbase = request.args.get("sdkbase", "master")
-    sdk_tag = request.args.get("repotag", sdkid.split("/")[-1].lower())
-
     if not sdkid:
         return {'message': 'sdkid is a required query parameter'}
 
+    sdkbase = request.args.get("sdkbase", "master")
+    sdk_tag = request.args.get("repotag", sdkid.split("/")[-1].lower())
+
     rest_bot = RestAPIRepoHandler(sdkid, sdk_tag, sdkbase)
     bot = BotHandler(rest_bot)
     github_index = {
@@ -105,7 +90,7 @@ def notify_github(github_index, event_type, json_body):
         return github_index[event_type](json_body)
     return {'message': 'Not handled currently'}
 
-def ping(body):
+def ping(body): # pylint: disable=unused-argument
     return {'message': 'Moi aussi zen beaucoup'}
 
 def push(body):
@@ -165,26 +150,26 @@ def rest_handle_action(body, sdkid, sdkbase, sdk_tag):
     _LOGGER.info("Received PR action %s", body["action"])
     with exception_to_github(dashboard, sdk_tag):
         if body["action"] in ["opened", "reopened"]:
-            return rest_pull_open(body, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
+            return rest_pull_open(dashboard, body, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
         if body["action"] == "closed":
-            return rest_pull_close(body, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
+            return rest_pull_close(dashboard, body, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
         if body["action"] == "synchronize": # push to a PR from a fork
-            return rest_pull_sync(body, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
+            return rest_pull_sync(dashboard, body, restapi_repo, sdk_pr_target_repo, sdkbase, sdk_tag)
 
-def rest_pull_open(body, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
+def rest_pull_open(commentable, body, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
     _LOGGER.info("Received a PR open event")
 
     rest_pr = restapi_repo.get_pull(body["number"])
-    rest_pr_management(rest_pr, sdk_pr_target_repo, sdk_tag, sdk_default_base)
+    rest_pr_management(commentable, rest_pr, sdk_pr_target_repo, sdk_tag, sdk_default_base)
 
 
-def rest_pull_close(body, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
+def rest_pull_close(commentable, body, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
     _LOGGER.info("Received a PR closed event")
 
     rest_pr = restapi_repo.get_pull(body["number"])
-    rest_pr_management(rest_pr, sdk_pr_target_repo, sdk_tag, sdk_default_base)
+    rest_pr_management(commentable, rest_pr, sdk_pr_target_repo, sdk_tag, sdk_default_base)
 
-def rest_pull_sync(body, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
+def rest_pull_sync(commentable, body, restapi_repo, sdk_pr_target_repo, sdk_default_base="master", sdk_tag=None):
 
     # If this sync has no commit change, save CPU time.
     if body["before"] == body["after"]:
@@ -197,7 +182,7 @@ def rest_pull_sync(body, restapi_repo, sdk_pr_target_repo, sdk_default_base="mas
         return
 
     rest_pr = restapi_repo.get_pull(body["number"])
-    rest_pr_management(rest_pr, sdk_pr_target_repo, sdk_tag, sdk_default_base)
+    rest_pr_management(commentable, rest_pr, sdk_pr_target_repo, sdk_tag, sdk_default_base)
 
 def consume():
     """Consume action and block if there is not.
@@ -207,7 +192,7 @@ def consume():
         _LOGGER.info("Pop from queue. Queue size: %d", _QUEUE.qsize())
         try:
             rest_handle_action(body, sdkid, sdkbase, sdk_tag)
-        except Exception as err:
+        except Exception: # pylint: disable=broad-except
             _LOGGER.critical("Worked thread issue:\n%s", traceback.format_exc())
     _LOGGER.info("End of WorkerThread")
 

diff --git a/swaggertosdk/restapi/github_handler.py b/swaggertosdk/restapi/github_handler.py
@@ -1,24 +1,11 @@
-from enum import Enum
 import logging
-import os
-from pathlib import Path
-import tempfile
+from enum import Enum
 
-from github import UnknownObjectException, GithubException
+from azure_devtools.ci_tools.github_tools import get_or_create_pull
+from github import GithubException, UnknownObjectException
 
-from swaggertosdk.SwaggerToSdkCore import (
-    get_context_tag_from_git_object,
-)
+from swaggertosdk.SwaggerToSdkCore import get_context_tag_from_git_object
 from swaggertosdk.SwaggerToSdkNewCLI import generate_sdk_from_git_object
-from azure_devtools.ci_tools.github_tools import (
-    get_or_create_pull,
-    DashboardCommentableObject,
-    manage_git_folder,
-    configure_user,
-    user_from_token
-)
-
-from git import Repo
 
 _LOGGER = logging.getLogger("swaggertosdk.restapi.github_handler")
 
@@ -65,11 +52,11 @@ def manage_labels(issue, to_add=None, to_remove=None):
     for label_add in to_add:
         try:
             issue.add_to_labels(get_or_create_label(issue.repository, label_add))
-        except Exception as err:
+        except Exception: # pylint: disable=broad-except
             # Never fail is adding a label was impossible
             _LOGGER.warning("Unable to add label: %s", label_add)
 
-def rest_pr_management(rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK_BRANCH):
+def rest_pr_management(commentable, rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK_BRANCH):
     """What to do when something happen to a PR in the Rest repo.
 
     :param restpr: a PyGithub pull object
@@ -84,19 +71,16 @@ def rest_pr_management(rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK
     # "repo" can be None if fork has been deleted.
     is_from_a_fork = rest_pr.head.repo is None or rest_pr.head.repo.full_name != rest_repo.full_name
 
-    # THE comment were we put everything
-    dashboard = DashboardCommentableObject(rest_pr, "# Automation for {}".format(sdk_tag))
-
     #
     # Work on context, ext if context is not good
     #
     context_tags = list(get_context_tag_from_git_object(rest_pr))
     if not context_tags:
-        dashboard.create_comment("Unable to detect any generation context from this PR.")
+        commentable.create_comment("### No generation context\nUnable to detect any generation context from this pull request.")
         return
     if len(context_tags) > _CONTEXT_TAG_LIMITS:
-        dashboard.create_comment(
-            "This PR contains more than {} context, SDK generation is not enabled. Contexts found:\n{}".format(
+        commentable.create_comment(
+            "### Too many contexts\nThis pull request contains more than {} contexts, SDK generation has been skipped. Contexts found:\n{}".format(
                 _CONTEXT_TAG_LIMITS,
                 "\n".join(["- {}".format(ctxt) for ctxt in context_tags])
             ))
@@ -164,13 +148,13 @@ def rest_pr_management(rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK
             head=sdk_repo.owner.login+":"+sdk_pr_head,
             base=sdk_pr_base,
         )
-    except Exception as err:
+    except Exception as err: # pylint: disable=broad-except
         _LOGGER.warning("Unable to create SDK PR: %s", err)
-        dashboard.create_comment("Nothing to generate for {}".format(sdk_tag))
+        commentable.create_comment("### Nothing to generate\nThe changes in this pull request did not result in any changes in the {} repository.".format(sdk_tag))
         return
 
     # Replace whatever message it was if we were able to do a PR
-    dashboard.create_comment("A PR has been created for you:\n{}".format(sdk_pr.html_url))
+    commentable.create_comment("### Pull request created\nPull request {}#{} has been created for you based on the changes in this pull request.".format(sdk_repo.full_name, sdk_pr.number))
 
     #
     # Manage labels/state on this SDK PR.
@@ -190,7 +174,7 @@ def rest_pr_management(rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK
                     sdk_pr_merged = True
                     # Delete branch from merged PR
                     head_ref.delete()
-                except Exception as err:
+                except Exception as err: # pylint: disable=broad-except
                     _LOGGER.warning("Was unable to merge: %s", err)
         else:
             manage_labels(sdk_pr_as_issue,
@@ -217,7 +201,7 @@ def rest_pr_management(rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK
                 head=sdk_repo.owner.login+":"+sdk_pr_base,
                 base=sdk_default_base,
             )
-        except Exception as err:
+        except Exception as err: # pylint: disable=broad-except
             _LOGGER.warning("Unable to create context PR: %s", err)
             return
         # We got the context PR!
@@ -227,19 +211,20 @@ def rest_pr_management(rest_pr, sdk_repo, sdk_tag, sdk_default_base=_DEFAULT_SDK
         if sdk_pr_merged:
             sdk_pr.create_issue_comment("This PR has been merged into {}".format(context_pr.html_url))
         # Update dashboar to talk about this PR
+        msg = "Pull request {}#{} was created for you based on the changes in this pull request.\n".format(sdk_repo.full_name, sdk_pr.number)
         if sdk_pr.merged:
-            msg = "The initial [PR]({}) has been merged into your service PR:\n{}".format(
-                sdk_pr.html_url,
-                context_pr.html_url
+            title = "Pull request created and integrated"
+            msg += "It has been merged into the service pull request located here: {}#{}".format(
+                sdk_repo.full_name,
+                context_pr.number
             )
         else:
-            msg = "A [PR]({}) has been created for you based on this PR content.\n\n".format(
-                sdk_pr.html_url
-            )
-            msg += "Once this PR will be merged, content will be added to your service PR:\n{}".format(
-                context_pr.html_url
+            title = "Pull request created, pending integration"
+            msg += "Once this pull request is merged, content will be added to the service pull request located here: {}#{}".format(
+                sdk_repo.full_name,
+                context_pr.number
             )
-        dashboard.create_comment(msg)
+        commentable.create_comment("### {}\n{}".format(title, msg))
 
 def clean_sdk_pr(rest_pr, sdk_repo):
     """Look for the SDK pr created by this RestPR and wipe it.

diff --git a/swaggertosdk/restapi/restbot.py b/swaggertosdk/restapi/restbot.py
@@ -1,10 +1,10 @@
-import os
 import logging
+import os
 
+from azure_devtools.ci_tools.bot_framework import order
 from github import Github
 
-from azure_devtools.ci_tools.bot_framework import order
-from .github_handler import rest_pr_management, clean_sdk_pr
+from .github_handler import clean_sdk_pr, rest_pr_management
 
 _LOGGER = logging.getLogger("swaggertosdk.restapi.restbot")