[WIP] Converts Python coprocess glue module to handcrafted C (#1338)

* Converts Python coprocess glue module to handcrafted C

This replaces Cython-generated code in Python coprocess with handcrafted C using only limited Python C API subset so that Py_LIMITED_API preprocessor macro could be set and the resulting binary dynamically linked with whatever version of Python as long as it's 3.2+.

At the same time plugin interface is left compatible by providing a gateway.py module that actually calls the C wrapper functions.

In addition to this, Python code has been partly made PEP-8 conformant.

coprocess/python/dispatcher.py

@@ -1,21 +1,19 @@
 from glob import glob
-from os import getcwd, chdir, path
+from os import path
 import sys, traceback
 
-import tyk
 from tyk.middleware import TykMiddleware
 from tyk.object import TykCoProcessObject
 from tyk.event import TykEvent, TykEventHandler
 
 from gateway import TykGateway as tyk
 
-from importlib.machinery import SourceFileLoader
 
 class TykDispatcher:
     '''A simple dispatcher'''
 
     def __init__(self, middleware_path, event_handler_path, bundle_paths):
-        tyk.log( "Initializing dispatcher", "info" )
+        tyk.log("Initializing dispatcher", "info")
         self.event_handler_path = path.join(event_handler_path, '*.py')
         self.event_handlers = {}
         self.load_event_handlers()
@@ -29,7 +27,7 @@ def __init__(self, middleware_path, event_handler_path, bundle_paths):
 
     def get_modules(self, the_path):
         files = glob(the_path)
-        files = [ path.basename( f.replace('.py', '') ) for f in files ]
+        files = [path.basename(f.replace('.py', '')) for f in files]
         return files
 
     def find_middleware(self, path):
@@ -56,9 +54,8 @@ def load_bundle(self, base_bundle_path):
                 self.middlewares.append(middleware)
 
         self.update_hook_table()
-
     def load_middlewares(self):
-        tyk.log( "Loading middlewares.", "debug" )
+        tyk.log("Loading middlewares.", "debug")
         available_modules = self.get_modules(self.middleware_path)
         for module_name in available_modules:
             middleware = self.find_middleware(module_name)
@@ -70,11 +67,11 @@ def load_middlewares(self):
         self.update_hook_table()
 
     def purge_middlewares(self):
-        tyk.log( "Purging middlewares.", "debug" )
+        tyk.log("Purging middlewares.", "debug")
         available_modules = self.get_modules(self.middleware_path)
         for middleware in self.middlewares:
-            if not middleware.filepath in available_modules:
-                tyk.log( "Purging middleware: '{0}'".format(middleware.filepath), "warning" )
+            if middleware.filepath not in available_modules:
+                tyk.log("Purging middleware: '{0}'".format(middleware.filepath), "warning")
                 self.middlewares.remove(middleware)
 
     def update_hook_table(self):
@@ -110,21 +107,21 @@ def dispatch_hook(self, object_msg):
             if hook_handler:
                 object = middleware.process(hook_handler, object)
             else:
-                tyk.log( "Can't dispatch '{0}', hook is not defined.".format(object.hook_name), "error")
+                tyk.log("Can't dispatch '{0}', hook is not defined.".format(object.hook_name), "error")
             return object.dump()
         except:
             exc_trace = traceback.format_exc()
             print(exc_trace)
-            tyk.log_error( "Can't dispatch, error:" )
+            tyk.log_error("Can't dispatch, error:")
 
             return object_msg
 
     def purge_event_handlers(self):
-        tyk.log( "Purging event handlers.", "debug" )
+        tyk.log("Purging event handlers.", "debug")
         self.event_handlers = {}
 
     def load_event_handlers(self):
-        tyk.log( "Loading event handlers.", "debug" )
+        tyk.log("Loading event handlers.", "debug")
         for module_name in self.get_modules(self.event_handler_path):
             event_handlers = TykEventHandler.from_module(module_name)
             for event_handler in event_handlers:
@@ -143,10 +140,10 @@ def dispatch_event(self, event_json):
             if event_handler:
                 event_handler.process(event)
         except:
-            tyk.log_error( "Can't dispatch, error:")
+            tyk.log_error("Can't dispatch, error:")
 
     def reload(self):
-        tyk.log( "Reloading event handlers and middlewares.", "info" )
+        tyk.log("Reloading event handlers and middlewares.", "info")
         self.purge_event_handlers()
         self.load_event_handlers()
         self.load_middlewares()

coprocess/python/gateway.py

@@ -0,0 +1,33 @@
+"""
+This module provides interface compatibility and flexibility for the C glue code in tyk/gateway_wrapper.c
+"""
+from sys import exc_info
+
+import gateway_wrapper as gw
+
+
+class TykGateway:
+
+    @classmethod
+    def store_data(cls, key, value, ttl):
+        gw.store_data(key, value, ttl)
+
+    @classmethod
+    def get_data(cls, key):
+        return gw.get_data(key)
+
+    @classmethod
+    def trigger_event(cls, event_name, payload):
+        gw.trigger_event(event_name, payload)
+
+    @classmethod
+    def log(cls, msg, level):
+        gw.log(msg, level)
+
+    @classmethod
+    def log_error(cls, *args):
+        excp = exc_info()
+        if len(args) == 0:
+            cls.log("{0} {1}".format(excp[0], excp[1]), "error")
+        else:
+            cls.log("{0} {1} {2}".format(args[0], excp[0], excp[1]), "error")

coprocess/python/tyk/decorators.py

@@ -1,52 +1,60 @@
 from inspect import getargspec
 
+
 class HandlerDecorator(object):
     def __init__(self, f):
         self.name = f.__name__
         self.f = f
-        return
+
     def __call__(self, req, sess, spec):
         self.f
 
+
 class Hook(object):
     def __init__(self, f):
         self.name = f.__name__
         self.f = f
         self.arg_count = len(getargspec(f)[0])
-        return
+
     def __call__(self, *args, **kwargs):
         if self.arg_count == 3:
             return self.f(args[0], args[1], args[2])
         if self.arg_count == 4:
             return self.f(args[0], args[1], args[2], args[3])
 
+
 class Pre(HandlerDecorator):
     def __call__(self, req, sess, spec):
         return self.f(req, sess, spec)
 
+
 class Post(HandlerDecorator):
     def __call__(self, req, sess, spec):
         return self.f(req, sess, spec)
 
+
 class PostKeyAuth(HandlerDecorator):
     def __call__(self, req, sess, spec):
         return self.f(req, sess, spec)
 
+
 class CustomKeyCheck():
     def __init__(self, f):
         self.f = f
         self.name = f.__name__
-        return
+
     def __call__(self, req, sess, metadata, spec):
         return self.f(req, sess, metadata, spec)
 
+
 class Event(object):
     def __init__(self, f):
         self.name = f.__name__
         self.f = f
-        return
+
     def __call__(self, event, spec):
         self.f(event, spec)
 
+
 def ThisIsNotADecorator():
     pass

coprocess/python/tyk/event.py

@@ -3,12 +3,12 @@
 
 import tyk.decorators as decorators
 
+
 class TykEventHandler:
     def __init__(self, name, callback):
         self.name = name
         self.callback = callback
 
-
     def process(self, event):
         print("process", event)
         self.callback(event.message, event.spec)
@@ -25,6 +25,7 @@ def from_module(module_name):
                     event_handlers.append(event_handler)
         return event_handlers
 
+
 class TykEvent:
     def __init__(self, event_json):
         self.__dict__ = json.loads(event_json)