#implement #6 & #7, refactoring

httpreverse/__init__.py

@@ -4,5 +4,6 @@
 __email__ = 'petri@koodaamo.fi'
 __version__ = '0.1.0'
 
-from .httpreverse import expand_jinja, apply_template, parametrize
+from .httpreverse import expand_jinja, apply_template, parametrize, marshal
+from .httpreverse import marshal_request_params, marshal_request_body
 from .httpreverse import _load_parser, _load_generator

httpreverse/httpreverse.py

@@ -3,13 +3,14 @@
 import re
 import json
 from importlib import import_module
-from collections import ChainMap
+from collections import ChainMap, Mapping, MutableSequence
 import yaml
 import jinja2
 import xmltodict
 
-# parameter name extraction regexp
-prm_xpr = re.compile('(\\$([a-zA-Z0-9_])+)+')
+# parameter (name) extraction regexps
+prm_xpr = re.compile('(\$)[\w_]+') # used to get all parameters
+single_xpr = re.compile(" *\$[\w_]+ *$") # used to check if value is single parameter
 
 
 def expand_jinja(apispecstr, context):
@@ -24,63 +25,150 @@ def apply_template(opspec, templates):
    try:
       template = templates[opspec["template"]]
    except:
-      raise Exception("template not specified or found!")
+      raise Exception("not using a template or given template not found!")
+
+   # workarounds for when one dict shadows another's subelements in ChainMap
+   # todo: review the ChainMap-based implementation, possibly replace with a custom one
+
+   templated = ChainMap({}, opspec, template)
 
-   # workaround for the case when one dict shadows another's subelements in ChainMap;
-   # in this case, this may happen with request params
-   params =  ChainMap(opspec["request"].get("params", {}), template["request"].get("params", {}))
-   opspec["request"]["params"] = params
+   if "request" in opspec and "request" in template:
+      templated["request"] = ChainMap(opspec["request"], template["request"])
+      if "params" in opspec["request"] and "params" in template["request"]:
+         params = ChainMap(opspec["request"]["params"], template["request"]["params"])
+         templated["request"]["params"] = params
+   if "response" in opspec and "response" in template:
+      templated["response"] = ChainMap(opspec["response"], template["response"])
 
-   opspec["request"] = ChainMap(opspec["request"], template["request"])
-   opspec["response"] = ChainMap(opspec.get("response", {}), template.get("response", {}))
+   opspec["request"] = templated["request"]
+   opspec["response"] = templated["response"]
 
-   return opspec
+   return templated
+
+
+def _substitute(data, context):
+   "traverse the data structure and do parameter substitution"
+
+   if isinstance(data, Mapping):
+      iterable = data.items()
+   elif isinstance(data, MutableSequence):
+      iterable = enumerate(data)
+   elif isinstance(data, str):
+      # single replacable parameter name in string
+      if re.match(single_xpr, data):
+         return context[data.strip().lstrip('$')]
+      # multiple parameter names in string
+      else:
+         return re.sub(prm_xpr, lambda m: context[m.group()[1:]], data)
+   else:
+      return data
+
+   for k, v in iterable:
+      # try to substitute any string parts starting with marker
+      if type(v) == str:
+         # if the value is a single replacable, replace from context directly
+         if re.match(single_xpr, v):
+            data[k] = context[v.strip().lstrip('$')]
+         # if there are multiple replacables, they must be strings so do re.sub
+         else:
+            data[k] = re.sub(prm_xpr, lambda m: context[m.group()[1:]], v)
+      # or traverse deeper when needed, using recursion
+      elif isinstance(data, Mapping) or isinstance(data, MutableSequence):
+         _substitute(v, context) # RECURSE
+      else:
+         pass
+   return data
+
+
+def istypedvalue(v):
+   if isinstance(v, Mapping) and "type" in v and "value" in v and len(v) == 2:
+      return True
+   else:
+      return False
+
+
+def ismarshallable(v):
+   return True if isinstance(v, (Mapping, MutableSequence, tuple)) else False
+
+
+def marshal_typed_value(value, default):
+   "given a plain data structure or typed one, marshal it"
+
+   if istypedvalue(value):
+      marshal_to = value["type"]
+      marshallable = value["value"]
+   elif default:
+      marshal_to = default
+      marshallable = value
+   else:
+      raise Exception("marshaling requires default or explicit value for type")
+
+   if "json" in marshal_to:
+      marshalled = json.dumps(marshallable)
+   elif "xml" in marshal_to:
+      marshalled = xmltodict.unparse(marshallable)
+   else:
+      raise Exception("can only marshal to JSON or XML, not '%s'" % marshal_to)
+   return {"value": marshalled, "type": marshal_to}
 
 
-def _parametrize_mapping(mapping, context):
-   for k, v in mapping.items():
-      paramsfound = [g[0] for g in re.findall(prm_xpr, str(v) or "")]
-      paramnames = [n.lstrip('$') for n in paramsfound]
-      for param, name in zip(paramsfound, paramnames):
-         try:
-            mapping[k] = str(v).replace(param, str(context[name]))
-            #print("replaced '%s' with '%s'" % (v, v.replace(param, context[name])))
-         except AttributeError:
-            raise Exception("parameter %s not found in given context!" % name)
+def marshal_request_params(opspec, defaults):
+   "convert structural parameters to the specified type"
 
-         try:
-            mapping[k] = eval(mapping[k])
-         except:
-            pass
+   default = defaults.get("structured_param_type", "")
+   request = opspec["request"]
+   params = opspec["request"]["params"]
 
-   return mapping
+   for paramname, param in params.items():
+      if isinstance(param, (Mapping, MutableSequence)):
+         marshalled = marshal_typed_value(param, default)
+         request["params"][paramname] = marshalled["value"]
 
+   return request["params"]
 
-def parametrize(opspec, context={}, implicit=False, tojson=False, toxml=True):
-   "assign parameter values, optionally implicitly using parameter names"
 
-   # request parameters
-   rparams = opspec["request"].get("params")
-   if rparams:
+def marshal_request_body(opspec, defaults):
+   "convert body to the specified type"
 
-      if implicit:
-         for k, v in context.items():
-            if k in rparams:
-               rparams[k] = v
+   import pdb; pdb.set_trace()
 
-      _parametrize_mapping(rparams, context)
+   default = defaults.get("structured_body_type", "")
+   request = opspec["request"]
+   body = request.get("body", "")
 
-   # request body
-   rbody = opspec["request"].get("body")
-   if rbody:
-      _parametrize_mapping(rbody, context)
+   if isinstance(body, (Mapping, MutableSequence)):
+      marshalled = marshal_typed_value(body, default)
+      request["body"] = marshalled
 
-      # convert body to the type given
-      if "json" in opspec["request"].get("type", "") and tojson:
-         opspec["request"]["body"] = json.dumps(rbody)
-      elif "xml" in opspec["request"].get("type", "") and toxml:
-         opspec["request"]["body"] = xmltodict.unparse(rbody)
+   return request["body"]
 
+
+def marshal(opspec, defaults):
+   marshal_request_params(opspec, defaults)
+   if ismarshallable(opspec["request"].get("body")):
+      marshal_request_body(opspec, defaults)
+
+
+def _parametrize_request_params(request, context):
+   if "params" in request:
+      request["params"] = _substitute(request["params"], context)
+   return request
+
+
+def _parametrize_request_body(request, context):
+   if "body" in request:
+      request["body"] = _substitute(request["body"], context)
+   return request
+
+
+def parametrize(opspec, context={}):
+   "assign parameter values to params/body, optionally implicitly using parameter names"
+
+   request = opspec["request"]
+   if "params" in request:
+      opspec["request"] = _parametrize_request_params(request, context)
+   if "body" in request:
+      opspec["request"] = _parametrize_request_body(request, context)
    return opspec
 
 
@@ -110,19 +198,15 @@ def _load_parser(opspec, assign=True):
    "parse & load and (by default) assign response parser callable, if found"
 
    parser = _load_callable(opspec["response"]["parser"])
-
    if assign:
       opspec["response"]["parser"] = parser
-
    return parser
 
 
 def _load_generator(opspec, assign=True):
    "parse & load and (by default) assign request generator callable, if found"
 
    generator = _load_callable(opspec["request"]["generator"])
-
    if assign:
       opspec["request"]["generator"] = generator
-
    return generator

tests/test_httpreverse.py

@@ -12,6 +12,7 @@
 import yaml
 import xmltodict
 from httpreverse import expand_jinja, apply_template, parametrize
+from httpreverse import marshal_request_params, marshal_request_body
 from httpreverse import _load_parser, _load_generator
 
 
@@ -30,6 +31,7 @@ def _get_expanded_testop(self, name):
 
 
 class Test01_JinjaExpansion(BaseTestCase):
+   "test API expansion using Jinja templating"
 
    def test1_expansion(self):
       "the Jinja syntax is expanded with given context"
@@ -44,6 +46,7 @@ def test2_parsable(self):
 
 
 class Test02_TemplateApplication(BaseTestCase):
+   "test request template system"
 
    def setUp(self):
       super().setUp()
@@ -52,12 +55,14 @@ def setUp(self):
       self.templates = self.parsed["templates"]
 
    def test1_applytemplate_for_all(self):
+      "the request remplates are expanded"
       for opname, opspec in self.parsed["operations"].items():
          opspec = apply_template(opspec, templates=self.templates)
          assert "response" in opspec and "json" in opspec["response"].get("type", "")
 
 
 class Test03_Parametrization(BaseTestCase):
+   "test API operation spec parametrization"
 
    def setUp(self):
       super().setUp()
@@ -67,18 +72,21 @@ def setUp(self):
       self.contexts = self.parsed["contexts"]
 
    def test1_explicit_parametrization(self):
+      "data structure is correctly parametrized"
       testopname = "add-reservation"
       testop = self._get_expanded_testop(testopname)
-      testcontext = {"size":"double", "customers":["John Doe", "Jane Doe"]}
-      parametrized = parametrize(testop, context=testcontext)
-      assert parametrized["request"]["body"] == testcontext
+      testcontext = {"customers":["John Doe", "Jane Doe"], "size":"double"}
+      parametrize(testop, context=testcontext)
+      result = testop["request"]["body"]["value"]
+      assert testop["request"]["body"]["value"] == testcontext
 
    def test2_parametrize_from_static_context(self):
+      "data structure is correctly parametrized from context embedded in API"
       testopname = "add-reservation"
       testop = self._get_expanded_testop(testopname)
       testcontext = self.contexts[testop["context"]]
       parametrized = parametrize(testop, context=testcontext)
-      assert parametrized["request"]["body"] == testcontext
+      assert parametrized["request"]["body"]["value"] == testcontext
 
    def test3_parametrize_partially_from_static_context_nofail(self):
       "can handle partial parametrization from larger static context"
@@ -88,8 +96,17 @@ def test3_parametrize_partially_from_static_context_nofail(self):
       parametrized = parametrize(testop, context=testcontext)
       assert parametrized["request"]["params"]["size"] == testcontext["size"]
 
+   def test4_parametrize_two_variables(self):
+      "can replace $name1 $name2"
+      testopname = "add-note"
+      testop = self._get_expanded_testop(testopname)
+      testcontext = self.contexts[testop["context"]]
+      parametrized = parametrize(testop, context=testcontext)
+      assert parametrized["request"]["body"] == " ".join(testcontext.values())
+
 
 class Test04_loader(BaseTestCase):
+   "test loading of request generators and response parsers"
 
    def setUp(self):
       super().setUp()
@@ -118,28 +135,56 @@ def test2_generator_loading(self):
 
 
 class Test05_body_conversion(BaseTestCase):
+   "test body marshaling by setting body explicitly and then marshaling it"
 
    def setUp(self):
       super().setUp()
       self.expanded = expand_jinja(self.source, context=self.context)
       self.parsed = yaml.load(self.expanded)
 
-      # this will be converted to JSON and XML
-      self.data = {"root": {"size": "double", "customer": ["John", "Jane"]}}
-
+      # the test API operation
       testopname = "list-singlerooms"
       opspec = self.parsed["operations"][testopname]
       self.opspec = apply_template(opspec, templates=self.parsed["templates"])
-      self.opspec["request"]["body"] = self.data
-
-   def test1_json_convert(self):
-      self.opspec["request"]["type"] = "application/json"
-      parametrize(self.opspec, tojson=True)
-      body = self.opspec["request"]["body"]
-      assert self.data == json.loads(body)
-
-   def test2_xml_convert(self):
-      self.opspec["request"]["type"] = "application/xml"
-      parametrize(self.opspec)
-      body = self.opspec["request"]["body"]
-      assert self.data == xmltodict.parse(body)
+
+      # yes, neither the API spec nor the template has a body, we assign the body value
+      # explicitly in the test here, so that this will be converted (type is set by tests)
+      self.bodyvalue = {"root": {"size": "double", "customer": ["John", "Jane"]}}
+      self.opspec["request"]["body"] = {"value": self.bodyvalue}
+
+   def test1_body_json_marshal(self):
+      "request body is correctly marshalled to JSON"
+      self.opspec["request"]["body"]["type"] = "application/json"
+      marshal_request_body(self.opspec, self.parsed.get("defaults", {}))
+      result = self.opspec["request"]["body"]["value"]
+      assert self.bodyvalue == json.loads(result)
+
+   def test2_body_xml_marshal(self):
+      "request body is correctly marshalled to XML"
+      self.opspec["request"]["body"]["type"] = "application/xml"
+      marshal_request_body(self.opspec, self.parsed.get("defaults", {}))
+      result = self.opspec["request"]["body"]["value"] # this should now be XML ...
+      assert self.bodyvalue == xmltodict.parse(result)
+
+
+
+class Test06_params_marshaling(BaseTestCase):
+   "test params marshaling"
+
+   def setUp(self):
+      super().setUp()
+      self.expanded = expand_jinja(self.source, context=self.context)
+      self.parsed = yaml.load(self.expanded)
+
+      # the test API operation
+      testopname = "add-note"
+      opspec = self.parsed["operations"][testopname]
+      self.opspec = apply_template(opspec, templates=self.parsed["templates"])
+
+   def test1_param_json_marshal(self):
+      "request param is correctly marshalled to JSON"
+      context = self.parsed["contexts"][self.opspec["context"]]
+      parametrize(self.opspec, context=context)
+      marshal_request_params(self.opspec, self.parsed.get("defaults", {}))
+      result = self.opspec["request"]["params"]["note"]
+      assert self.opspec["request"]["params"]["note"] == result