diff --git a/objectpath/core/interpreter.py b/objectpath/core/interpreter.py index 4ab7fb2..8e8adfb 100644 --- a/objectpath/core/interpreter.py +++ b/objectpath/core/interpreter.py @@ -259,22 +259,21 @@ def exe(node): if type(fst) is tuple: fst=exe(fst) typefst=type(fst) - if D: self.debug("left is '%s'",fst) - if D: self.debug("node is '%s'",node) + if D: self.debug(color.op(".")+" left is '%s'", fst) try: if node[2][0] == "*": - if D: self.end("returning '%s'", typefst in ITER_TYPES and fst or [fst]) + if D: self.end(color.op(".")+" returning '%s'", typefst in ITER_TYPES and fst or [fst]) return typefst in ITER_TYPES and fst or [fst] except: pass snd=exe(node[2]) - #filterAttrs=type(node[2]) is list - if D: self.debug("right is '%s'",snd) + if D: self.debug(color.op(".")+" right is '%s'",snd) if typefst in ITER_TYPES: - if D: self.debug(color.op("..")+" filtering %s by %s",color.bold(fst),color.bold(snd)) + if D: self.debug(color.op(".")+" filtering %s by %s",color.bold(fst),color.bold(snd)) if type(snd) in ITER_TYPES: return filter_dict(fst, list(snd)) else: + #if D: self.debug(list(fst)) return (e[snd] for e in fst if type(e) is dict and snd in e) try: if D: self.end(color.op(".")+" returning '%s'",fst.get(snd)) @@ -293,14 +292,17 @@ def exe(node): if D: self.debug(color.op("..")+" returning '%s'", color.bold(fst)) return fst # reduce objects to selected attributes - #filterAttrs=type(node[2]) is list snd=exe(node[2]) if D: self.debug(color.op("..")+" finding all %s in %s", color.bold(snd), color.bold(fst)) - if type(snd) is list: - #if D: self.debug(color.op("..")+" returning %s",color.bold(ret)) - return filter_dict(fst, snd) + if type(snd) in ITER_TYPES: + ret=filter_dict(fst, snd) + if D: self.debug(color.op("..")+" returning %s",color.bold(ret)) + return ret else: - return (e[snd] for e in fst if snd in e) + ret=chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))) + #print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e)))) + if D: self.debug(color.op("..")+" returning %s",color.bold(ret)) + return ret elif op=="[": len_node=len(node) # TODO move it to tree generation phase @@ -317,8 +319,9 @@ def exe(node): return fst selector=node[2] if D: self.debug("found '%s' selector. executing on %s", color.bold(selector),color.bold(fst)) + selectorIsTuple=type(selector) is tuple - if type(selector) is tuple and selector[0] is "[": + if selectorIsTuple and selector[0] is "[": nodeList=[] nodeList_append=nodeList.append for i in fst: @@ -328,49 +331,47 @@ def exe(node): if D: self.debug("returning %s objects: %s", color.bold(len(nodeList)),color.bold(nodeList)) return nodeList - #if type(selector) is tuple and selector[0]=="fn": - # for i in fst: - - if type(selector) is tuple and selector[0] == "(current)": + if selectorIsTuple and selector[0] == "(current)": if D: self.warning(color.bold("$.*[@]")+" is eqivalent to "+color.bold("$.*")+"!") return fst - if type(selector) is tuple and selector[0] in SELECTOR_OPS: + if selectorIsTuple and selector[0] in SELECTOR_OPS: if D: self.debug("found %s operator in selector", color.bold(selector[0])) if type(fst) is dict: fst=[fst] - # TODO move it to tree building phase + # TODO move it to tree building phase if type(selector[1]) is tuple and selector[1][0]=="name": selector=(selector[0],selector[1][1],selector[2]) selector0=selector[0] selector1=selector[1] selector2=selector[2] - nodeList=[] - nodeList_append=nodeList.append - for i in fst: - if D: self.debug("setting self.current to %s", color.bold(i)) - self.current=i - if selector0=="fn": - nodeList_append(exe(selector)) - elif type(selector1) in STR_TYPES: - try: - if exe((selector0,i[selector1],selector2)): - nodeList_append(i) - if D: self.debug("appended") - if D: self.debug("discarded") - except Exception as e: - if D: self.debug("discarded, Exception: %s",color.bold(e)) - else: - try: - # TODO optimize an event when @ is not used. exe(selector1) can be cached - if exe((selector0,exe(selector1),exe(selector2))): - nodeList_append(i) - if D: self.debug("appended") - if D: self.debug("discarded") - except Exception: - if D: self.debug("discarded") + + def exeSelector(fst): + for i in fst: + if D: self.debug("setting self.current to %s", color.bold(i)) + self.current=i + if selector0=="fn": + yield exe(selector) + elif type(selector1) in STR_TYPES: + try: + if exe((selector0,i[selector1],selector2)): + yield i + if D: self.debug("appended") + if D: self.debug("discarded") + except Exception as e: + if D: self.debug("discarded, Exception: %s",color.bold(e)) + else: + try: + # TODO optimize an event when @ is not used. exe(selector1) can be cached + if exe((selector0,exe(selector1),exe(selector2))): + yield i + if D: self.debug("appended") + if D: self.debug("discarded") + except Exception: + if D: self.debug("discarded") + if D: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList)) - return nodeList + return exeSelector(fst) self.current=fst snd=exe(node[2]) typefst=type(fst) diff --git a/objectpath/shell.py b/objectpath/shell.py index 7e3dae8..ed999af 100644 --- a/objectpath/shell.py +++ b/objectpath/shell.py @@ -10,16 +10,17 @@ # this is to prevent various tools from deleting import readline ___x=readline.__doc__ -from objectpath import Tree +from objectpath import Tree, ITER_TYPES from objectpath.utils.colorify import * # pylint: disable=W0614 -from objectpath.utils import json_compat as json -from objectpath.utils.json_compat import printJSON +from objectpath.utils import json_ext as json +from objectpath.utils.json_ext import printJSON def main(): parser=argparse.ArgumentParser(description='Command line options') parser.add_argument('-u', '--url', dest='URL', help='URL containing JSON document.') #parser.add_argument('-xml', dest='xml', help='[EXPERIMENTAL] Expect XML input.',action='store_true') parser.add_argument('-d', '--debug', dest='debug', help='Debbuging on/off.', action='store_true') + parser.add_argument('-p', '--profile', dest='profile', help='Profiling on/off.', action='store_true') parser.add_argument('-e', '--expr', dest='expr', help='Expression/query to execute on file, print on stdout and exit.') parser.add_argument('file', metavar='FILE', nargs="?", help='File name') @@ -31,6 +32,11 @@ def main(): if args.debug: a["debug"]=True + if args.profile: + try: + from guppy import hpy + except: + pass File=args.file #if args.xml: # from utils.xmlextras import xml2tree @@ -61,23 +67,48 @@ def main(): if not expr: print(" "+bold("done")+".") if expr: - print(json.dumps(tree.execute(expr))) - exit() + if args.profile: + import cProfile, pstats, StringIO + pr = cProfile.Profile() + pr.enable() + ret=tree.execute(expr) + if type(ret) in ITER_TYPES: + ret=list(ret) + print(json.dumps(ret)) + if args.profile: + pr.disable() + s = StringIO.StringIO() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print s.getvalue() + return try: while True: + limitResult=5 try: - if sys.version_info.major >= 3: - r=tree.execute(input(">>> ")) + if sys.version >= '3': + r=input(">>> ") else: - r=tree.execute(raw_input(">>> ")) + r=raw_input(">>> ") + + if r.startswith("all"): + limitResult=-1 + r=tree.execute(r[3:].strip()) + else: + r=tree.execute(r) + # python 3 raises error here - unicode is not a proper type there try: if type(r) is unicode: r=r.encode("utf8") except NameError: pass - print(printJSON(r)) + print(printJSON(r,length=limitResult)) + if args.profile: + h = hpy() + print h.heap() except Exception as e: print(e) except KeyboardInterrupt: diff --git a/objectpath/utils/json_ext.py b/objectpath/utils/json_ext.py index a6ecb05..ef55018 100644 --- a/objectpath/utils/json_ext.py +++ b/objectpath/utils/json_ext.py @@ -1,9 +1,9 @@ #!/usr/bin/env python try: - import json + import simplejson as json except ImportError: try: - import simplejson as json + import json except ImportError: raise Exception("JSONNotFound") try: @@ -92,8 +92,11 @@ def rec(o): for i in o[0:length]: rec(i) out(",\n") + if length is -1: + rec(o[-1]) + out(",\n") - if len(o)>length: + if length is not -1 and len(o)>length: out("... ("+str(len(o)-length)+" more items)\n") else: ret.pop() diff --git a/shell.py b/shell.py index 1e126ec..e76fc94 100755 --- a/shell.py +++ b/shell.py @@ -4,3 +4,4 @@ from objectpath import shell shell.main() + diff --git a/tests/test_ObjectPath.py b/tests/test_ObjectPath.py index c78f78f..91b0c2a 100644 --- a/tests/test_ObjectPath.py +++ b/tests/test_ObjectPath.py @@ -73,17 +73,22 @@ def execute_raw(expr): return tree1.execute(expr) +TYPES=[generator,chain] +if sys.version_info.major>2: + TYPES+=[map] + +TYPES=tuple(TYPES) def execute(expr): r=tree1.execute(expr) - if isinstance(r, generator): + if isinstance(r, TYPES): return list(r) else: return r def execute2(expr): r=tree2.execute(expr) - if isinstance(r, generator): + if isinstance(r, TYPES): return list(r) else: return r @@ -283,12 +288,12 @@ def test_builtin_string(self): self.assertEqual(execute("join(['aaą','aaę'],'ć')"),"aaąćaaę") self.assertEqual(execute("join(['aaa','aaa'])"),"aaaaaa") self.assertEqual(execute("join(['aaa','aaa',3,55])"),"aaaaaa355") - self.assertEqual(list(execute("map(upper,['aaa','aaa'])")),["AAA","AAA"]) + self.assertEqual(execute("map(upper,['aaa','aaa'])"),["AAA","AAA"]) def test_builtin_arrays(self): self.assertEqual(execute("sort([1,2,3,4]+[2,4])"), [1,2,2,3,4,4]) self.assertEqual(execute("sort($.._id)"), [1,2,3,4]) - self.assertEqual(list(execute("sort($..l[0].*, _id)")), [{'_id': 3, 'aaa': 'ddd', 'false': 2}, {'_id': 4}]) + self.assertEqual(execute("sort($..l.*, _id)"), [{'_id': 3, 'aaa': 'ddd', 'false': 2}, {'_id': 4}]) self.assertEqual(execute("reverse([1,2,3,4]+[2,4])"), [4,2,4,3,2,1]) self.assertEqual(execute("reverse(sort($.._id))"), [4,3,2,1]) self.assertEqual(execute("len([1,2,3,4]+[2,4])"), 6) @@ -374,12 +379,12 @@ def test_simple_paths(self): self.assertEqual(execute("$.*['test'][0].o._id"), 2) self.assertEqual(execute('[1,"aa",{"a":2,"c":3},{"c":3},{"a":1,"b":2}].(a,b)'), [{"a":2},{"a":1,"b":2}]) self.assertEqual(execute2("$.store.book.(price,title)[0]"), {"price": 8.95, "title": "Sayings of the Century"}) - self.assertEqual(execute2("$..book[0].(price,title)[0]"), {"price": 8.95, "title": "Sayings of the Century"}) + self.assertEqual(execute2("$..book.(price,title)"), [{'price': 8.95, 'title': 'Sayings of the Century'}, {'price': 12.99, 'title': 'Sword of Honour'}, {'price': 8.99, 'title': 'Moby Dick'}, {'price': 22.99, 'title': 'The Lord of the Rings'}]) self.assertIsInstance(execute("now().year"),int) def test_complex_paths(self): self.assertEqual(sorted(execute("$.._id")), [1, 2, 3, 4]) - self.assertEqual(execute("$..l"), [object1["test"]["l"]]) + self.assertEqual(execute("$..l"), object1["test"]["l"]) self.assertEqual(execute("$..l.._id"), [3,4]) self.assertEqual(execute2("$.store.*"), [object2["store"]]) self.assertEqual(execute2("$.store.book.author"), ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien'])