added 'all' to shell and fixed .. operator

objectpath/core/interpreter.py

@@ -259,22 +259,21 @@ def exe(node):
 				if type(fst) is tuple:
 					fst=exe(fst)
 				typefst=type(fst)
-				if D: self.debug("left is '%s'",fst)
-				if D: self.debug("node is '%s'",node)
+				if D: self.debug(color.op(".")+" left is '%s'", fst)
 				try:
 					if node[2][0] == "*":
-						if D: self.end("returning '%s'", typefst in ITER_TYPES and fst or [fst])
+						if D: self.end(color.op(".")+" returning '%s'", typefst in ITER_TYPES and fst or [fst])
 						return typefst in ITER_TYPES and fst or [fst]
 				except:
 					pass
 				snd=exe(node[2])
-				#filterAttrs=type(node[2]) is list
-				if D: self.debug("right is '%s'",snd)
+				if D: self.debug(color.op(".")+" right is '%s'",snd)
 				if typefst in ITER_TYPES:
-					if D: self.debug(color.op("..")+" filtering %s by %s",color.bold(fst),color.bold(snd))
+					if D: self.debug(color.op(".")+" filtering %s by %s",color.bold(fst),color.bold(snd))
 					if type(snd) in ITER_TYPES:
 						return filter_dict(fst, list(snd))
 					else:
+						#if D: self.debug(list(fst))
 						return (e[snd] for e in fst if type(e) is dict and snd in e)
 				try:
 					if D: self.end(color.op(".")+" returning '%s'",fst.get(snd))
@@ -293,14 +292,17 @@ def exe(node):
 					if D: self.debug(color.op("..")+" returning '%s'", color.bold(fst))
 					return fst
 				# reduce objects to selected attributes
-				#filterAttrs=type(node[2]) is list
 				snd=exe(node[2])
 				if D: self.debug(color.op("..")+" finding all %s in %s", color.bold(snd), color.bold(fst))
-				if type(snd) is list:
-					#if D: self.debug(color.op("..")+" returning %s",color.bold(ret))
-					return filter_dict(fst, snd)
+				if type(snd) in ITER_TYPES:
+					ret=filter_dict(fst, snd)
+					if D: self.debug(color.op("..")+" returning %s",color.bold(ret))
+					return ret
 				else:
-					return (e[snd] for e in fst if snd in e)
+					ret=chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e)))
+					#print list(chain(*(type(x) in ITER_TYPES and x or [x] for x in (e[snd] for e in fst if snd in e))))
+					if D: self.debug(color.op("..")+" returning %s",color.bold(ret))
+					return ret
 			elif op=="[":
 				len_node=len(node)
 				# TODO move it to tree generation phase
@@ -317,8 +319,9 @@ def exe(node):
 						return fst
 					selector=node[2]
 					if D: self.debug("found '%s' selector. executing on %s", color.bold(selector),color.bold(fst))
+					selectorIsTuple=type(selector) is tuple
 
-					if type(selector) is tuple and selector[0] is "[":
+					if selectorIsTuple and selector[0] is "[":
 						nodeList=[]
 						nodeList_append=nodeList.append
 						for i in fst:
@@ -328,49 +331,47 @@ def exe(node):
 						if D: self.debug("returning %s objects: %s", color.bold(len(nodeList)),color.bold(nodeList))
 						return nodeList
 
-					#if type(selector) is tuple and selector[0]=="fn":
-					#	for i in fst:
-
-					if type(selector) is tuple and selector[0] == "(current)":
+					if selectorIsTuple and selector[0] == "(current)":
 						if D: self.warning(color.bold("$.*[@]")+" is eqivalent to "+color.bold("$.*")+"!")
 						return fst
 
-					if type(selector) is tuple and selector[0] in SELECTOR_OPS:
+					if selectorIsTuple and selector[0] in SELECTOR_OPS:
 						if D: self.debug("found %s operator in selector", color.bold(selector[0]))
 						if type(fst) is dict:
 							fst=[fst]
-							# TODO move it to tree building phase
+						# TODO move it to tree building phase
 						if type(selector[1]) is tuple and selector[1][0]=="name":
 							selector=(selector[0],selector[1][1],selector[2])
 						selector0=selector[0]
 						selector1=selector[1]
 						selector2=selector[2]
-						nodeList=[]
-						nodeList_append=nodeList.append
-						for i in fst:
-							if D: self.debug("setting self.current to %s", color.bold(i))
-							self.current=i
-							if selector0=="fn":
-								nodeList_append(exe(selector))
-							elif type(selector1) in STR_TYPES:
-								try:
-									if exe((selector0,i[selector1],selector2)):
-										nodeList_append(i)
-										if D: self.debug("appended")
-									if D: self.debug("discarded")
-								except Exception as e:
-									if D: self.debug("discarded, Exception: %s",color.bold(e))
-							else:
-								try:
-									# TODO optimize an event when @ is not used. exe(selector1) can be cached
-									if exe((selector0,exe(selector1),exe(selector2))):
-										nodeList_append(i)
-										if D: self.debug("appended")
-									if D: self.debug("discarded")
-								except Exception:
-									if D: self.debug("discarded")
+
+						def exeSelector(fst):
+							for i in fst:
+								if D: self.debug("setting self.current to %s", color.bold(i))
+								self.current=i
+								if selector0=="fn":
+									yield exe(selector)
+								elif type(selector1) in STR_TYPES:
+									try:
+										if exe((selector0,i[selector1],selector2)):
+											yield i
+											if D: self.debug("appended")
+										if D: self.debug("discarded")
+									except Exception as e:
+										if D: self.debug("discarded, Exception: %s",color.bold(e))
+								else:
+									try:
+										# TODO optimize an event when @ is not used. exe(selector1) can be cached
+										if exe((selector0,exe(selector1),exe(selector2))):
+											yield i
+											if D: self.debug("appended")
+										if D: self.debug("discarded")
+									except Exception:
+										if D: self.debug("discarded")
+
 						if D: self.debug("returning '%s' objects: '%s'", color.bold(len(nodeList)), color.bold(nodeList))
-						return nodeList
+						return exeSelector(fst)
 					self.current=fst
 					snd=exe(node[2])
 					typefst=type(fst)

objectpath/shell.py

@@ -10,16 +10,17 @@
 # this is to prevent various tools from deleting import readline
 ___x=readline.__doc__
 
-from objectpath import Tree
+from objectpath import Tree, ITER_TYPES
 from objectpath.utils.colorify import * # pylint: disable=W0614
-from objectpath.utils import json_compat as json
-from objectpath.utils.json_compat import printJSON
+from objectpath.utils import json_ext as json
+from objectpath.utils.json_ext import printJSON
 
 def main():
 	parser=argparse.ArgumentParser(description='Command line options')
 	parser.add_argument('-u', '--url', dest='URL', help='URL containing JSON document.')
 	#parser.add_argument('-xml', dest='xml', help='[EXPERIMENTAL] Expect XML input.',action='store_true')
 	parser.add_argument('-d', '--debug', dest='debug', help='Debbuging on/off.', action='store_true')
+	parser.add_argument('-p', '--profile', dest='profile', help='Profiling on/off.', action='store_true')
 	parser.add_argument('-e', '--expr', dest='expr', help='Expression/query to execute on file, print on stdout and exit.')
 	parser.add_argument('file', metavar='FILE', nargs="?", help='File name')
 
@@ -31,6 +32,11 @@ def main():
 
 	if args.debug:
 		a["debug"]=True
+	if args.profile:
+		try:
+			from guppy import hpy
+		except:
+			pass
 	File=args.file
 	#if args.xml:
 	#	from utils.xmlextras import xml2tree
@@ -61,23 +67,48 @@ def main():
 		if not expr: print(" "+bold("done")+".")
 
 	if expr:
-		print(json.dumps(tree.execute(expr)))
-		exit()
+		if args.profile:
+			import cProfile, pstats, StringIO
+			pr = cProfile.Profile()
+			pr.enable()
+		ret=tree.execute(expr)
+		if type(ret) in ITER_TYPES:
+			ret=list(ret)
+		print(json.dumps(ret))
+		if args.profile:
+			pr.disable()
+			s = StringIO.StringIO()
+			sortby = 'cumulative'
+			ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
+			ps.print_stats()
+			print s.getvalue()
+		return
 
 	try:
 		while True:
+			limitResult=5
 			try:
-				if sys.version_info.major >= 3:
-						r=tree.execute(input(">>> "))
+				if sys.version >= '3':
+					r=input(">>> ")
 				else:
-						r=tree.execute(raw_input(">>> "))
+					r=raw_input(">>> ")
+
+				if r.startswith("all"):
+					limitResult=-1
+					r=tree.execute(r[3:].strip())
+				else:
+					r=tree.execute(r)
+
 				# python 3 raises error here - unicode is not a proper type there
 				try:
 					if type(r) is unicode:
 						r=r.encode("utf8")
 				except NameError:
 					pass
-				print(printJSON(r))
+				print(printJSON(r,length=limitResult))
+				if args.profile:
+					h = hpy()
+					print h.heap()
 			except Exception as e:
 				print(e)
 	except KeyboardInterrupt:

objectpath/utils/json_ext.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python
 try:
-	import json
+	import simplejson as json
 except ImportError:
 	try:
-		import simplejson as json
+		import json
 	except ImportError:
 		raise Exception("JSONNotFound")
 try:
@@ -92,8 +92,11 @@ def rec(o):
 				for i in o[0:length]:
 					rec(i)
 					out(",\n")
+				if length is -1:
+					rec(o[-1])
+					out(",\n")
 
-				if len(o)>length:
+				if length is not -1 and len(o)>length:
 					out("... ("+str(len(o)-length)+" more items)\n")
 				else:
 					ret.pop()

shell.py

@@ -4,3 +4,4 @@
 from objectpath import shell
 
 shell.main()
+

tests/test_ObjectPath.py

@@ -73,17 +73,22 @@
 
 def execute_raw(expr):
 	return tree1.execute(expr)
+TYPES=[generator,chain]
+if sys.version_info.major>2:
+	TYPES+=[map]
+
+TYPES=tuple(TYPES)
 
 def execute(expr):
 	r=tree1.execute(expr)
-	if isinstance(r, generator):
+	if isinstance(r, TYPES):
 		return list(r)
 	else:
 		return r
 
 def execute2(expr):
 	r=tree2.execute(expr)
-	if isinstance(r, generator):
+	if isinstance(r, TYPES):
 		return list(r)
 	else:
 		return r
@@ -283,12 +288,12 @@ def test_builtin_string(self):
 		self.assertEqual(execute("join(['aaą','aaę'],'ć')"),"aaąćaaę")
 		self.assertEqual(execute("join(['aaa','aaa'])"),"aaaaaa")
 		self.assertEqual(execute("join(['aaa','aaa',3,55])"),"aaaaaa355")
-		self.assertEqual(list(execute("map(upper,['aaa','aaa'])")),["AAA","AAA"])
+		self.assertEqual(execute("map(upper,['aaa','aaa'])"),["AAA","AAA"])
 
 	def test_builtin_arrays(self):
 		self.assertEqual(execute("sort([1,2,3,4]+[2,4])"), [1,2,2,3,4,4])
 		self.assertEqual(execute("sort($.._id)"), [1,2,3,4])
-		self.assertEqual(list(execute("sort($..l[0].*, _id)")), [{'_id': 3, 'aaa': 'ddd', 'false': 2}, {'_id': 4}])
+		self.assertEqual(execute("sort($..l.*, _id)"), [{'_id': 3, 'aaa': 'ddd', 'false': 2}, {'_id': 4}])
 		self.assertEqual(execute("reverse([1,2,3,4]+[2,4])"), [4,2,4,3,2,1])
 		self.assertEqual(execute("reverse(sort($.._id))"), [4,3,2,1])
 		self.assertEqual(execute("len([1,2,3,4]+[2,4])"), 6)
@@ -374,12 +379,12 @@ def test_simple_paths(self):
 		self.assertEqual(execute("$.*['test'][0].o._id"), 2)
 		self.assertEqual(execute('[1,"aa",{"a":2,"c":3},{"c":3},{"a":1,"b":2}].(a,b)'), [{"a":2},{"a":1,"b":2}])
 		self.assertEqual(execute2("$.store.book.(price,title)[0]"), {"price": 8.95, "title": "Sayings of the Century"})
-		self.assertEqual(execute2("$..book[0].(price,title)[0]"), {"price": 8.95, "title": "Sayings of the Century"})
+		self.assertEqual(execute2("$..book.(price,title)"), [{'price': 8.95, 'title': 'Sayings of the Century'}, {'price': 12.99, 'title': 'Sword of Honour'}, {'price': 8.99, 'title': 'Moby Dick'}, {'price': 22.99, 'title': 'The Lord of the Rings'}])
 		self.assertIsInstance(execute("now().year"),int)
 
 	def test_complex_paths(self):
 		self.assertEqual(sorted(execute("$.._id")), [1, 2, 3, 4])
-		self.assertEqual(execute("$..l"), [object1["test"]["l"]])
+		self.assertEqual(execute("$..l"), object1["test"]["l"])
 		self.assertEqual(execute("$..l.._id"), [3,4])
 		self.assertEqual(execute2("$.store.*"), [object2["store"]])
 		self.assertEqual(execute2("$.store.book.author"), ['Nigel Rees', 'Evelyn Waugh', 'Herman Melville', 'J. R. R. Tolkien'])