Merge pull request #240 from vertexproject/visi-dev

several workflow features and a critical fix to storm syntax and prop normalization

synapse/cmds/cortex.py

@@ -31,6 +31,12 @@ def runCmdOpts(self, opts):
         core = self.getCmdItem()
         resp = core.ask(ques)
 
+        oplog = resp.get('oplog')
+
+        # check for an error condition
+        if oplog and oplog[-1].get('excinfo'):
+            opts['debug'] = 1
+
         if opts.get('debug'):
 
             self.printf('oplog:')
@@ -39,33 +45,52 @@ def runCmdOpts(self, opts):
                 took = opfo.get('took')
                 self.printf('    %s (took:%d) %r' % (mnem,took,opfo))
 
+            self.printf('')
+
             self.printf('options:')
             for name,valu in sorted(resp.get('options').items()):
                 self.printf('    %s = %s' % (name,valu))
 
+            self.printf('')
+
             self.printf('limits:')
             for name,valu in sorted(resp.get('limits').items()):
                 self.printf('    %s = %s' % (name,valu))
 
+            self.printf('')
+
         def nodevalu(t):
             return repr( t[1].get( t[1].get('tufo:form') ) )
 
         nodes = list(sorted( resp.get('data'), key=nodevalu))
 
+        if len(nodes) == 0:
+            self.printf('(0 results)')
+            return
+
+        forms = set([ node[1].get('tufo:form') for node in nodes ])
+
+        fsize = max([ len(f) for f in forms ])
+
         for node in nodes:
             form = node[1].get('tufo:form')
             valu = node[1].get(form)
 
-            tags = s_tufo.tags(node)
+            tags = sorted(s_tufo.tags(node,leaf=True))
+            tags = [ '#'+tag for tag in tags ]
 
             # FIXME local typelib and datamodel
             disp = core.getPropRepr(form,valu)
-            self.printf('%s - %s' % (disp,','.join(tags)))
+            self.printf('%s = %s - %s' % (form.ljust(fsize),disp,' '.join(tags)))
             if opts.get('props'):
-                props = list(s_tufo.props(node).items())
-                for prop,valu in sorted(props):
+                pref = form + ':'
+                flen = len(form)
+                for prop in sorted([ k for k in node[1].keys() if k.startswith(pref) ]):
+                    valu = node[1].get(prop)
                     disp = core.getPropRepr(prop,valu)
-                    self.printf('    %s = %s' % (prop,disp))
+                    self.printf('    %s = %s' % (prop[flen:],disp))
+
+        self.printf('(%d results)' % (len(nodes),))
 
         return resp
 
@@ -87,6 +112,7 @@ class AddNodeCmd(s_cli.Cmd):
 
     _cmd_name = 'addnode'
     _cmd_syntax = (
+        ('--tags',{'type':'valu'}),
         ('prop',{'type':'valu'}),
         ('valu',{'type':'valu'}),
         ('props',{'type':'kwlist'}),
@@ -100,12 +126,21 @@ def runCmdOpts(self, opts):
             self.printf(self.__doc__)
             return
 
+        tags = ()
+
+        tstr = opts.get('tags')
+        if tstr != None:
+            tags = tstr.split(',')
+
         kwlist = opts.get('props')
         props = dict( opts.get('props') )
 
         core = self.getCmdItem()
 
         node = core.formTufoByFrob(prop,valu,**props)
+        if tags:
+            node = core.addTufoTags(node,tags)
+
         self.printf('formed: %r' % (node,))
 
 class AddTagCmd(s_cli.Cmd):

synapse/cores/common.py

@@ -1465,13 +1465,16 @@ def getTufosByProp(self, prop, valu=None, mintime=None, maxtime=None, limit=None
                 dostuff(tufo)
 
         '''
+        norm = None
         if valu != None:
-            valu,subs = self.getPropNorm(prop,valu)
+            norm,subs = self.getPropNorm(prop,valu)
+            if norm == None:
+                raise BadPropValu(prop=prop,valu=valu)
 
         if self.caching and mintime == None and maxtime == None:
-            return self._getTufosByCache(prop,valu,limit)
+            return self._getTufosByCache(prop,norm,limit)
 
-        return self._getTufosByProp(prop, valu=valu, mintime=mintime, maxtime=maxtime, limit=limit)
+        return self._getTufosByProp(prop, valu=norm, mintime=mintime, maxtime=maxtime, limit=limit)
 
     def _getTufosByProp(self, prop, valu=None, mintime=None, maxtime=None, limit=None):
         rows = self.getJoinByProp(prop, valu=valu, mintime=mintime, maxtime=maxtime, limit=limit)
@@ -1487,9 +1490,6 @@ def getTufosByFrob(self, prop, valu=None, mintime=None, maxtime=None, limit=None
                 dostuff(tufo)
 
         '''
-        if valu != None:
-            valu,_ = self.getPropFrob(prop,valu)
-
         return self.getTufosByProp(prop, valu=valu, mintime=mintime, maxtime=maxtime, limit=limit)
 
     def getTufosByPropType(self, name, valu=None, mintime=None, maxtime=None, limit=None):
@@ -1533,6 +1533,7 @@ def addTufoTags(self, tufo, tags, asof=None):
         '''
         with self.getCoreXact():
             [ self.addTufoTag(tufo,tag,asof=asof) for tag in tags ]
+            return tufo
 
     def addTufoTag(self, tufo, tag, asof=None):
         '''
@@ -2529,7 +2530,6 @@ def _stormOperDset(self, query, oper):
 
     # some helpers to allow *all* queries to be processed via getTufosBy()
     def _tufosByEq(self, prop, valu, limit=None):
-        valu,_ = self.getPropFrob(prop,valu)
         return self.getTufosByProp(prop,valu=valu,limit=limit)
 
     def _tufosByHas(self, prop, valu, limit=None):

synapse/cores/ram.py

@@ -54,13 +54,20 @@ def _tufosByLe(self, prop, valu, limit=None):
         return self.getTufosByIdens([ r[0] for r in rows ])
 
     def _sizeByRange(self, prop, valu, limit=None):
-        return sum( 1 for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= valu[0] and r[2] < valu[1] )
+        minval = int(valu[0])
+        maxval = int(valu[1])
+        return sum( 1 for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= minval and r[2] < maxval )
 
     def _rowsByRange(self, prop, valu, limit=None):
+        minval = int(valu[0])
+        maxval = int(valu[1])
+
         # HACK: for speed
-        ret = [ r for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= valu[0] and r[2] < valu[1] ]
+        ret = [ r for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= minval and r[2] < maxval ]
+
         if limit != None:
             ret = ret[:limit]
+
         return ret
 
     def _sizeByGe(self, prop, valu, limit=None):

synapse/cores/sqlite.py

@@ -237,9 +237,11 @@ def _rowsByRange(self, prop, valu, limit=None):
         limit = self._getDbLimit(limit)
 
         q = self._q_getrows_by_range
-        args = [ prop, valu[0], valu[1], limit ]
 
-        rows = self.select(q, prop=prop, minvalu=valu[0], maxvalu=valu[1], limit=limit)
+        minvalu = int(valu[0])
+        maxvalu = int(valu[1])
+
+        rows = self.select(q, prop=prop, minvalu=minvalu, maxvalu=maxvalu, limit=limit)
         return self._foldTypeCols(rows)
 
     def _rowsByGe(self, prop, valu, limit=None):

synapse/datamodel.py

@@ -272,6 +272,18 @@ def _addSubRefs(self, pdef):
                 continue
             self.subprops[prop].append(pdef)
 
+    def getPropsByType(self, name):
+        '''
+        Return a list of prop def tuples (name,info) for all props of the given type.
+
+        Example:
+
+            for prop,info in modl.getPropsByType('guid'):
+                dostuff()
+
+        '''
+        return self.propsbytype.get(name,())
+
     def _addPropGlob(self, form, prop, **info):
         prop = '%s:%s' % (form,prop)
         info['form'] = form