From 5ecbf19f14fbc8fe8a53718130d139a286e4ad10 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Sun, 14 May 2017 17:23:51 -0400 Subject: [PATCH 01/18] added alltag() oper and macro. added expand() oper --- synapse/cmds/cortex.py | 24 ++++++++++--- synapse/lib/storm.py | 79 +++++++++++++++++++++++++++++++++++++++--- synapse/lib/syntax.py | 10 +++++- 3 files changed, 103 insertions(+), 10 deletions(-) diff --git a/synapse/cmds/cortex.py b/synapse/cmds/cortex.py index 73c4fccd89..ba0e4b2ed2 100644 --- a/synapse/cmds/cortex.py +++ b/synapse/cmds/cortex.py @@ -31,6 +31,12 @@ def runCmdOpts(self, opts): core = self.getCmdItem() resp = core.ask(ques) + oplog = resp.get('oplog') + + # check for an error condition + if oplog and oplog[-1].get('excinfo'): + opts['debug'] = 1 + if opts.get('debug'): self.printf('oplog:') @@ -52,6 +58,14 @@ def nodevalu(t): nodes = list(sorted( resp.get('data'), key=nodevalu)) + if len(nodes) == 0: + self.printf('(no results)') + return + + forms = set([ node[1].get('tufo:form') for node in nodes ]) + + fsize = max([ len(f) for f in forms ]) + for node in nodes: form = node[1].get('tufo:form') valu = node[1].get(form) @@ -60,12 +74,14 @@ def nodevalu(t): # FIXME local typelib and datamodel disp = core.getPropRepr(form,valu) - self.printf('%s - %s' % (disp,','.join(tags))) + self.printf('%s = %s - %s' % (form.ljust(fsize),disp,','.join(tags))) if opts.get('props'): - props = list(s_tufo.props(node).items()) - for prop,valu in sorted(props): + pref = form + ':' + flen = len(form) + for prop in sorted([ k for k in node[1].keys() if k.startswith(pref) ]): + valu = node[1].get(prop) disp = core.getPropRepr(prop,valu) - self.printf(' %s = %s' % (prop,disp)) + self.printf(' %s = %s' % (prop[flen:],disp)) return resp diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index 96960ae392..e102fb4121 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -36,7 +36,7 @@ def __exit__(self, exc, cls, tb): } if exc != None: - info.update( excinfo(exc) ) + info['excinfo'] = excinfo(exc) self.query.clear() self.query.log(**info) @@ -217,10 +217,12 @@ def __init__(self, **opts): self.setOperFunc('join', self._stormOperJoin) self.setOperFunc('lift', self._stormOperLift) self.setOperFunc('pivot', self._stormOperPivot) - self.setOperFunc('addtag',self._stormOperAddTag) - self.setOperFunc('deltag',self._stormOperDelTag) - self.setOperFunc('nexttag',self._stormOperNextSeq) - self.setOperFunc('setprop',self._stormOperSetProp) + self.setOperFunc('expand', self._stormOperExpand) + self.setOperFunc('alltag', self._stormOperAllTag) + self.setOperFunc('addtag', self._stormOperAddTag) + self.setOperFunc('deltag', self._stormOperDelTag) + self.setOperFunc('nexttag', self._stormOperNextSeq) + self.setOperFunc('setprop', self._stormOperSetProp) def getStormCore(self, name=None): ''' @@ -644,6 +646,73 @@ def _stormOperSetProp(self, query, oper): [ core.setTufoProps(node,**props) for node in query.data() ] + def _iterPropTags(self, props, tags): + for prop in props: + for tag in tags: + yield prop,tag + + def _stormOperAllTag(self, query, oper): + + tags = oper[1].get('args') + opts = dict(oper[1].get('kwlist')) + + limit = opts.get('limit') + + core = self.getStormCore() + forms = core.getTufoForms() + + for form,tag in self._iterPropTags(forms,tags): + nodes = core.getTufosByTag(form,tag,limit=limit) + + for node in nodes: + query.add(node) + + if limit != None: + limit -= len(nodes) + if limit <= 0: + break + + def _stormOperExpand(self, query, oper): + + args = oper[1].get('args') + opts = dict(oper[1].get('kwlist')) + + # total node lift limit + limit = opts.get('limit') + + # how many degress of expansion? + degs = int(opts.get('degrees',1)) + + done = set() + todo = collections.deque([ (0,node) for node in query.data() ]) + + core = self.getStormCore() + + while len(todo): + + if limit != None and limit <= 0: + # TODO: plumb a way to communicate this in the result? + break + + deg,node = todo.popleft() + if node[0] in done: + continue + + done.add(node[0]) + if deg >= degs: + continue + + form = node[1].get('tufo:form') + valu = node[1].get(form) + + nods = core.getTufosByPropType(form, valu, limit=limit) + for newn in nods: + query.add(newn) + todo.append( (deg+1, newn) ) + + if limit != None: + limit -= len(newn) + def _stormOperAddTag(self, query, oper): tags = oper[1].get('args') diff --git a/synapse/lib/syntax.py b/synapse/lib/syntax.py index cf134df69f..1b2fb97ba3 100644 --- a/synapse/lib/syntax.py +++ b/synapse/lib/syntax.py @@ -468,7 +468,7 @@ def parse_oper(text, off=0): if nextchar(text,off,','): off += 1 - + def parse(text, off=0): ''' Parse and return a set of instruction tufos. @@ -492,6 +492,14 @@ def parse(text, off=0): ret.append(inst) continue + # lift by tag alone macro + if nextstr(text,off,'#'): + _,off = nom(text,off+1,whites) + name,off = nom(text,off,varset) + inst = ('alltag',{'args':[name],'kwlist':[]}) + ret.append(inst) + continue + # must() macro syntax: +foo:bar="woot" if nextchar(text,off,'+'): inst,off = parse_macro_filt(text,off+1,mode='must') From 3b411478a5d75b3618780cbe93a2f52923d1a1e3 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Sun, 14 May 2017 17:35:04 -0400 Subject: [PATCH 02/18] added unit tests for new storm operators --- synapse/tests/test_lib_storm.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index f239499b18..a0df1f30e2 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -43,6 +43,7 @@ def test_storm_pivot(self): def test_storm_setprop(self): with s_cortex.openurl('ram:///') as core: + core.setConfOpt('enforce',1) node = core.formTufoByProp('inet:fqdn','vertex.link') @@ -52,6 +53,38 @@ def test_storm_setprop(self): self.eq( node[1].get('inet:fqdn:created'), 1462406400000 ) self.eq( node[1].get('inet:fqdn:updated'), 1493942400000 ) + def test_storm_expand(self): + + with s_cortex.openurl('ram:///') as core: + core.setConfOpt('enforce',1) + + iden = guid() + node = core.formTufoByProp('inet:dns:a','vertex.link/1.2.3.4') + + nodes = core.eval('inet:fqdn=vertex.link expand()') + + forms = list(sorted([ n[1].get('tufo:form') for n in nodes ])) + + self.eq( forms, ['inet:dns:a','inet:fqdn'] ) + + def test_storm_alltag(self): + + with s_cortex.openurl('ram:///') as core: + core.setConfOpt('enforce',1) + + iden = guid() + node = core.formTufoByProp('inet:fqdn','vertex.link') + + core.addTufoTag(node,'foo.bar') + core.addTufoTag(node,'baz.faz') + + node = core.eval('#foo.bar')[0] + + self.eq( node[1].get('inet:fqdn'), 'vertex.link' ) + + self.nn( node[1].get('*|inet:fqdn|baz') ) + self.nn( node[1].get('*|inet:fqdn|foo.bar') ) + def test_storm_addtag(self): with s_cortex.openurl('ram:///') as core: From be89a2cc65567d958bb413e8e6fe3a002ba8c49b Mon Sep 17 00:00:00 2001 From: invisig0th Date: Sun, 14 May 2017 18:38:47 -0400 Subject: [PATCH 03/18] fixed regex filter operator for relative props. more todo --- synapse/lib/storm.py | 13 ++++++++++++- synapse/tests/test_lib_storm.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index e102fb4121..48b597498e 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -454,10 +454,21 @@ def cmpr(tufo): def _cmprCtorRe(self, oper): prop = oper[1].get('prop') + + isrel = prop.startswith(':') reobj = re.compile( oper[1].get('valu') ) def cmpr(tufo): - return reobj.search(tufo[1].get(prop)) != None + + full = prop + if isrel: + full = tufo[1].get('tufo:form') + prop + + valu = tufo[1].get(full) + if valu == None: + return False + + return reobj.search(valu) != None return cmpr diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index a0df1f30e2..baeb5ef4f3 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -67,6 +67,22 @@ def test_storm_expand(self): self.eq( forms, ['inet:dns:a','inet:fqdn'] ) + def test_storm_filt_regex(self): + + with s_cortex.openurl('ram:///') as core: + core.setConfOpt('enforce',1) + + iden0 = guid() + iden1 = guid() + iden2 = guid() + + node0 = core.formTufoByProp('file:bytes', iden0) + node1 = core.formTufoByProp('file:bytes', iden1, name='woot.exe') + node2 = core.formTufoByProp('file:bytes', iden2, name='toow.exe') + + nodes = core.eval('file:bytes +:name~=exe') + self.eq( len(nodes), 2 ) + def test_storm_alltag(self): with s_cortex.openurl('ram:///') as core: From a3858089c6f5463d61f75bcf9334ee23495035b5 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 06:21:15 -0400 Subject: [PATCH 04/18] fix addTufoTags to return the tufo --- synapse/cores/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/cores/common.py b/synapse/cores/common.py index 9a025c5bb5..9e8eefe48a 100644 --- a/synapse/cores/common.py +++ b/synapse/cores/common.py @@ -1533,6 +1533,7 @@ def addTufoTags(self, tufo, tags, asof=None): ''' with self.getCoreXact(): [ self.addTufoTag(tufo,tag,asof=asof) for tag in tags ] + return tufo def addTufoTag(self, tufo, tag, asof=None): ''' From ec854872f3d950b701c3887cba67839fc6949c9d Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 06:22:53 -0400 Subject: [PATCH 05/18] make ask a bit prittier and add --tags option to addnode --- synapse/cmds/cortex.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/synapse/cmds/cortex.py b/synapse/cmds/cortex.py index ba0e4b2ed2..f30a77145d 100644 --- a/synapse/cmds/cortex.py +++ b/synapse/cmds/cortex.py @@ -45,21 +45,27 @@ def runCmdOpts(self, opts): took = opfo.get('took') self.printf(' %s (took:%d) %r' % (mnem,took,opfo)) + self.printf('') + self.printf('options:') for name,valu in sorted(resp.get('options').items()): self.printf(' %s = %s' % (name,valu)) + self.printf('') + self.printf('limits:') for name,valu in sorted(resp.get('limits').items()): self.printf(' %s = %s' % (name,valu)) + self.printf('') + def nodevalu(t): return repr( t[1].get( t[1].get('tufo:form') ) ) nodes = list(sorted( resp.get('data'), key=nodevalu)) if len(nodes) == 0: - self.printf('(no results)') + self.printf('(0 results)') return forms = set([ node[1].get('tufo:form') for node in nodes ]) @@ -70,11 +76,12 @@ def nodevalu(t): form = node[1].get('tufo:form') valu = node[1].get(form) - tags = s_tufo.tags(node) + tags = sorted(s_tufo.tags(node,leaf=True)) + tags = [ '#'+tag for tag in tags ] # FIXME local typelib and datamodel disp = core.getPropRepr(form,valu) - self.printf('%s = %s - %s' % (form.ljust(fsize),disp,','.join(tags))) + self.printf('%s = %s - %s' % (form.ljust(fsize),disp,' '.join(tags))) if opts.get('props'): pref = form + ':' flen = len(form) @@ -83,6 +90,8 @@ def nodevalu(t): disp = core.getPropRepr(prop,valu) self.printf(' %s = %s' % (prop[flen:],disp)) + self.printf('(%d results)' % (len(nodes),)) + return resp class AddNodeCmd(s_cli.Cmd): @@ -103,6 +112,7 @@ class AddNodeCmd(s_cli.Cmd): _cmd_name = 'addnode' _cmd_syntax = ( + ('--tags',{'type':'valu'}), ('prop',{'type':'valu'}), ('valu',{'type':'valu'}), ('props',{'type':'kwlist'}), @@ -116,12 +126,21 @@ def runCmdOpts(self, opts): self.printf(self.__doc__) return + tags = () + + tstr = opts.get('tags') + if tstr != None: + tags = tstr.split(',') + kwlist = opts.get('props') props = dict( opts.get('props') ) core = self.getCmdItem() node = core.formTufoByFrob(prop,valu,**props) + if tags: + node = core.addTufoTags(node,tags) + self.printf('formed: %r' % (node,)) class AddTagCmd(s_cli.Cmd): From 477d114d718f007bbd9240679e9b1fe222856c5d Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 06:23:44 -0400 Subject: [PATCH 06/18] added leaf only tags() option --- synapse/lib/tufo.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/synapse/lib/tufo.py b/synapse/lib/tufo.py index dd59678404..90db14a3ca 100644 --- a/synapse/lib/tufo.py +++ b/synapse/lib/tufo.py @@ -39,8 +39,23 @@ def props(tufo,pref=None): plen = len(pref) return { p[plen:]:v for (p,v) in tufo[1].items() if p.startswith(pref) } -def tags(tufo): - return [ p.split('|',2)[2] for p in tufo[1].keys() if p.startswith('*') ] +def tags(tufo,leaf=False): + + fulltags = [ p.split('|',2)[2] for p in tufo[1].keys() if p.startswith('*|') ] + if not leaf: + return fulltags + + # longest first + retn = [] + + # brute force rather than build a tree. faster in small sets. + for size,tag in sorted([ (len(t),t) for t in fulltags ], reverse=True): + look = tag + '.' + if any([ r.startswith(look) for r in retn]): + continue + retn.append(tag) + + return retn def equal(tuf0,tuf1): ''' From f070ea57d00d333615f158b051fb6126af450f6e Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 06:25:37 -0400 Subject: [PATCH 07/18] added seen and range filters as well as addxref() operator --- synapse/lib/storm.py | 89 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index 48b597498e..86b52a3297 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -5,6 +5,7 @@ import synapse.eventbus as s_eventbus +import synapse.lib.cache as s_cache import synapse.lib.scope as s_scope import synapse.lib.syntax as s_syntax import synapse.lib.threads as s_threads @@ -202,6 +203,8 @@ def __init__(self, **opts): self.setCmprCtor('or', self._cmprCtorOr ) self.setCmprCtor('and', self._cmprCtorAnd ) self.setCmprCtor('tag', self._cmprCtorTag ) + self.setCmprCtor('seen', self._cmprCtorSeen) + self.setCmprCtor('range', self._cmprCtorRange) self.setCmprCtor('in', self._cmprCtorIn ) self.setCmprCtor('re', self._cmprCtorRe ) @@ -224,6 +227,8 @@ def __init__(self, **opts): self.setOperFunc('nexttag', self._stormOperNextSeq) self.setOperFunc('setprop', self._stormOperSetProp) + self.setOperFunc('addxref', self._stormOperAddXref) + def getStormCore(self, name=None): ''' Return the (optionally named) cortex for use in direct storm @@ -514,6 +519,74 @@ def cmpr(tufo): return cmpr + def _cmprCtorSeen(self, oper): + + args = [ str(v) for v in oper[1].get('args') ] + + core = self.getStormCore() + + vals = [ core.getTypeNorm('time',t)[0] for t in args ] + + seenprops = {} + def getseen(form): + stup = seenprops.get(form) + if stup == None: + stup = seenprops[form] = (form+':seen:min',form+':seen:max') + return stup + + def cmpr(tufo): + + form = tufo[1].get('tufo:form') + + minprop,maxprop = getseen(form) + + smin = tufo[1].get(minprop) + smax = tufo[1].get(maxprop) + + if smin == None or smax == None: + return False + + for valu in vals: + if valu >= smin and valu <= smax: + return True + + return cmpr + + def _cmprCtorRange(self, oper): + + prop = self._reqOperArg(oper,'prop') + valu = self._reqOperArg(oper,'valu') + + #TODO unified syntax plumbing with in-band help + + core = self.getStormCore() + isrel = prop.startswith(':') + + def initMinMax(key): + xmin,_ = core.getPropNorm(key,valu[0]) + xmax,_ = core.getPropNorm(key,valu[1]) + return int(xmin),int(xmax) + + norms = s_cache.KeyCache( initMinMax ) + + def cmpr(tufo): + + full = prop + if isrel: + form = tufo[1].get('tufo:form') + full = form + prop + + valu = tufo[1].get(full) + if valu == None: + return False + + minval,maxval = norms.get(full) + + return valu >= minval and valu <= maxval + + return cmpr + + def _stormOperAnd(self, query, oper): funcs = [ self.getCmprFunc(op) for op in oper[1].get('args') ] for tufo in query.take(): @@ -649,6 +722,22 @@ def _stormOperJoin(self, query, oper): for tufo in self.stormTufosBy('in', dstp, vals, limit=opts.get('limit') ): query.add(tufo) + def _stormOperAddXref(self, query, oper): + + args = oper[1].get('args') + if len(args) != 3: + raise SyntaxError('addxref(,
,)') + + xref,form,valu = args + + core = self.getStormCore() + + # TODO clearer error handling + for node in query.take(): + sorc = node[1].get( node[1].get('tufo:form') ) + node = core.formTufoByProp(xref,(sorc,form,valu)) + query.add(node) + def _stormOperSetProp(self, query, oper): args = oper[1].get('args') props = dict( oper[1].get('kwlist') ) From 20267f46a2308f880fba353a74c3354b8ff5b0ee Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 06:26:26 -0400 Subject: [PATCH 08/18] making quote requirements more friendly in storm syntax --- synapse/lib/syntax.py | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/synapse/lib/syntax.py b/synapse/lib/syntax.py index 1b2fb97ba3..6f6c5332a2 100644 --- a/synapse/lib/syntax.py +++ b/synapse/lib/syntax.py @@ -54,7 +54,7 @@ def is_literal(text,off): def parse_literal(text, off,trim=True): if text[off] == '(': - return parse_list(text,off,trim=trim) + return parse_cmd_list(text,off,trim=trim) if text[off] == '"': return parse_string(text,off,trim=trim) @@ -96,7 +96,7 @@ def isquote(text,off): def parse_cmd_list(text,off=0,trim=True): ''' - Parse a list (likely for comp type ) coming from a command line input. + Parse a list (likely for comp type) coming from a command line input. The string elements within the list may optionally be quoted. ''' @@ -188,6 +188,19 @@ def parse_macro_filt(text,off=0,trim=True, mode='must'): oper = ('filt',{'cmp':'tag','mode':mode,'valu':tag}) return oper,off + # check for non-macro syntax + name,xoff = nom(text,off,varset) + _,xoff = nom(text,xoff,whites) + if nextchar(text,xoff,'('): + inst,off = parse_oper(text,off) + + opfo = {'cmp':inst[0],'mode':mode} + + opfo['args'] = inst[1].get('args',()) + opfo['kwlist'] = inst[1].get('kwlist',()) + + return ('filt',opfo),off + ques,off = parse_ques(text,off,trim=trim) ques['mode'] = mode return ('filt',ques),off @@ -340,12 +353,14 @@ def parse_ques(text,off=0,trim=True): if text[off] != '=': raise SyntaxError(text=text, off=off, mesg='expected equals for by syntax') - ques['valu'],off = parse_oarg(text,off+1) + #ques['valu'],off = parse_oarg(text,off+1) + ques['valu'],off = parse_macro_valu(text,off+1) return ques,off if text[off] == '=': ques['cmp'] = 'eq' - ques['valu'],off = parse_oarg(text,off+1) + #ques['valu'],off = parse_oarg(text,off+1) + ques['valu'],off = parse_macro_valu(text,off+1) break # TODO: handle "by" syntax @@ -362,6 +377,20 @@ def parse_ques(text,off=0,trim=True): return ques,off +def parse_macro_valu(text,off=0): + ''' + Special syntax for the right side of equals in a macro + ''' + if nextchar(text,off,'('): + return parse_cmd_list(text,off) + + if isquote(text,off): + return parse_string(text,off) + + # since it's not quoted, we can assume we are white + # whit space bound ( only during macro syntax ) + return meh(text,off,whites) + def parse_when(text,off,trim=True): whenstr,off = nom(text,off,whenset) # FIXME validate syntax From 9606bb7f5b65d5992db59f87a5c99408019a45d5 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 06:48:44 -0400 Subject: [PATCH 09/18] removed a couple more timemstamp defvals and added inet:netfile --- synapse/models/inet.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/synapse/models/inet.py b/synapse/models/inet.py index 4ffde9e922..0d353ea9fe 100644 --- a/synapse/models/inet.py +++ b/synapse/models/inet.py @@ -53,6 +53,7 @@ def getDataModel(): ('inet:netgroup', {'subof':'sepr','sep':'/','fields':'site,inet:fqdn|name,ou:name','doc':'A group within an online community'}), ('inet:netpost', {'subof':'comp','fields':'netuser,inet:netuser|text,str:txt', 'doc':'A post made by a netuser'}), + ('inet:netfile', {'subof':'comp','fields':'netuser,inet:netuser|file,file:bytes', 'doc':'A file posted by a netuser'}), ('inet:netmemb', {'subof':'comp','fields':'user,inet:netuser|group,inet:netgroup'}), ('inet:follows', {'subof':'comp','fields':'follower,inet:netuser|followee,inet:netuser'}), @@ -189,8 +190,8 @@ def getDataModel(): ('realname',{'ptype':'ps:name'}), ('email',{'ptype':'inet:email'}), ('phone',{'ptype':'tel:phone'}), - ('signup',{'ptype':'time','defval':0,'doc':'The time the netuser account was registered'}), - ('signup:ipv4',{'ptype':'inet:ipv4','defval':0,'doc':'The original ipv4 address used to sign up for the account'}), + ('signup',{'ptype':'time','doc':'The time the netuser account was registered'}), + ('signup:ipv4',{'ptype':'inet:ipv4','doc':'The original ipv4 address used to sign up for the account'}), ('passwd',{'ptype':'inet:passwd','doc':'The current passwd for the netuser account'}), ('seen:min',{'ptype':'time:min'}), ('seen:max',{'ptype':'time:max'}), @@ -255,6 +256,19 @@ def getDataModel(): ]), + ('inet:netfile',{},[ + + ('netuser',{'ptype':'inet:netuser','ro':1}), + ('netuser:site',{'ptype':'inet:fqdn','ro':1}), + ('netuser:user',{'ptype':'inet:user','ro':1}), + + ('file',{'ptype':'file:bytes','ro':1}), + + ('posted',{'ptype':'time'}), + ('seen:min',{'ptype':'time:min'}), + ('seen:max',{'ptype':'time:max'}), + ]), + ('inet:whois:reg',{},[]), ('inet:whois:rar',{},[]), @@ -269,9 +283,9 @@ def getDataModel(): ('fqdn',{'ptype':'inet:fqdn'}), ('asof',{'ptype':'time'}), ('text',{'ptype':'str:lwr'}), - ('created',{'ptype':'time','defval':0,'doc':'The "created" time from the whois record'}), - ('updated',{'ptype':'time','defval':0,'doc':'The "last updated" time from the whois record'}), - ('expires',{'ptype':'time','defval':0,'doc':'The "expires" time from the whois record'}), + ('created',{'ptype':'time','doc':'The "created" time from the whois record'}), + ('updated',{'ptype':'time','doc':'The "last updated" time from the whois record'}), + ('expires',{'ptype':'time','doc':'The "expires" time from the whois record'}), ('registrar',{'ptype':'inet:whois:rar','defval':'??'}), ('registrant',{'ptype':'inet:whois:reg','defval':'??'}), ('ns1',{'ptype':'inet:fqdn'}), From 432aea7935ff3d88cb21565149989ffe009de24f Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 10:08:22 -0400 Subject: [PATCH 10/18] added convention for guid types to generate a new guid on string * --- synapse/lib/types.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/lib/types.py b/synapse/lib/types.py index b9f150a0ec..ee289a3a5c 100644 --- a/synapse/lib/types.py +++ b/synapse/lib/types.py @@ -111,6 +111,10 @@ def norm(self, valu, oldval=None): if not s_compat.isstr(valu) or len(valu) < 1: self._raiseBadValu(valu) + # generate me one. we dont care. + if valu == '*': + return guid(),{} + if valu[0] != '$': retn = valu.lower().replace('-','') if not isguid(retn): From 415ab92e7c38a25bdcd91363b6551ca055cfe5a5 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 14:41:20 -0400 Subject: [PATCH 11/18] added suborg to the model --- synapse/models/orgs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/synapse/models/orgs.py b/synapse/models/orgs.py index f2c4578c98..e4e98df5b5 100644 --- a/synapse/models/orgs.py +++ b/synapse/models/orgs.py @@ -14,6 +14,8 @@ def getDataModel(): ('ou:sic',{'subof':'int','doc':'Standard Industrial Classification Code'}), ('ou:naics',{'subof':'int','doc':'North American Industry Classification System'}), + ('ou:suborg', {'subof':'comp','fields':'org,ou:org|sub,ou:org','doc':'An org which owns a sub org'}), + ('ou:hasfile',{'subof':'comp','fields':'org,ou:org|file,file:bytes'}), ('ou:hasfqdn',{'subof':'comp','fields':'org,ou:org|fqdn,inet:fqdn'}), ('ou:hasipv4',{'subof':'comp','fields':'org,ou:org|ipv4,inet:ipv4'}), @@ -38,6 +40,15 @@ def getDataModel(): ('url',{'ptype':'inet:url'}), ]), + ('ou:suborg',{},[ + ('org',{'ptype':'ou:org','doc':'The org which owns sub'}), + ('sub',{'ptype':'ou:org','doc':'The the sub which is owned by org'}), + ('perc',{'ptype':'int','doc':'The optional percentage of sub which is owned by org'}), + ('current',{'ptype':'bool','defval':1,'doc':'Is the suborg relationship still current'}), + ('seen:min',{'ptype':'time:min','doc':'The optional time the suborg relationship began'}), + ('seen:max',{'ptype':'time:max','doc':'The optional time the suborg relationship ended'}), + ]), + ('ou:user',{},[ ('org',{'ptype':'ou:org'}), ('user',{'ptype':'inet:user'}), From d80bf60204bbf04e9a1c5ff898c70d32fd2cbc25 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 15:05:16 -0400 Subject: [PATCH 12/18] added guid alias for geo:place --- synapse/models/geospace.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/models/geospace.py b/synapse/models/geospace.py index 5541a98738..f2e5b24d7e 100644 --- a/synapse/models/geospace.py +++ b/synapse/models/geospace.py @@ -7,13 +7,14 @@ def getDataModel(): 'version':201611251209, 'types':( - ('geo:place',{'subof':'guid','doc':'A GUID for a specific place'}), - ('geo:latlong',{'subof':'str', 'regex':latlongre, - 'nullval':'??','doc':'A Lat/Long string specifying a point'}), + ('geo:place',{'subof':'guid','alias':'geo:place:alias','doc':'A GUID for a specific place'}), + ('geo:alias',{'subof':'str:lwr','regex':'^[0-9a-z]+$','doc':'An alias for the place GUID','ex':'foobar'}), + ('geo:latlong',{'subof':'str', 'regex':latlongre, 'nullval':'??','doc':'A Lat/Long string specifying a point'}), ), 'forms':( ('geo:place',{'ptype':'geo:place'},[ + ('alias',{'ptype':'geo:alias'}), ('name',{'ptype':'str','lower':1,'doc':'The name of the place'}), ('latlong',{'ptype':'geo:latlong','defval':'??','doc':'The location of the place'}), ]), From 23faad17c1b014433f0f9466dcd05ff892f7c36e Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 20:39:56 -0400 Subject: [PATCH 13/18] emergency fix for frob vs norm issue --- synapse/cores/common.py | 13 ++++++------- synapse/models/inet.py | 6 ++++++ synapse/tests/test_cortex.py | 5 +++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/synapse/cores/common.py b/synapse/cores/common.py index 9e8eefe48a..5ce1bc7239 100644 --- a/synapse/cores/common.py +++ b/synapse/cores/common.py @@ -1465,13 +1465,16 @@ def getTufosByProp(self, prop, valu=None, mintime=None, maxtime=None, limit=None dostuff(tufo) ''' + norm = None if valu != None: - valu,subs = self.getPropNorm(prop,valu) + norm,subs = self.getPropNorm(prop,valu) + if norm == None: + raise BadPropValu(prop=prop,valu=valu) if self.caching and mintime == None and maxtime == None: - return self._getTufosByCache(prop,valu,limit) + return self._getTufosByCache(prop,norm,limit) - return self._getTufosByProp(prop, valu=valu, mintime=mintime, maxtime=maxtime, limit=limit) + return self._getTufosByProp(prop, valu=norm, mintime=mintime, maxtime=maxtime, limit=limit) def _getTufosByProp(self, prop, valu=None, mintime=None, maxtime=None, limit=None): rows = self.getJoinByProp(prop, valu=valu, mintime=mintime, maxtime=maxtime, limit=limit) @@ -1487,9 +1490,6 @@ def getTufosByFrob(self, prop, valu=None, mintime=None, maxtime=None, limit=None dostuff(tufo) ''' - if valu != None: - valu,_ = self.getPropFrob(prop,valu) - return self.getTufosByProp(prop, valu=valu, mintime=mintime, maxtime=maxtime, limit=limit) def getTufosByPropType(self, name, valu=None, mintime=None, maxtime=None, limit=None): @@ -2530,7 +2530,6 @@ def _stormOperDset(self, query, oper): # some helpers to allow *all* queries to be processed via getTufosBy() def _tufosByEq(self, prop, valu, limit=None): - valu,_ = self.getPropFrob(prop,valu) return self.getTufosByProp(prop,valu=valu,limit=limit) def _tufosByHas(self, prop, valu, limit=None): diff --git a/synapse/models/inet.py b/synapse/models/inet.py index 0d353ea9fe..3c73836bb0 100644 --- a/synapse/models/inet.py +++ b/synapse/models/inet.py @@ -264,7 +264,13 @@ def getDataModel(): ('file',{'ptype':'file:bytes','ro':1}), + ('name',{'ptype':'file:base','doc':'The basename of the file in the post'}), + ('posted',{'ptype':'time'}), + + ('ipv4',{'ptype':'inet:ipv4','doc':'The source IPv4 address of the post.'}), + ('ipv6',{'ptype':'inet:ipv6','doc':'The source IPv6 address of the post.'}), + ('seen:min',{'ptype':'time:min'}), ('seen:max',{'ptype':'time:max'}), ]), diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index c9de9d64c8..7409b2c0f3 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -1496,3 +1496,8 @@ def test_cortex_syncs_errs(self): self.eq( len(errs), 1 ) self.eq( errs[0][0][1]['mesg'][0], 'newp:fake' ) self.nn( core.getTufoByProp('inet:fqdn','vertex.link') ) + + def test_cortex_norm_fail(self): + with s_cortex.openurl('ram:///') as core: + core.formTufoByProp('inet:netuser','vertex.link/visi') + self.raises( BadTypeValu, core.eval, 'inet:netuser="totally invalid input"' ) From bddeed55c42093bdc8e9135c490fcba354568d43 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 21:07:38 -0400 Subject: [PATCH 14/18] fix tests and tweak syntax int handling --- synapse/lib/syntax.py | 9 ++++++++- synapse/tests/test_cmds_cortex.py | 2 +- synapse/tests/test_lib_ingest.py | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/synapse/lib/syntax.py b/synapse/lib/syntax.py index 6f6c5332a2..e788d1abdb 100644 --- a/synapse/lib/syntax.py +++ b/synapse/lib/syntax.py @@ -389,7 +389,14 @@ def parse_macro_valu(text,off=0): # since it's not quoted, we can assume we are white # whit space bound ( only during macro syntax ) - return meh(text,off,whites) + valu,off = meh(text,off,whites) + if valu.isdigit(): + return int(valu),off + + if valu.startswith('0x') and valu[2:].isdigit(): + return int(valu,0),off + + return valu,off def parse_when(text,off,trim=True): whenstr,off = nom(text,off,whenset) diff --git a/synapse/tests/test_cmds_cortex.py b/synapse/tests/test_cmds_cortex.py index 6960ff2f97..605d5c1ce6 100644 --- a/synapse/tests/test_cmds_cortex.py +++ b/synapse/tests/test_cmds_cortex.py @@ -146,7 +146,7 @@ def test_cmds_ask(self): core.formTufoByProp('inet:email','visi@vertex.link') resp = cmdr.runCmdLine('ask inet:email="visi@vertex.link"') self.eq( len(resp['data']), 1 ) - self.eq( str(outp).strip(), 'visi@vertex.link -') + self.ne( str(outp).strip().find('visi@vertex.link'), -1 ) def test_cmds_ask_debug(self): with self.getDmonCore() as core: diff --git a/synapse/tests/test_lib_ingest.py b/synapse/tests/test_lib_ingest.py index d2cb7d472b..cd83a09f61 100644 --- a/synapse/tests/test_lib_ingest.py +++ b/synapse/tests/test_lib_ingest.py @@ -729,7 +729,7 @@ def test_ingest_embed_props(self): info = { "embed":[ { - "props":{ "tld":1 }, + "props":{ "sfx":1 }, "nodes":[ ["inet:fqdn",[ "com", @@ -748,7 +748,7 @@ def test_ingest_embed_props(self): self.nn( core.getTufoByProp('inet:fqdn','net') ) self.nn( core.getTufoByProp('inet:fqdn','org') ) - self.eq( 3, len(core.eval('inet:fqdn:tld=1'))) + self.eq( 3, len(core.eval('inet:fqdn:sfx=1'))) def test_ingest_embed_pernode_tagsprops(self): with s_cortex.openurl('ram://') as core: From fcdeb9399330e8cb74747f49929bc7942afccdb1 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Mon, 15 May 2017 21:51:14 -0400 Subject: [PATCH 15/18] explicit int casting for range oper (for non-modeled types) and slightly better DWIM case for storm ints in unmodeled props --- synapse/cores/ram.py | 11 +++++++++-- synapse/cores/sqlite.py | 6 ++++-- synapse/lib/syntax.py | 20 +++++++++++++++----- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/synapse/cores/ram.py b/synapse/cores/ram.py index 5fda4307ed..4e648159b6 100644 --- a/synapse/cores/ram.py +++ b/synapse/cores/ram.py @@ -54,13 +54,20 @@ def _tufosByLe(self, prop, valu, limit=None): return self.getTufosByIdens([ r[0] for r in rows ]) def _sizeByRange(self, prop, valu, limit=None): - return sum( 1 for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= valu[0] and r[2] < valu[1] ) + minval = int(valu[0]) + maxval = int(valu[1]) + return sum( 1 for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= minval and r[2] < maxval ) def _rowsByRange(self, prop, valu, limit=None): + minval = int(valu[0]) + maxval = int(valu[1]) + # HACK: for speed - ret = [ r for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= valu[0] and r[2] < valu[1] ] + ret = [ r for r in self.rowsbyprop.get(prop,()) if isint(r[2]) and r[2] >= minval and r[2] < maxval ] + if limit != None: ret = ret[:limit] + return ret def _sizeByGe(self, prop, valu, limit=None): diff --git a/synapse/cores/sqlite.py b/synapse/cores/sqlite.py index 21474c845c..adea1782cc 100644 --- a/synapse/cores/sqlite.py +++ b/synapse/cores/sqlite.py @@ -237,9 +237,11 @@ def _rowsByRange(self, prop, valu, limit=None): limit = self._getDbLimit(limit) q = self._q_getrows_by_range - args = [ prop, valu[0], valu[1], limit ] - rows = self.select(q, prop=prop, minvalu=valu[0], maxvalu=valu[1], limit=limit) + minvalu = int(valu[0]) + maxvalu = int(valu[1]) + + rows = self.select(q, prop=prop, minvalu=minvalu, maxvalu=maxvalu, limit=limit) return self._foldTypeCols(rows) def _rowsByGe(self, prop, valu, limit=None): diff --git a/synapse/lib/syntax.py b/synapse/lib/syntax.py index e788d1abdb..46ddd11a15 100644 --- a/synapse/lib/syntax.py +++ b/synapse/lib/syntax.py @@ -1,3 +1,5 @@ +import re + import synapse.lib.sched as s_sched import synapse.lib.service as s_service @@ -377,6 +379,17 @@ def parse_ques(text,off=0,trim=True): return ques,off +hexnumre = re.compile(r'^0x[0-9a-fA-F]+$') +def isint(text): + + if text.isdigit(): + return True + + if hexnumre.match(text): + return True + + return False + def parse_macro_valu(text,off=0): ''' Special syntax for the right side of equals in a macro @@ -388,12 +401,9 @@ def parse_macro_valu(text,off=0): return parse_string(text,off) # since it's not quoted, we can assume we are white - # whit space bound ( only during macro syntax ) + # space bound ( only during macro syntax ) valu,off = meh(text,off,whites) - if valu.isdigit(): - return int(valu),off - - if valu.startswith('0x') and valu[2:].isdigit(): + if isint(valu): return int(valu,0),off return valu,off From a41afc13792f8157ef9df492f1d085cd293160ab Mon Sep 17 00:00:00 2001 From: invisig0th Date: Tue, 16 May 2017 10:26:44 -0400 Subject: [PATCH 16/18] final syntax fix and new refs() storm oper --- synapse/datamodel.py | 12 +++ synapse/lib/storm.py | 139 ++++++++++++++++++++++---------- synapse/tests/test_lib_storm.py | 15 ++++ 3 files changed, 124 insertions(+), 42 deletions(-) diff --git a/synapse/datamodel.py b/synapse/datamodel.py index 7d290f4c2a..fcf603f911 100644 --- a/synapse/datamodel.py +++ b/synapse/datamodel.py @@ -272,6 +272,18 @@ def _addSubRefs(self, pdef): continue self.subprops[prop].append(pdef) + def getPropsByType(self, name): + ''' + Return a list of prop def tuples (name,info) for all props of the given type. + + Example: + + for prop,info in modl.getPropsByType('guid'): + dostuff() + + ''' + return self.propsbytype.get(name,()) + def _addPropGlob(self, form, prop, **info): prop = '%s:%s' % (form,prop) info['form'] = form diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index 86b52a3297..e58296a135 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -15,6 +15,25 @@ logger = logging.getLogger(__name__) +class LimitHelp: + + def __init__(self, limit): + self.limit = limit + + def get(self): + return self.limit + + def reached(self): + return self.limit != None and self.limit == 0 + + def dec(self, size=1): + + if self.limit == None: + return False + + self.limit = max(self.limit-size,0) + return self.limit == 0 + class OperWith: def __init__(self, query, oper): @@ -219,8 +238,8 @@ def __init__(self, **opts): self.setOperFunc('join', self._stormOperJoin) self.setOperFunc('lift', self._stormOperLift) + self.setOperFunc('refs', self._stormOperRefs) self.setOperFunc('pivot', self._stormOperPivot) - self.setOperFunc('expand', self._stormOperExpand) self.setOperFunc('alltag', self._stormOperAllTag) self.setOperFunc('addtag', self._stormOperAddTag) self.setOperFunc('deltag', self._stormOperDelTag) @@ -738,6 +757,83 @@ def _stormOperAddXref(self, query, oper): node = core.formTufoByProp(xref,(sorc,form,valu)) query.add(node) + def _stormOperRefs(self, query, oper): + args = oper[1].get('args') + opts = dict( oper[1].get('kwlist') ) + + #TODO opts.get('degrees') + limt = LimitHelp( opts.get('limit') ) + + core = self.getStormCore() + + nodes = query.data() + + #NOTE: we only actually want refs where type is form + + done = set() + if not args or 'in' in args: + + for node in nodes: + + if limt.reached(): + break + + form = node[1].get('tufo:form') + valu = node[1].get(form) + + dkey = (form,valu) + if dkey in done: + continue + + done.add(dkey) + + for prop,info in core.getPropsByType(form): + + pkey = (prop,valu) + if pkey in done: + continue + + done.add(pkey) + + news = core.getTufosByProp(prop,valu=valu, limit=limt.get()) + + [ query.add(n) for n in news ] + + if limt.dec(len(news)): + break + + if not args or 'out' in args: + + for node in nodes: + + if limt.reached(): + break + + form = node[1].get('tufo:form') + for prop,info in core.getSubProps(form): + + valu = node[1].get(prop) + if valu == None: + continue + + # ensure that the prop's type is also a form + name = core.getPropTypeName(prop) + if not core.isTufoForm(name): + continue + + pkey = (prop,valu) + if pkey in done: + continue + + done.add(pkey) + + news = core.getTufosByProp(name,valu=valu,limit=limt.get()) + + [ query.add(n) for n in news ] + + if limt.dec(len(news)): + break + def _stormOperSetProp(self, query, oper): args = oper[1].get('args') props = dict( oper[1].get('kwlist') ) @@ -772,47 +868,6 @@ def _stormOperAllTag(self, query, oper): if limit <= 0: break - def _stormOperExpand(self, query, oper): - - args = oper[1].get('args') - opts = dict(oper[1].get('kwlist')) - - # total node lift limit - limit = opts.get('limit') - - # how many degress of expansion? - degs = int(opts.get('degrees',1)) - - done = set() - todo = collections.deque([ (0,node) for node in query.data() ]) - - core = self.getStormCore() - - while len(todo): - - if limit != None and limit <= 0: - # TODO: plumb a way to communicate this in the result? - break - - deg,node = todo.popleft() - if node[0] in done: - continue - - done.add(node[0]) - if deg >= degs: - continue - - form = node[1].get('tufo:form') - valu = node[1].get(form) - - nods = core.getTufosByPropType(form, valu, limit=limit) - for newn in nods: - query.add(newn) - todo.append( (deg+1, newn) ) - - if limit != None: - limit -= len(newn) - def _stormOperAddTag(self, query, oper): tags = oper[1].get('args') diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index baeb5ef4f3..cbda8abcd8 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -136,3 +136,18 @@ def test_storm_deltag(self): self.none( node[1].get('*|inet:fqdn|foo') ) self.none( node[1].get('*|inet:fqdn|foo.bar') ) self.none( node[1].get('*|inet:fqdn|baz.faz') ) + + def test_storm_refs(self): + + with s_cortex.openurl('ram:///') as core: + core.setConfOpt('enforce',1) + + iden = guid() + core.formTufoByProp('inet:dns:a','foo.com/1.2.3.4') + core.formTufoByProp('inet:dns:a','bar.com/1.2.3.4') + + self.eq( len(core.eval('inet:ipv4=1.2.3.4 refs(in)')), 3 ) + self.eq( len(core.eval('inet:ipv4=1.2.3.4 refs(in,limit=1)')), 2 ) + + self.eq( len(core.eval('inet:dns:a=foo.com/1.2.3.4 refs(out)')), 3 ) + self.eq( len(core.eval('inet:dns:a=foo.com/1.2.3.4 refs(out,limit=1)')), 2 ) From be6d6d287af56eeddc4d2842d43689e449cda261 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Tue, 16 May 2017 10:27:41 -0400 Subject: [PATCH 17/18] acutally commit all the files for the delta :D --- synapse/lib/syntax.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/synapse/lib/syntax.py b/synapse/lib/syntax.py index e788d1abdb..def195ff34 100644 --- a/synapse/lib/syntax.py +++ b/synapse/lib/syntax.py @@ -353,13 +353,11 @@ def parse_ques(text,off=0,trim=True): if text[off] != '=': raise SyntaxError(text=text, off=off, mesg='expected equals for by syntax') - #ques['valu'],off = parse_oarg(text,off+1) ques['valu'],off = parse_macro_valu(text,off+1) return ques,off if text[off] == '=': ques['cmp'] = 'eq' - #ques['valu'],off = parse_oarg(text,off+1) ques['valu'],off = parse_macro_valu(text,off+1) break @@ -388,15 +386,8 @@ def parse_macro_valu(text,off=0): return parse_string(text,off) # since it's not quoted, we can assume we are white - # whit space bound ( only during macro syntax ) - valu,off = meh(text,off,whites) - if valu.isdigit(): - return int(valu),off - - if valu.startswith('0x') and valu[2:].isdigit(): - return int(valu,0),off - - return valu,off + # space bound ( only during macro syntax ) + return meh(text,off,whites) def parse_when(text,off,trim=True): whenstr,off = nom(text,off,whenset) From e881350a0208852b0cf6ba90747fa7dc4029d942 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Tue, 16 May 2017 10:39:39 -0400 Subject: [PATCH 18/18] fix unit tests and tweak syntax for backward compat for now --- synapse/lib/syntax.py | 14 +++++++++++++- synapse/tests/test_lib_storm.py | 14 -------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/synapse/lib/syntax.py b/synapse/lib/syntax.py index 6228a62835..eb3efa68f3 100644 --- a/synapse/lib/syntax.py +++ b/synapse/lib/syntax.py @@ -389,7 +389,19 @@ def parse_macro_valu(text,off=0): # since it's not quoted, we can assume we are white # space bound ( only during macro syntax ) - return meh(text,off,whites) + valu,off = meh(text,off,whites) + + # for now, give it a shot as an int... maybe eventually + # we'll be able to disable this completely, but for now + # lets maintain backward compatibility... + try: + # NOTE: this is ugly, but faster than parsing the string + valu = int(valu,0) + except ValueError as e: + pass + + return valu,off + def parse_when(text,off,trim=True): whenstr,off = nom(text,off,whenset) diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index cbda8abcd8..40cea9d399 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -53,20 +53,6 @@ def test_storm_setprop(self): self.eq( node[1].get('inet:fqdn:created'), 1462406400000 ) self.eq( node[1].get('inet:fqdn:updated'), 1493942400000 ) - def test_storm_expand(self): - - with s_cortex.openurl('ram:///') as core: - core.setConfOpt('enforce',1) - - iden = guid() - node = core.formTufoByProp('inet:dns:a','vertex.link/1.2.3.4') - - nodes = core.eval('inet:fqdn=vertex.link expand()') - - forms = list(sorted([ n[1].get('tufo:form') for n in nodes ])) - - self.eq( forms, ['inet:dns:a','inet:fqdn'] ) - def test_storm_filt_regex(self): with s_cortex.openurl('ram:///') as core: