diff --git a/.eslintignore b/.eslintignore index 6d07cae4fb..99e026597f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -7,7 +7,6 @@ /wiseService/vueapp/dist/ /wiseService/node_modules /wiseService/sprintf.js -/wiseService/example.source.js /viewer/vueapp/build/ /viewer/vueapp/config/ /viewer/vueapp/dist/ diff --git a/wiseService/simpleSource.js b/wiseService/simpleSource.js index 274584341f..304faa8afd 100644 --- a/wiseService/simpleSource.js +++ b/wiseService/simpleSource.js @@ -116,11 +116,6 @@ class SimpleSource extends WISESource { return cb(null, newresult); }; - // ---------------------------------------------------------------------------- - getTypes () { - return [this.type]; - }; - // ---------------------------------------------------------------------------- /** * This loads the data for the simple source. SimpleSource will call on creation and on reloads. @@ -174,7 +169,7 @@ class SimpleSource extends WISESource { if (this.reload > 0) { setInterval(this.load.bind(this), this.reload * 1000 * 60); } - this.api.addSource(this.section, this); + this.api.addSource(this.section, this, [this.type]); } // Process results diff --git a/wiseService/source.alienvault.js b/wiseService/source.alienvault.js index a7f8f88b15..414a5143b0 100644 --- a/wiseService/source.alienvault.js +++ b/wiseService/source.alienvault.js @@ -32,7 +32,7 @@ class AlienVaultSource extends WISESource { } this.ips = new Map(); - this.api.addSource('alienvault', this); + this.api.addSource('alienvault', this, ['ip']); this.idField = this.api.addField('field:alienvault.id;db:alienvault.id;kind:integer;friendly:Id;help:Alien Vault ID;count:true'); this.reliabilityField = this.api.addField('field:alienvault.reliability;db:alienvault.reliability;kind:integer;friendly:Reliability;help:Alien Vault Reliability;count:true'); diff --git a/wiseService/source.elasticesearch.js b/wiseService/source.elasticesearch.js index 03c057122c..8bdf192d56 100644 --- a/wiseService/source.elasticesearch.js +++ b/wiseService/source.elasticesearch.js @@ -48,7 +48,7 @@ class ElasticsearchSource extends WISESource { apiVersion: '7.7' }); - api.addSource(section, this); + api.addSource(section, this, [this.type]); this.sourceFields = [this.esResultField]; for (const k in this.shortcuts) { diff --git a/wiseService/source.emergingthreats.js b/wiseService/source.emergingthreats.js index 6e859fcae6..d5a725bd5d 100644 --- a/wiseService/source.emergingthreats.js +++ b/wiseService/source.emergingthreats.js @@ -36,7 +36,7 @@ class EmergingThreatsSource extends WISESource { this.domains = new Map(); this.categories = {}; - this.api.addSource('emergingthreats', this); + this.api.addSource('emergingthreats', this, ['ip', 'domain']); this.scoreField = this.api.addField('field:emergingthreats.score;db:et.score;kind:integer;friendly:Score;help:Emerging Threats Score;count:true'); this.categoryField = this.api.addField('field:emergingthreats.category;db:et.category;kind:termfield;friendly:Category;help:Emerging Threats Category;count:true'); diff --git a/wiseService/source.hodi.js b/wiseService/source.hodi.js index 270923fed3..8a56606edd 100644 --- a/wiseService/source.hodi.js +++ b/wiseService/source.hodi.js @@ -24,7 +24,7 @@ const LRU = require('lru-cache'); class HODISource extends WISESource { // ---------------------------------------------------------------------------- constructor (api, section) { - super(api, section, { }); + super(api, section, { fullQuery: true }); this.contentTypes = {}; const contentTypes = this.api.getConfig(section, 'contentTypes', @@ -34,7 +34,6 @@ class HODISource extends WISESource { this.esHost = api.getConfig('hodi', 'esHost'); this.bulk = []; - this.fullQuery = true; if (this.esHost === undefined) { console.log(this.section, '- No esHost defined'); @@ -97,7 +96,7 @@ class HODISource extends WISESource { }); }); - this.api.addSource('hodi', this); + this.api.addSource('hodi', this, ['domain', 'email', 'ip', 'md5']); setInterval(this.sendBulk.bind(this), 1000); } diff --git a/wiseService/source.hodiredis.js b/wiseService/source.hodiredis.js index d10ea55cf4..e3ea59aefa 100644 --- a/wiseService/source.hodiredis.js +++ b/wiseService/source.hodiredis.js @@ -23,7 +23,7 @@ const redis = require('ioredis'); class HODIRedisSource extends WISESource { // ---------------------------------------------------------------------------- constructor (api, section) { - super(api, section, { dontCache: true }); + super(api, section, { dontCache: true, fullQuery: true }); this.contentTypes = {}; const contentTypes = this.api.getConfig(section, 'contentTypes', @@ -36,9 +36,8 @@ class HODIRedisSource extends WISESource { return; } - this.fullQuery = true; this.client = redis.createClient({ url: this.url }); - this.api.addSource(section, this); + this.api.addSource(section, this, ['domain', 'email', 'ip', 'md5']); const tagsField = this.api.addField('field:tags'); this.tagsDomain = WISESource.encodeResult(tagsField, 'nbs-domain'); diff --git a/wiseService/source.opendns.js b/wiseService/source.opendns.js index ceb2e2a0ea..58db8b65e7 100644 --- a/wiseService/source.opendns.js +++ b/wiseService/source.opendns.js @@ -34,7 +34,7 @@ class OpenDNSSource extends WISESource { this.processing = {}; this.statuses = { '-1': 'malicious', '0': 'unknown', '1': 'benign' }; - this.api.addSource('opendns', this); + this.api.addSource('opendns', this, ['domain']); this.getCategories(); setInterval(this.getCategories.bind(this), 10 * 60 * 1000); setInterval(this.performQuery.bind(this), 500); diff --git a/wiseService/source.passivetotal.js b/wiseService/source.passivetotal.js index a4933c12fc..8da75e8683 100644 --- a/wiseService/source.passivetotal.js +++ b/wiseService/source.passivetotal.js @@ -39,7 +39,7 @@ class PassiveTotalSource extends WISESource { this.waiting = []; this.processing = {}; - this.api.addSource('passivetotal', this); + this.api.addSource('passivetotal', this, ['domain', 'ip']); setInterval(this.performQuery.bind(this), 500); diff --git a/wiseService/source.redis.js b/wiseService/source.redis.js index 0fab657db9..da760f3f32 100644 --- a/wiseService/source.redis.js +++ b/wiseService/source.redis.js @@ -41,7 +41,7 @@ class RedisSource extends WISESource { this[this.typeFunc] = RedisSource.prototype.fetch; } - this.api.addSource(this.section, this); + this.api.addSource(this.section, this, [this.type]); } // ---------------------------------------------------------------------------- diff --git a/wiseService/source.reversedns.js b/wiseService/source.reversedns.js index 5679f4eb94..1172146b7e 100644 --- a/wiseService/source.reversedns.js +++ b/wiseService/source.reversedns.js @@ -64,7 +64,7 @@ class ReverseDNSSource extends WISESource { this.trie.add(parts[0], +parts[1] || (parts[0].includes(':') ? 128 : 32), true); }); - this.api.addSource('reversedns', this); + this.api.addSource('reversedns', this, ['ip']); } // ---------------------------------------------------------------------------- diff --git a/wiseService/source.splunk.js b/wiseService/source.splunk.js index 331e378e73..e5983d6b7e 100644 --- a/wiseService/source.splunk.js +++ b/wiseService/source.splunk.js @@ -65,7 +65,7 @@ class SplunkSource extends WISESource { console.log('Login was successful: ' + success); }); - api.addSource(section, this); + api.addSource(section, this, [this.type]); this.sourceFields = [this.esResultField]; for (const k in this.shortcuts) { diff --git a/wiseService/source.threatq.js b/wiseService/source.threatq.js index 8db0cce76e..3d9b47ce1a 100644 --- a/wiseService/source.threatq.js +++ b/wiseService/source.threatq.js @@ -66,7 +66,7 @@ class ThreatQSource extends WISESource { setImmediate(this.loadFile.bind(this)); setInterval(this.loadFile.bind(this), 24 * 60 * 60 * 1000); // Reload file every 24 hours - this.api.addSource('threatq', this); + this.api.addSource('threatq', this, ['domain', 'email', 'ip', 'md5']); } // ---------------------------------------------------------------------------- diff --git a/wiseService/source.threatstream.js b/wiseService/source.threatstream.js index c7618b73f8..dc0e812a39 100644 --- a/wiseService/source.threatstream.js +++ b/wiseService/source.threatstream.js @@ -54,7 +54,7 @@ class ThreatStreamSource extends WISESource { ThreatStreamSource.prototype.getIp = ThreatStreamSource.prototype.getIpApi; ThreatStreamSource.prototype.getMd5 = ThreatStreamSource.prototype.getMd5Api; ThreatStreamSource.prototype.getEmail = ThreatStreamSource.prototype.getEmailApi; - this.loadTypes(); + this.loadTypes(false); break; case 'zip': this.ips = new Map(); @@ -70,7 +70,7 @@ class ThreatStreamSource extends WISESource { ThreatStreamSource.prototype.getEmail = ThreatStreamSource.prototype.getEmailZip; ThreatStreamSource.prototype.getURL = ThreatStreamSource.prototype.getURLZip; ThreatStreamSource.prototype.dump = ThreatStreamSource.prototype.dumpZip; - api.addSource('threatstream', this); + api.addSource('threatstream', this, ['domain', 'email', 'ip', 'md5', 'url']); api.app.get('/threatstream/_reload', (req, res) => { this.loadFile.bind(this); res.send('Ok'); @@ -91,7 +91,7 @@ class ThreatStreamSource extends WISESource { ThreatStreamSource.prototype.getMd5 = ThreatStreamSource.prototype.getMd5Sqlite3; ThreatStreamSource.prototype.getEmail = ThreatStreamSource.prototype.getEmailSqlite3; ThreatStreamSource.prototype.getURL = ThreatStreamSource.prototype.getURLSqlite3; - this.loadTypes(); + this.loadTypes(true); api.app.get('/threatstream/_reload', (req, res) => { this.openDB.bind(this); res.send('Ok'); @@ -393,7 +393,7 @@ class ThreatStreamSource extends WISESource { } // ---------------------------------------------------------------------------- - loadTypes () { + loadTypes (includeUrl) { // Threatstream doesn't have a way to just ask for type matches, so we need to figure out which itypes are various types. this.types = {}; this.typesWithQuotes = {}; @@ -417,7 +417,11 @@ class ThreatStreamSource extends WISESource { } // Wait to register until request is done - this.api.addSource('threatstream', this); + if (includeUrl) { + this.api.addSource('threatstream', this, ['domain', 'email', 'ip', 'md5', 'url']); + } else { + this.api.addSource('threatstream', this, ['domain', 'email', 'ip', 'md5']); + } }); }; diff --git a/wiseService/source.valueactions.js b/wiseService/source.valueactions.js index 0d321b4f28..e804f3fe0c 100644 --- a/wiseService/source.valueactions.js +++ b/wiseService/source.valueactions.js @@ -43,7 +43,7 @@ class ValueActionsSource extends WISESource { return; } - this.api.addSource(section, this); + this.api.addSource(section, this, []); setImmediate(this.load.bind(this)); diff --git a/wiseService/source.virustotal.js b/wiseService/source.virustotal.js index e697db1894..2a123b54b8 100644 --- a/wiseService/source.virustotal.js +++ b/wiseService/source.virustotal.js @@ -25,7 +25,7 @@ let source; class VirusTotalSource extends WISESource { // ---------------------------------------------------------------------------- constructor (api, section) { - super(api, section, { }); + super(api, section, { fullQuery: true }); this.waiting = []; this.processing = {}; @@ -45,9 +45,8 @@ class VirusTotalSource extends WISESource { this.dataSources = this.api.getConfig('virustotal', 'dataSources', 'McAfee,Symantec,Microsoft,Kaspersky').split(',').map(item => item.trim()); this.dataSourcesLC = this.dataSources.map((x) => { return x.toLowerCase(); }); this.dataFields = []; - this.fullQuery = true; - this.api.addSource('virustotal', this); + this.api.addSource('virustotal', this, ['md5']); setInterval(this.performQuery.bind(this), 60000 / this.queriesPerMinute); let str = diff --git a/wiseService/source.wiseproxy.js b/wiseService/source.wiseproxy.js index 58e8a5b6ab..88192cca7e 100644 --- a/wiseService/source.wiseproxy.js +++ b/wiseService/source.wiseproxy.js @@ -62,7 +62,7 @@ class WiseProxySource extends WISESource { this.updateInfo(); setTimeout(this.updateInfo.bind(this), 5 * 60 * 1000); - this.api.addSource(this.section, this); + this.api.addSource(this.section, this, ['domain', 'email', 'ip', 'md5', 'url']); setInterval(this.performQuery.bind(this), 500); } diff --git a/wiseService/wiseService.js b/wiseService/wiseService.js index 311acec638..2a5a59668a 100644 --- a/wiseService/wiseService.js +++ b/wiseService/wiseService.js @@ -545,33 +545,15 @@ class WISESourceAPI { * * @param {string} section - The section name * @param {WISESource} src - A WISESource object + * @param {string|Array} types - An array of the types that this source supports */ - addSource (section, src) { - internals.sources[section] = src; - - let types; - - if (src.getTypes) { - // getTypes function defined, we can just use it - types = src.getTypes(); - } else { - // No getTypes function, go thru all the default types and any types we already know and guess - types = []; - for (let i = 0; i < internals.type2Name.length; i++) { - if (src[funcName(internals.type2Name[i])]) { - types.push(internals.type2Name[i]); - } - } - for (let type in internals.types) { - const typeInfo = internals.types[type]; - if (src[typeInfo.funcName] && !types.includes(type)) { - types.push(type); - } - } - src.getTypes = function () { - return types; - }; + addSource (section, src, types) { + if (section === undefined || src === undefined || types === undefined) { + console.log(`ERROR - bad call to addSource for ${section}`); + return; } + internals.sources[section] = src; + internals.sources[section].types = types; for (let i = 0; i < types.length; i++) { addType(types[i], src); @@ -987,7 +969,7 @@ function processQuery (req, query, cb) { // First query for this value src.srcInProgress[query.typeName][query.value] = [cb]; let startTime = Date.now(); - src[typeInfo.funcName](src.fullQuery === true ? query : query.value, (err, result) => { + src[typeInfo.funcName](src.fullQuery=== true ? query : query.value, (err, result) => { src.average100MS = (99.0 * src.average100MS + (Date.now() - startTime)) / 100.0; if (!err && src.cacheTimeout !== -1 && result !== undefined) { // If err or cacheTimeout is -1 then don't cache @@ -1323,7 +1305,7 @@ app.put(`/config/save`, [isConfigWeb, doAuth, noCacheJson, checkAdmin, jsonParse app.get('/types/:source?', [noCacheJson], (req, res) => { if (req.params.source) { if (internals.sources[req.params.source]) { - return res.send(internals.sources[req.params.source].getTypes().sort()); + return res.send(internals.sources[req.params.source].types.sort()); } else { return res.send([]); } diff --git a/wiseService/wiseSource.js b/wiseService/wiseSource.js index 0a83b82082..99d1141ffe 100644 --- a/wiseService/wiseSource.js +++ b/wiseService/wiseSource.js @@ -37,6 +37,7 @@ class WISESource { * @param {boolean} [options.tagsSetting=false] - load the optional tags setting * @param {boolean} [options.typeSetting=false] - load the required type setting * @param {boolean} [options.formatSetting=false] - load the format setting + * @param {boolean} [options.fullQuery=false] - for MD5/SHA, query will be query.value and query.contentType */ constructor (api, section, options) { this.api = api; @@ -56,6 +57,7 @@ class WISESource { this.cacheDroppedStat = 0; this.average100MS = 0; this.srcInProgress = {}; + this.fullQuery = !!options.fullQuery; if (options.typeSetting) { this.typeSetting(); @@ -376,11 +378,6 @@ class WISESource { throw new Error(`${this.section} - ERROR not loading since missing required type setting`); } this.typeFunc = this.api.funcName(this.type); - if (this.getTypes === undefined) { - this.getTypes = function () { - return [this.type]; - }; - } }; // ---------------------------------------------------------------------------- @@ -560,17 +557,10 @@ class WISESource { * @param {object} res - The express res object * @abstract */ -/** - * Get the types this source supports. - * @returns {string|array} the array of types this source supports, by default the type setting from config file if option.typeSetting was set - * - * @method - * @name WISESource#getTypes - * @abstract - */ /** - * Every source needs to implement this method, usually with + * Every source needs to implement this method. If a singleton it will just create the source object direction. + * If not it should loop thru all keys that start with sourcekind: * @method * @name WISESource.initSource * @param {WISESourceAPI} api - The api back into the WISE Service