Skip to content
Permalink
Browse files

ENH+TST: sql bots enhancements

upgrade function for migration
clean default values for parameters
clearer documentation
  • Loading branch information...
wagner-certat committed Sep 30, 2019
1 parent 3e51c5e commit 401c3a203600f17dbf0c04e2fce20f964883c52c
@@ -69,19 +69,15 @@ CHANGELOG
#### Experts
- Add geohash expert.
- `intelmq.bot.experts.generic_db_lookup.expert`
* got required parameter "engine"
* engine can be set to "postgresql" to drop-in replacement for previous functionality
* can be set to sqlite to search via this engine
- new optional parameter `engine` with `postgresql` (default) and `sqlite` (new) as possible values.

#### Outputs
- Add `intelmq.bots.outputs.touch.output`.
- `intelmq.bot.outputs.postgresql.output`:
- deprecated in favor of `intelmq.bot.outputs.sql.output`
- Compatibility shim will be available in the 2.x series.
- `intelmq.bot.outputs.sql.output` added generic SQL output bot. Comparted to
- got required parameter `engine`.
- engine can be set to `postgresql` is a drop-in replacement for the deprecated postgresql output bot.
- New engine `sqlite`.
- new optional parameter `engine` with `postgresql` (default) and `sqlite` (new) as possible values.

### Documentation
- Feeds: Add ViriBack feed.
@@ -24,7 +24,7 @@ The PostgreSQL Output Bot has been integrated into the new and generic SQL Outpu
* new: `intelmq.bots.outputs.sql.output`
* parameters:
* new: `engine` = `postgresql`
IntelMQ versions 2.x will be compatible with previous configurations.
IntelMQ versions 2.x will be compatible with previous configurations. `intelmqctl upgrade-config` migrates configurations.

### Libraries

@@ -1071,23 +1071,23 @@ class SQLBot(Bot):

POSTGRESQL = "postgresql"
SQLITE = "sqlite"
default_engine = "postgresql"

def init(self):
name = self.parameters.engine = self.parameters.engine.lower()
engines = {SQLBot.POSTGRESQL: (self._init_postgresql, "%s"), SQLBot.SQLITE: (self._init_sqlite, "?")}
self.engine_name = getattr(self.parameters, 'engine', self.default_engine).lower()
engines = {SQLBot.POSTGRESQL: (self._init_postgresql, "%s"),
SQLBot.SQLITE: (self._init_sqlite, "?")}
for key, val in engines.items():
if name == key:
if self.engine_name == key:
val[0]()
self.format_char = val[1]
break
else:
raise ValueError("Wrong parameter engine '{0}', possible values are {1}".format(name, engines))
raise ValueError("Wrong parameter 'engine' {0!r}, possible values are {1}".format(self.engine_name, engines))

def _connect(self, engine, connect_args, autocommitable=False):
self.engine = engine # imported external library that connects to the DB
self.logger.debug("Connecting to database.")
if self.engine is None:
raise ValueError('Could not import {0}. Please install it.'.format(self.parameters.engine))

try:
self.con = self.engine.connect(**connect_args)
@@ -1104,7 +1104,7 @@ def _init_postgresql(self):
import psycopg2
import psycopg2.extras
except ImportError:
psycopg2 = None
raise ValueError("Could not import 'psycopg2'. Please install it.")

self._connect(psycopg2,
{"database": self.parameters.database,
@@ -1121,7 +1121,7 @@ def _init_sqlite(self):
try:
import sqlite3
except ImportError:
sqlite3 = None
raise ValueError("Could not import 'sqlite3'. Please install it.")

self._connect(sqlite3,
{"database": self.parameters.database,
@@ -259,6 +259,14 @@ def v210_deprecations(defaults, runtime, dry_run):
bot["parameters"]["extract_files"] = bot["parameters"]["unzip_attachment"]
del bot["parameters"]["unzip_attachment"]
changed = True
if bot["module"] in ("intelmq.bots.experts.generic_db_lookup.expert",
"intelmq.bots.outputs.postgresql.output"):
if "engine" not in bot["parameters"]:
bot["parameters"]["engine"] = "postgresql"
changed = True
if bot["module"] == "intelmq.bots.outputs.postgresql.output":
bot["module"] = "intelmq.bots.outputs.sql.output"
changed = True
return changed, defaults, runtime


@@ -49,7 +49,7 @@ def set_bot(cls):
"database": "intelmq",
"user": "intelmq",
"password": "intelmq",
"sslmode": "require",
"sslmode": "allow",
"table": "lookuptests",
"match_fields": {"source.asn": "asn"},
"replace_fields": {"note": "comment",
@@ -104,6 +104,41 @@
"module": "intelmq.bots.collectors.rt.collector_rt",
"parameters": {
},
},
"postgresql-output": {
"group": "Output",
"module": "intelmq.bots.outputs.postgresql.output",
"parameters": {
"autocommit": True,
"database": "intelmq-events",
"host": "localhost",
"jsondict_as_string": True,
"password": "<password>",
"port": "5432",
"sslmode": "require",
"table": "events",
"user": "intelmq"
},
},
"db-lookup": {
"module": "intelmq.bots.experts.generic_db_lookup.expert",
"parameters": {
"database": "intelmq",
"host": "localhost",
"match_fields": {
"source.asn": "asn"
},
"overwrite": False,
"password": "<password>",
"port": "5432",
"replace_fields": {
"contact": "source.abuse_contact",
"note": "comment"
},
"sslmode": "require",
"table": "contacts",
"user": "intelmq"
}
}
}
V210_EXP = {"test-collector": {
@@ -118,6 +153,43 @@
"module": "intelmq.bots.collectors.rt.collector_rt",
"parameters": {
},
},
"postgresql-output": {
"group": "Output",
"module": "intelmq.bots.outputs.sql.output",
"parameters": {
"autocommit": True,
"database": "intelmq-events",
"engine": "postgresql",
"host": "localhost",
"jsondict_as_string": True,
"password": "<password>",
"port": "5432",
"sslmode": "require",
"table": "events",
"user": "intelmq"
},
},
"db-lookup": {
"module": "intelmq.bots.experts.generic_db_lookup.expert",
"parameters": {
"engine": "postgresql",
"database": "intelmq",
"host": "localhost",
"match_fields": {
"source.asn": "asn"
},
"overwrite": False,
"password": "<password>",
"port": "5432",
"replace_fields": {
"contact": "source.abuse_contact",
"note": "comment"
},
"sslmode": "require",
"table": "contacts",
"user": "intelmq"
}
}
}

0 comments on commit 401c3a2

Please sign in to comment.
You can’t perform that action at this time.