Permalink
Browse files

Merge branch 'maintenance' into develop

  • Loading branch information...
wagner-certat committed Oct 19, 2018
2 parents 99ed9b5 + 26dcb26 commit 3700c4ed32b6be063d4f4ea6608f0b4fe40b0efa
@@ -121,7 +121,9 @@ CHANGELOG
### Documentation
- FAQ: Explanation and solution on orphaned queues.
- Add or fix the tables of contents for all documentation files.
- Feeds: Fix Autoshun Feed URL (#1325).
- Feeds:
- Fix Autoshun Feed URL (#1325).
- Add parameters `name` and `provider` to `intelmq/etc/feeds.yaml`, `docs/Feeds.md` and `intelmq/bots/BOTS` (#1321).
### Packaging
- Change the maintainer from Sasche Wilde to Sebastian Wagner (#1320).
@@ -138,6 +140,9 @@ CHANGELOG
### Contrib
- elasticsearch/elasticmapper: Add tlp field (#1308).
- `feeds-config-generator/intelmq_gen_feeds_conf`:
- Add parameters to write resulting configuration directly to files (#1321).
- Handle collector's `feed.name` and `feed.provider` (#1314).
### Known issues
@@ -7,6 +7,8 @@ Quickly generate feeds configurations (runtime and pipeline configs).
```
./intelmq_gen_feeds_conf -h
usage: intelmq_gen_feeds_conf [-h] --feeds-file <filepath> [--all]
[--runtime-output-file <filepath>]
[--pipeline-output-file <filepath>]
IntelMQ Feeds Config Generator tool
@@ -15,6 +17,10 @@ optional arguments:
--feeds-file <filepath>
feeds.yaml config file
--all iterate through all feeds
--runtime-output-file <filepath>
/tmp/runtime.conf
--pipeline-output-file <filepath>
/tmp/pipeline.conf
```
### Generate a bot configuration
@@ -27,3 +33,9 @@ optional arguments:
```
./intelmq_gen_feeds_conf --feeds-file=../../intelmq/etc/feeds.yaml --all
```
### Generate all bots configurations and send to file
```
./intelmq_gen_feeds_conf --feeds-file=../../intelmq/etc/feeds.yaml --all --runtime-output-file=/tmp/runtime.conf --pipeline-output-file=/tmp/pipeline.conf
```
@@ -11,7 +11,6 @@ except ImportError:
sys.exit(-1)
def print_feeds_list(feeds_config):
options = list()
index = 0
@@ -24,6 +23,7 @@ def print_feeds_list(feeds_config):
index += 1
return options
def sanitize_bot_id(id):
id = id.replace('.', '')
id = id.replace('_', '-')
@@ -59,15 +59,18 @@ def gen_configurations(provider, feed):
else:
parameters = dict()
runtime_config[bot_id] = dict(description="placeholder",
group=bot_type.title(),
module=info['module'],
name=provider + " " + feed,
parameters=parameters)
if bot_type == "collector":
parameters["name"] = feed
parameters["provider"] = provider
runtime_config[bot_id] = dict(description="N/A",
group=bot_type.title(),
module=info['module'],
parameters=parameters)
collector_bot_id = sanitize_bot_id(provider + "-" + feed + "-collector")
parser_bot_id = sanitize_bot_id(provider + "-" + feed + "-parser")
pipeline_config[collector_bot_id] = dict()
pipeline_config[collector_bot_id]['destination-queues'] = [parser_bot_id + "-queue"]
@@ -98,14 +101,25 @@ if __name__ == "__main__":
default=False,
help='iterate through all feeds')
parser.add_argument('--runtime-output-file',
action="store",
dest="runtime_file",
metavar="<filepath>",
required=False,
help="write runtime configuration to the given file (e.g. '/tmp/runtime.conf') instead of stdout")
arguments = parser.parse_args()
parser.add_argument('--pipeline-output-file',
action="store",
dest="pipeline_file",
metavar="<filepath>",
required=False,
help="write pipeline configuration to the given file (e.g. '/tmp/pipeline.conf') instead of stdout")
arguments = parser.parse_args()
with open(arguments.feeds_file) as fp:
feeds_config = yaml.load(fp.read())
runtime_config = dict()
pipeline_config = dict()
@@ -128,11 +142,18 @@ if __name__ == "__main__":
runtime_config, pipeline_config = gen_configurations(provider, feed)
if arguments.runtime_file:
with open(arguments.runtime_file, "w") as fp:
json.dump(runtime_config, fp, indent=4, sort_keys=True, separators=(',', ': '))
print("Runtime configuration written to: %s" % arguments.runtime_file)
else:
print("\nRuntime configuration:\n")
print(json.dumps(runtime_config, indent=4, sort_keys=True, separators=(',', ': ')))
print("RUNTIME CONFIGURATION:\n")
print(json.dumps(runtime_config, indent=4))
print("\n\n\n")
print("PIPELINE CONFIGURATION:\n")
print(json.dumps(pipeline_config, indent=4))
if arguments.pipeline_file:
with open(arguments.pipeline_file, "w") as fp:
json.dump(pipeline_config, fp, indent=4, sort_keys=True, separators=(',', ': '))
print("Pipeline configuration written to: %s" % arguments.pipeline_file)
else:
print("\nPipeline configuration:\n")
print(json.dumps(pipeline_config, indent=4, sort_keys=True, separators=(',', ': ')))
Oops, something went wrong.

0 comments on commit 3700c4e

Please sign in to comment.