rulecat: by default ignore *deleted.rules.

doc/tools/rulecat.rst

@@ -28,9 +28,9 @@ Options
 
 .. option:: -t <directory>, --temp-dir=<directory>
 
-   Temporary working directory (default:
-   /var/tmp/idstools-rulecat). This is where downloaded files will be
-   stored.
+   Temporary working directory (default: /var/tmp/idstools-rulecat).
+
+   This is where downloaded files will be stored.
 
 .. option:: --suricata=<path>
 
@@ -57,7 +57,7 @@ Options
 .. option:: --merged=<filename>
 
    Write a single file containing all rules. This can be used in
-   addition to ``-o`` or instead of ``-o``.
+   addition to ``--output`` or instead of ``--output``.
 
 .. option:: --yaml-fragment=<filename.yaml>
 
@@ -110,15 +110,21 @@ Options
 
 .. option:: --ignore=<filename>
 
-   Filenames to ignore. This only deals with the base filename for now
-   such as ``emering-deleted.rules``, NOT
-   ``rules/emerging-deleted.rules``.
+   Filenames to ignore. This is a pattern that will be matched against
+   the basename of a rule files.
 
    This argument may be specified multiple times.
 
+   Default: *deleted.rules
+
    Alternatively the **group** matcher may be used in the file passed
    to ``--disable``.
 
+.. option:: --no-ignore
+
+   Disable the --ignore option. Most useful to disable the default
+   ignore pattern without adding others.
+
 .. option:: --etopen
 
    Download the ET open ruleset. This is the default if ``--url`` or

idstools/scripts/rulecat.py

@@ -635,8 +635,8 @@ def resolve_etopen_url(suricata_version):
     return ET_OPEN_URL % mappings
 
 def ignore_file(ignore_files, filename):
-    for name in ignore_files:
-        if name == filename:
+    for pattern in ignore_files:
+        if fnmatch.fnmatch(os.path.basename(filename), pattern):
             return True
     return False
 
@@ -695,7 +695,9 @@ def main():
 
     parser.add_argument("--ignore", metavar="<filename>", action="append",
                         default=[],
-                        help="Filenames to ignore")
+                        help="Filenames to ignore (default: *deleted.rules)")
+    parser.add_argument("--no-ignore", action="store_true", default=False,
+                        help="Disables the ignore option.")
 
     parser.add_argument("--threshold-in", metavar="<filename>",
                         help="Filename of rule thresholding configuration")
@@ -732,6 +734,13 @@ def main():
     if args.dump_sample_configs:
         return dump_sample_configs()
 
+    # If --no-ignore was provided, make sure args.ignore is
+    # empty. Otherwise if no ignores are provided, set a sane default.
+    if args.no_ignore:
+        args.ignore = []
+    elif len(args.ignore) == 0:
+        args.ignore.append("*deleted.rules")
+
     if args.suricata_version:
         suricata_version = idstools.suricata.parse_version(args.suricata_version)
         if not suricata_version:
@@ -774,16 +783,19 @@ def main():
 
     files = Fetch(args).run()
 
+    # Remove ignored files.
+    for filename in list(files.keys()):
+        if ignore_file(args.ignore, filename):
+            logger.info("Ignoring file %s" % (filename))
+            del(files[filename])
+
     for path in args.local:
         load_local(path, files)
 
     rules = []
     for filename in files:
         if not filename.endswith(".rules"):
             continue
-        if ignore_file(args.ignore, filename):
-            logger.info("Ignoring file %s" % (filename))
-            continue
         logger.debug("Parsing %s." % (filename))
         rules += idstools.rule.parse_fileobj(
             io.BytesIO(files[filename]), filename)