Add support for custom filters in JSON files

doc/csdiff.h2m

@@ -1,6 +1,28 @@
 [NAME]
 csdiff - take two lists of defects and output either added or fixed ones
 
+[OPTIONS]
+The \fB\-\-filter\-file\fR option takes a list of JSON files in the following
+format.  Missing replace entry is equal to replace : "".
+
+.RS 4
+.nf
+{
+    "msg-filter" : [
+        {
+             "checker" : "DIVINE|SYMBIOTIC",
+             "regexp" : "memory"
+        },
+        {
+             "checker" : "COMPILER_WARNING",
+             "regexp" : "called on unallocated object",
+             "replace" : "called correctly, no UB here"
+        }
+    ]
+}
+.fi
+.RE
+
 [EXIT STATUS]
 csdiff exits with status 0 if arguments are valid and input files are parsed
 successfully.  It does not matter whether any defects were matched or not.

doc/csgrep.h2m

@@ -1,6 +1,28 @@
 [NAME]
 csgrep - filter the list of defects by the specified regex-based predicates
 
+[OPTIONS]
+The \fB\-\-filter\-file\fR option takes a list of JSON files in the following
+format.  Missing replace entry is equal to replace : "".
+
+.RS 4
+.nf
+{
+    "msg-filter" : [
+        {
+             "checker" : "DIVINE|SYMBIOTIC",
+             "regexp" : "memory"
+        },
+        {
+             "checker" : "COMPILER_WARNING",
+             "regexp" : "called on unallocated object",
+             "replace" : "called correctly, no UB here"
+        }
+    ]
+}
+.fi
+.RE
+
 [EXIT STATUS]
 csgrep exits with status 0 if arguments are valid and input files are parsed
 successfully.  It does not matter whether any defects were matched or not.

src/abstract-tree.hh

@@ -60,14 +60,14 @@ inline T valueOf(const pt::ptree &node, const char *path, const T &defVal)
     return opt.get_value_or(defVal);
 }
 
-inline std::string getStringValue(const pt::ptree *const node)
+inline std::string getStringValue(const pt::ptree &node)
 {
-    return node->get_value<std::string>();
+    return node.get_value<std::string>();
 }
 
 inline std::string getStringValue(const pt::ptree::const_iterator it)
 {
-    return getStringValue(&it->second);
+    return getStringValue(it->second);
 }
 
 #endif /* H_GUARD_ABSTRACT_TREE_H */

src/csdiff.cc

@@ -50,7 +50,9 @@ int main(int argc, char *argv[])
             ("json-output,j", "write the result in JSON format")
             ("html-output", "write the result in HTML format")
             ("file-rename,s", po::value<TStringList>(),
-             "account the file base-name change, [OLD,NEW] (*testing*)");
+             "account the file base-name change, [OLD,NEW] (*testing*)")
+            ("filter-file,f", po::value<TStringList>(),
+             "read custom filtering rules from a file in JSON format");
 
         addColorOptions(&desc);
 
@@ -151,6 +153,13 @@ int main(int argc, char *argv[])
     const bool showInternal = vm.count("show-internal");
     const bool silent       = vm.count("quiet");
 
+    if (vm.count("filter-file")) {
+        const TStringList &filterFiles = vm["filter-file"].as<TStringList>();
+        if (!MsgFilter::inst()->setFilterFiles(filterFiles, silent))
+            // an error message already printed out
+            return 1;
+    }
+
     try {
         // open streams
         InStream strOld(fnOld, silent);

src/csfilter.cc

@@ -17,10 +17,11 @@
  * along with csdiff.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "abstract-tree.hh"
 #include "csfilter.hh"
 #include "regex.hh"
 
-#include <iostream>
+#include <boost/property_tree/json_parser.hpp>
 
 // Setup verbosity for debugging string substitions while matching them.
 // Verbosity levels are from 0 to 3 (0 is off)
@@ -150,6 +151,56 @@ void MsgFilter::setIgnorePath(bool enable)
     d->ignorePath = enable;
 }
 
+bool MsgFilter::setFilterFiles(
+                const TStringList &fileNames,
+                bool               silent)
+{
+    try {
+        for (const std::string &file : fileNames) {
+            InStream filter(file, silent);
+            if (!setJSONFilter(filter))
+                return false;
+        }
+        return true;
+    }
+    catch (const InFileException &e) {
+        std::cerr << e.fileName << ": failed to open filter file\n";
+        return false;
+    }
+}
+
+bool MsgFilter::setJSONFilter(InStream &input)
+{
+    using namespace boost::property_tree;
+
+    try {
+        // parse JSON
+        ptree root;
+        read_json(input.str(), root);
+
+        // read filtering rules
+        for (const auto &filter_rule : root.get_child("msg-filter")) {
+            const auto &filter = filter_rule.second;
+            d->addMsgFilter(getStringValue(filter.get_child("checker")),
+                            getStringValue(filter.get_child("regexp")),
+                            valueOf(filter, "replace", std::string{}));
+        }
+        return true;
+    }
+    catch (boost::regex_error &e) {
+        input.handleError(e.what());
+        return false;
+    }
+    catch (file_parser_error &e) {
+        input.handleError(e.message(), e.line());
+        return false;
+    }
+    catch (ptree_error &e) {
+        input.handleError(e.what());
+        return false;
+    }
+}
+
 void MsgFilter::setFileNameSubstitution(
                 const std::string      &oldFile,
                 const std::string      &newFile)

src/csfilter.hh

@@ -20,9 +20,13 @@
 #ifndef H_GUARD_CSFILTER_H
 #define H_GUARD_CSFILTER_H
 
+#include "instream.hh"
+
 #include <map>
 #include <string>
+#include <vector>
 
+typedef std::vector<std::string> TStringList;
 typedef std::map<std::string, std::string> TSubstMap;
 
 class MsgFilter {
@@ -35,6 +39,8 @@ class MsgFilter {
         }
 
         void setIgnorePath(bool);
+        bool setFilterFiles(const TStringList &fileNames,
+                            bool               silent);
         void setFileNameSubstitution(
                 const std::string      &oldFile,
                 const std::string      &newFile);
@@ -48,6 +54,8 @@ class MsgFilter {
         MsgFilter();
         ~MsgFilter();
 
+        bool setJSONFilter(InStream &filter);
+
         static MsgFilter *self_;
         struct Private;
         Private *d;

src/csgrep.cc

@@ -586,8 +586,8 @@ int main(int argc, char *argv[])
 
             ("ignore-case,i",                                   "ignore case when matching regular expressions")
             ("invert-match,v",                                  "select defects that do not match the selected criteria")
-            ("invert-regex,n",                                  "invert regular expressions in all predicates");
-
+            ("invert-regex,n",                                  "invert regular expressions in all predicates")
+            ("filter-file,f",       po::value<TStringList>(),   "read custom filtering rules from a file in JSON format");
         addColorOptions(&desc);
         desc.add_options()
             ("quiet,q",                                         "do not report any parsing errors")
@@ -639,6 +639,15 @@ int main(int argc, char *argv[])
         return 1;
     }
 
+    const bool silent = vm.count("quiet");
+
+    if (vm.count("filter-file")) {
+        const TStringList &filterFiles = vm["filter-file"].as<TStringList>();
+        if (!MsgFilter::inst()->setFilterFiles(filterFiles, silent))
+            // an error message already printed out
+            return 1;
+    }
+
     // create a writer according to the selected mode
     WriterFactory factory;
     AbstractWriter *eng = factory.create(mode);
@@ -664,14 +673,13 @@ int main(int argc, char *argv[])
     if (vm.count("remove-duplicates"))
         eng = new DuplicateFilter(eng);
 
-    const bool silent = vm.count("quiet");
-    bool hasError = false;
-
     if (!chainDecorator<EventPrunner>(&eng, vm, "prune-events")
             || !chainDecorator<CtxEmbedder>(&eng, vm, "embed-context"))
         // error message already printed, eng already feeed
         return 1;
 
+    bool hasError = false;
+
     if (!vm.count("input-file")) {
         hasError = !eng->handleFile("-", silent);
     }

src/xml-parser.cc

@@ -144,12 +144,12 @@ std::string readMsg(const pt::ptree &defNode)
     const pt::ptree *whatNode;
     if (findChildOf(&whatNode, defNode, "what"))
         // message found in <what>...</what>
-        return getStringValue(whatNode);
+        return getStringValue(*whatNode);
 
     if (findChildOf(&whatNode, defNode, "xwhat")
             && findChildOf(&whatNode, *whatNode, "text"))
         // message found in <xwhat><text>...</text></xwhat>
-        return getStringValue(whatNode);
+        return getStringValue(*whatNode);
 
     // message not found
     return "<unknown>";
@@ -195,7 +195,7 @@ void readStack(Defect *pDef, const pt::ptree &stackNode)
         const pt::ptree *fileNode;
         if (findChildOf(&fileNode, frameNode, "file")) {
             // read absolute path of the source file
-            noteEvt.fileName = getStringValue(fileNode);
+            noteEvt.fileName = getStringValue(*fileNode);
             const std::string dir = valueOf<std::string>(frameNode, "dir", "");
             if (!dir.empty())
                 noteEvt.fileName = dir + "/" + noteEvt.fileName;
@@ -206,12 +206,12 @@ void readStack(Defect *pDef, const pt::ptree &stackNode)
         }
         else if (findChildOf(&fileNode, frameNode, "obj")) {
             // pick path of the object file
-            noteEvt.fileName = getStringValue(fileNode);
+            noteEvt.fileName = getStringValue(*fileNode);
             keyEventScore = 4;
         }
         else if (findChildOf(&fileNode, frameNode, "ip")) {
             // pick address of the code in memory
-            noteEvt.fileName = getStringValue(fileNode);
+            noteEvt.fileName = getStringValue(*fileNode);
             keyEventScore = 2;
         }
         else {
@@ -270,7 +270,7 @@ bool ValgrindTreeDecoder::readNode(Defect *pDef, pt::ptree::const_iterator defIt
         DefEvent auxEvent = def.events[def.keyEventIdx];
         auxEvent.event = "note";
         auxEvent.verbosityLevel = /* note */ 1;
-        auxEvent.msg = getStringValue(auxwhat);
+        auxEvent.msg = getStringValue(*auxwhat);
         def.events.insert(def.events.begin() + def.keyEventIdx + 1, auxEvent);
     }
 

tests/csdiff/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (C) 2011 - 2020 Red Hat, Inc.
+# Copyright (C) 2011 - 2021 Red Hat, Inc.
 #
 # This file is part of csdiff.
 #
@@ -77,3 +77,5 @@ test_csdiff(diff-misc 09-shellcheck-raw)
 test_csdiff(diff-misc 10-pylint-copr)
 test_csdiff(diff-misc 11-pylint-copr-json)
 test_csdiff(diff-misc 12-shellcheck-sc222x)
+
+add_subdirectory(filter-file)

tests/csdiff/filter-file/01-basic-filter.json

@@ -0,0 +1,13 @@
+{
+    "msg-filter" : [
+        {
+            "checker" : "DIVINE|SYMBIOTIC",
+            "regexp" : "memory"
+        },
+        {
+            "checker" : "COMPILER_WARNING",
+            "regexp" : "called on unallocated object",
+            "replace" : "called correctly, no UB here"
+        }
+    ]
+}

tests/csdiff/filter-file/01-basic-fix-z.err

@@ -0,0 +1,5 @@
+Error: COMPILER_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-calloc/0004-calloc-plain-leak/0004-test.c: scope_hint: In function 'main'
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-calloc/0004-calloc-plain-leak/0004-test.c:5:5: warning[-Wunused-result]: ignoring return value of 'calloc' declared with attribute 'warn_unused_result'
+#    5 |     calloc(1, sizeof(char)); /* leak */
+#      |     ^~~~~~~~~~~~~~~~~~~~~~~

tests/csdiff/filter-file/01-basic-fix.err

@@ -0,0 +1,5 @@
+Error: COMPILER_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-calloc/0004-calloc-plain-leak/0004-test.c: scope_hint: In function 'main'
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-calloc/0004-calloc-plain-leak/0004-test.c:5:5: warning[-Wunused-result]: ignoring return value of 'calloc' declared with attribute 'warn_unused_result'
+#    5 |     calloc(1, sizeof(char)); /* leak */
+#      |     ^~~~~~~~~~~~~~~~~~~~~~~

tests/csdiff/filter-file/01-basic-new.err

@@ -0,0 +1,20 @@
+Error: COMPILER_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c: scope_hint: In function 'main'
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:6:5: warning[-Wfree-nonheap-object]: 'free' called correctly, no UB here 'a'
+#    6 |     free(&a); /* invalid free */
+#      |     ^~~~~~~~
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:5:9: note: declared here
+#    5 |     int a;
+#      |         ^
+
+Error: SYMBIOTIC_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-malloc/0009-malloc-zerosize-leak/0009-test.c:10: error: memory error: memory not cleaned up
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-malloc/0009-malloc-zerosize-leak/0009-test.c:10: note: call stack: main ()
+
+Error: DIVINE_WARNING:
+/home/lukas/tmp/assert-bounds.c: scope_hint: In function 'main':
+/home/lukas/tmp/assert-bounds.c:8: error: access of size 1 at [alloca*] is 1 bytes out of bounds
+/home/lukas/tmp/assert-bounds.c:8: note: memory error in userspace
+/opt/divine/include/dios/sys/fault.hpp:119: note: void __dios::FaultBase::handler<__dios::Context>(_VM_Fault, _VM_Frame*, void (*)())
+/home/lukas/tmp/assert-bounds.c:8: note: main
+/opt/divine/include/dios/libc/sys/start.cpp:91: note: __dios_start

tests/csdiff/filter-file/01-basic-old.err

@@ -0,0 +1,26 @@
+Error: COMPILER_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c: scope_hint: In function 'main'
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:6:5: warning[-Wfree-nonheap-object]: 'free' called on unallocated object 'a'
+#    6 |     free(&a); /* invalid free */
+#      |     ^~~~~~~~
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:5:9: note: declared here
+#    5 |     int a;
+#      |         ^
+
+Error: COMPILER_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-calloc/0004-calloc-plain-leak/0004-test.c: scope_hint: In function 'main'
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-calloc/0004-calloc-plain-leak/0004-test.c:5:5: warning[-Wunused-result]: ignoring return value of 'calloc' declared with attribute 'warn_unused_result'
+#    5 |     calloc(1, sizeof(char)); /* leak */
+#      |     ^~~~~~~~~~~~~~~~~~~~~~~
+
+Error: SYMBIOTIC_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-malloc/0009-malloc-zerosize-leak/0009-test.c:10: error: memory error: memory not cleaned up
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-malloc/0009-malloc-zerosize-leak/0009-test.c:10: note: call stack: main ()
+
+Error: DIVINE_WARNING:
+/home/lukas/tmp/assert-bounds.c: scope_hint: In function 'main':
+/home/lukas/tmp/assert-bounds.c:8: error: access of size 1 at [alloca*] is 1 bytes out of bounds
+/home/lukas/tmp/assert-bounds.c:8: note: memory error in userspace
+/opt/divine/include/dios/sys/fault.hpp:119: note: void __dios::FaultBase::handler<__dios::Context>(_VM_Fault, _VM_Frame*, void (*)())
+/home/lukas/tmp/assert-bounds.c:8: note: main
+/opt/divine/include/dios/libc/sys/start.cpp:91: note: __dios_start

tests/csdiff/filter-file/02-checker-regex-add-z.err

@@ -0,0 +1,8 @@
+Error: COMPILER_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c: scope_hint: In function 'main'
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:6:5: warning[-Wfree-nonheap-object]: 'free' called correctly, no UB here 'a'
+#    6 |     free(&a); /* invalid free */
+#      |     ^~~~~~~~
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:5:9: note: declared here
+#    5 |     int a;
+#      |         ^

tests/csdiff/filter-file/02-checker-regex-add.err

@@ -0,0 +1,8 @@
+Error: COMPILER_WARNING:
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c: scope_hint: In function 'main'
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:6:5: warning[-Wfree-nonheap-object]: 'free' called correctly, no UB here 'a'
+#    6 |     free(&a); /* invalid free */
+#      |     ^~~~~~~~
+/home/lukas/aufover-benchmark/tests/single-c/mem-basic-free/0006-free-stack/0006-test.c:5:9: note: declared here
+#    5 |     int a;
+#      |         ^