Add a parser for 'splint' (#40)

* Add a parser for 'splint'
* splint: open csv file in text mode

docs/parsers.rst

@@ -52,3 +52,7 @@ turn them into :py:class:`firehose.model.Analysis` instances.
 * flawfinder.py
 
   Parser for warnings emitted by `flawfinder <https://www.dwheeler.com/flawfinder/>`_.
+
+* splint.py
+
+  Parser for the :option:`-csv` output format from `splint <http://splint.org/>`_.

examples/unconditional-file-leak.c

@@ -0,0 +1,10 @@
+#include <stdio.h>
+
+void test (const char *filename)
+{
+  int i;
+  FILE *f;
+  f = fopen (filename, "w");
+  for (i = 0; i < 10; i++)
+    fprintf (f, "%i: %i",  i,  i * i);
+}

firehose/parsers/splint.py

@@ -0,0 +1,103 @@
+#   Copyright 2017 David Malcolm <dmalcolm@redhat.com>
+#   Copyright 2017 Red Hat, Inc.
+#
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+#   USA
+
+# Parser for .csv files emitted by "splint"
+#   http://splint.org/
+
+from collections import namedtuple
+import csv
+import os
+import re
+
+from firehose.model import Message, Function, Point, Range, \
+    File, Location, Generator, Metadata, Analysis, Issue, Sut, Trace, \
+    State, Notes, CustomFields
+
+FIELDS = ['warning', 'flag_code', 'flag_name', 'priority',
+          'file', 'line', 'column',
+          'warning_text', 'additional_text']
+WARNING_TEXT_IDX = FIELDS.index('warning_text')
+
+class Row(namedtuple('Row', FIELDS)):
+    def to_issue(self):
+        """
+        Generate an Issue from this csv row.
+        """
+        location = Location(file=File(givenpath=self.file,
+                                      abspath=None),
+                            function=None, # FIXME
+                            point=Point(int(self.line),
+                                        int(self.column)))
+        return Issue(cwe=None,
+                     testid=self.flag_name,
+                     location=location,
+                     message=Message(self.warning_text),
+                     notes=Notes(self.additional_text),
+                     trace=None,
+                     severity=self.priority,
+                     customfields=None)
+
+def parse_row(row):
+    """
+    Convert a list generated by csv.reader into a Row
+    """
+    # splint doesn't escape quotes that occur within the messages, which
+    # can lead to extra fields:
+    #   https://github.com/ravenexp/splint/issues/6
+    # Workaround this by assuming that such quotes occurred in the
+    # "Warning Text" field.
+    # This fires for warnings 4 and 6
+    #
+    # This fixes the issue, but there is still some minor information loss
+    # e.g. some quote characters in the text disappear, and we sometimes
+    # gain a trailing quote character
+    if len(row) > len(FIELDS):
+        joined_text = ','.join(row[WARNING_TEXT_IDX:-1])
+        row = row[:WARNING_TEXT_IDX] + [joined_text, row[-1]]
+    return Row(*row)
+
+def parse_splint_csv(path):
+    """
+    Parse a .csv file written by splint's "-csv FILENAME" option.
+    Generate a list of Result instances.
+    """
+    generator = Generator(name='splint')
+    metadata = Metadata(generator, None, None, None)
+    analysis = Analysis(metadata, [])
+    with open(path, 'r') as f:
+        reader = csv.reader(f)
+        for raw_row in reader:
+            # Skip the initial title row
+            if raw_row[0] == 'Warning':
+                continue
+            rowobj = parse_row(raw_row)
+            analysis.results.append(rowobj.to_issue())
+
+    return analysis
+
+def parse_splint_stderr(stderr):
+    """
+    Parse the stderr from splint (as a string).
+    Return a version string, or None.
+    """
+    lines = stderr.splitlines()
+    if not lines:
+        return None
+    m = re.match("Splint\s+(.*)\s+---.*", lines[0])
+    if m:
+        return m.group(1)

tests/parsers/example-output/splint/README.txt

@@ -0,0 +1,8 @@
+These files were generated using:
+
+  splint \
+      -strict \
+      -csv unconditional-file-leak.csv \
+      examples/unconditional-file-leak.c \
+   > unconditional-file-leak.stdout \
+   2> unconditional-file-leak.stderr

tests/parsers/example-output/splint/unconditional-file-leak.csv

@@ -0,0 +1,9 @@
+Warning, Flag Code, Flag Name, Priority, File, Line, Column, Warning Text, Additional Text
+1,136,internalglobs,1,examples/unconditional-file-leak.c,7,7,"Called procedure fopen may access file system state, but globals list does not include globals fileSystem","A called function uses internal state, but the globals list for the function being checked does not include internalState"
+2,126,modfilesys,1,examples/unconditional-file-leak.c,7,7,"Undocumented modification of file system state possible from call to fopen: fopen(filename, "w")","report undocumented file system modifications (applies to unspecified functions if modnomods is set)"
+3,136,internalglobs,1,examples/unconditional-file-leak.c,9,5,"Called procedure fprintf may access file system state, but globals list does not include globals fileSystem","A called function uses internal state, but the globals list for the function being checked does not include internalState"
+4,126,modfilesys,1,examples/unconditional-file-leak.c,9,5,"Undocumented modification of file system state possible from call to fprintf: fprintf(f, "%i: %i", i, i * i)","report undocumented file system modifications (applies to unspecified functions if modnomods is set)"
+5,2,nullpass,1,examples/unconditional-file-leak.c,9,14,"Possibly null storage f passed as non-null param: fprintf (f, ...)","A possibly null pointer is passed as a parameter corresponding to a formal parameter with no /*@null@*/ annotation.  If NULL may be used for this parameter, add a /*@null@*/ annotation to the function parameter declaration."
+6,182,forblock,1,examples/unconditional-file-leak.c,9,5,"Body of for statement is not a block: fprintf(f, "%i: %i", i, i * i);","Loop body is a single statement, not a compound block."
+7,300,fcnuse,1,examples/unconditional-file-leak.c,3,6,"Function test declared but not used","A function is declared but not used. Use /*@unused@*/ in front of function header to suppress message."
+8,295,exportheader,1,examples/unconditional-file-leak.c,3,6,"Function test exported but not declared in header file","A declaration is exported, but does not appear in a header file."

tests/parsers/example-output/splint/unconditional-file-leak.stderr

@@ -0,0 +1,5 @@
+Splint 3.1.2 --- 04 Aug 2013
+
+Specified CSV output file already exists (use +csvoverwrite to automatically
+overwrite): unconditional-file-leak.csv
+Finished checking --- 8 code warnings

tests/parsers/example-output/splint/unconditional-file-leak.stdout

@@ -0,0 +1,39 @@
+examples/unconditional-file-leak.c: (in function test)
+examples/unconditional-file-leak.c:7:7:
+    Called procedure fopen may access file system state, but globals list does
+    not include globals fileSystem
+  A called function uses internal state, but the globals list for the function
+  being checked does not include internalState (Use -internalglobs to inhibit
+  warning)
+examples/unconditional-file-leak.c:7:7:
+    Undocumented modification of file system state possible from call to fopen:
+    fopen(filename, "w")
+  report undocumented file system modifications (applies to unspecified
+  functions if modnomods is set) (Use -modfilesys to inhibit warning)
+examples/unconditional-file-leak.c:9:5:
+    Called procedure fprintf may access file system state, but globals list
+    does not include globals fileSystem
+examples/unconditional-file-leak.c:9:5:
+    Undocumented modification of file system state possible from call to
+    fprintf: fprintf(f, "%i: %i", i, i * i)
+examples/unconditional-file-leak.c:9:14:
+    Possibly null storage f passed as non-null param: fprintf (f, ...)
+  A possibly null pointer is passed as a parameter corresponding to a formal
+  parameter with no /*@null@*/ annotation.  If NULL may be used for this
+  parameter, add a /*@null@*/ annotation to the function parameter declaration.
+  (Use -nullpass to inhibit warning)
+   examples/unconditional-file-leak.c:7:7: Storage f may become null
+examples/unconditional-file-leak.c:9:5: Body of for statement is not a block:
+                                           fprintf(f, "%i: %i", i, i * i);
+  Loop body is a single statement, not a compound block. (Use -forblock to
+  inhibit warning)
+examples/unconditional-file-leak.c:3:6: Function test declared but not used
+  A function is declared but not used. Use /*@unused@*/ in front of function
+  header to suppress message. (Use -fcnuse to inhibit warning)
+   examples/unconditional-file-leak.c:10:1: Definition of test
+examples/unconditional-file-leak.c:3:6:
+    Function test exported but not declared in header file
+  A declaration is exported, but does not appear in a header file. (Use
+  -exportheader to inhibit warning)
+   examples/unconditional-file-leak.c:10:1: Definition of test
+

tests/parsers/test_splint_parser.py

@@ -0,0 +1,90 @@
+#   Copyright 2017 David Malcolm <dmalcolm@redhat.com>
+#   Copyright 2017 Red Hat, Inc.
+#
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the Free Software
+#   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+#   USA
+
+import os
+import unittest
+
+from firehose.model import Issue
+
+from firehose.parsers.splint import parse_splint_csv, parse_splint_stderr
+
+class TestParser(unittest.TestCase):
+    def locate_filename(self, filename):
+        return os.path.join(os.path.dirname(__file__),
+                            'example-output',
+                            'splint',
+                            filename)
+
+    def parse_example_csv(self, filename):
+        return parse_splint_csv(self.locate_filename(filename))
+
+    def test_unconditional_file_leak(self):
+        a = self.parse_example_csv('unconditional-file-leak.csv')
+        self.assertEqual(a.metadata.generator.name, 'splint')
+        self.assertEqual(a.metadata.generator.version, None) # FIXME
+        self.assertEqual(a.metadata.sut, None)
+        self.assertEqual(a.metadata.file_, None)
+        self.assertEqual(a.metadata.stats, None)
+        self.assertEqual(len(a.results), 8)
+        r0 = a.results[0]
+        self.assertIsInstance(r0, Issue)
+        self.assertEqual(r0.cwe, None)
+        self.assertEqual(r0.testid, 'internalglobs')
+        self.assertEqual(r0.message.text,
+                         'Called procedure fopen may access file system'
+                         ' state, but globals list does not include globals'
+                         ' fileSystem')
+        self.assertEqual(r0.notes.text,
+                         'A called function uses internal state, but the'
+                         ' globals list for the function being checked does'
+                         ' not include internalState')
+        self.assertEqual(r0.location.file.givenpath,
+                         'examples/unconditional-file-leak.c')
+        self.assertEqual(r0.location.file.abspath, None)
+        self.assertEqual(r0.location.function, None)
+        self.assertEqual(r0.trace, None)
+        self.assertEqual(r0.severity, '1')
+
+        # Verify that rows with unescaped quotes are correctly worked around
+        # In this example, this affects warnings #4 and #6 (these are 1-based,
+        # so index 3 and 5 in the list).
+        r3 = a.results[3]
+        self.assertEqual(r3.location.line, 9)
+        self.assertEqual(r3.location.column, 5)
+        # Quote-handling is not quite perfect, we've lost the open quote of
+        # the format string, and there an erroneous trailing quote in this
+        # message.
+        self.assertEqual(r3.message.text,
+                         'Undocumented modification of file system state'
+                         ' possible from call to fprintf:'
+                         ' fprintf(f, %i: %i", i, i * i)"')
+
+        r5 = a.results[5]
+        # Similar issues here:
+        self.assertEqual(r5.message.text,
+                         'Body of for statement is not a block:'
+                         ' fprintf(f, %i: %i", i, i * i);"')
+
+    def parse_example_stderr(self, filename):
+        with open(self.locate_filename(filename)) as f:
+            stderr = f.read()
+        return parse_splint_stderr(stderr)
+
+    def test_parse_stderr(self):
+        v = self.parse_example_stderr('unconditional-file-leak.stderr')
+        self.assertEqual(v, '3.1.2')