Shift in result types & ranking scales

This commit completes work that was discussed during the OSSG mid-cycle.

Summarizing the major changes:
 - Result 'severity' has been moved from INFO/WARN/ERROR to a LOW/MEDIUM/HIGH
scale.  This, as previously, is intended to indicate the risk associated with
the issue being reported.
 - Result 'confidence' has been introduced, also with a LOW/MEDIUM/HIGH scale.
This is intended to allow a plugin writer to indicate their level of confidence
that the issue raised is truly a security problem.
 - A Bandit plugin may now return a result as a 2-tuple (issue_severity,
issue_text) or - optionally - a 3-tuple (issue_severity, issue_confidence,
issue_text).  Confidence will default to UNDEFINED if the plugin does not
return a value.

The commit includes updates across Bandit and plugins to utilize the new scales,
and updates to the associated functional test suite to maintain test coverage.

Additionally, this commit:
 - Modifies the labels on fields included in JSON and CSV output formats to
align with Bandit-internal terminology (likely a breaking change for consumers
of these formats).
 - Removes color output codes from Bandit configuration when not running in a
TTY, as they are not likely to be supported.
 - Adds severity and confidence to the standard Bandit output for each issue.
 - Adds a few more debug statements and includes a traceback when a plugin
fails, to aid in developing plugins.
 - Cleans up the command line arguments used with argparse a little.
 - Fixes PEP8 niggles.

 It does not implement filtering by confidence, that will be a future todo.

Change-Id: Ia0f76b96d1e756ce605ce6b0b29eee8bd2830574

Author: Jamie Finnigan

bandit.yaml

@@ -11,9 +11,9 @@ plugin_name_pattern: '*.py'
 #output_colors:
 #    DEFAULT: '\033[0m'
 #    HEADER: '\033[95m'
-#    INFO: '\033[94m'
-#    WARN: '\033[93m'
-#    ERROR: '\033[91m'
+#    LOW: '\033[94m'
+#    MEDIUM: '\033[93m'
+#    HIGH: '\033[91m'
 
 # optional: log format string
 #log_format: "[%(module)s]\t%(levelname)s\t%(message)s"
@@ -46,7 +46,7 @@ profiles:
         include:
             - hardcoded_sql_expressions
 
-blacklist_functions:
+blacklist_calls:
     bad_name_sets:
         - pickle:
             qualnames: [pickle.loads, pickle.load, pickle.Unpickler,
@@ -95,11 +95,11 @@ blacklist_imports:
     bad_import_sets:
         - telnet:
             imports: [telnetlib]
-            level: ERROR
+            level: HIGH
             message: "Telnet is considered insecure. Use SSH or some other encrypted protocol."
         - info_libs:
             imports: [pickle, cPickle, subprocess, Crypto]
-            level: INFO
+            level: LOW
             message: "Consider possible security implications associated with {module} module."
 
 hardcoded_password:

bandit/__init__.py

@@ -24,4 +24,5 @@
 from core import tester  # noqa
 from core import utils  # noqa
 from core.constants import *  # noqa
+from core.objects import *  # noqa
 from core.test_properties import *  # noqa

bandit/bandit.py

@@ -39,12 +39,13 @@ def main():
     parser.add_argument(
         '-a', '--aggregate', dest='agg_type',
         action='store', default='file', type=str,
-        help='group results by (vuln)erability type or (file) it occurs in'
+        choices=['file', 'vuln'],
+        help='group results by vulnerability type or file it occurs in'
     )
     parser.add_argument(
         '-n', '--number', dest='context_lines',
         action='store', default=-1, type=int,
-        help='max number of code lines to display'
+        help='max number of code lines to display for each issue identified'
     )
     parser.add_argument(
         '-c', '--configfile', dest='config_file',

bandit/core/__init__.py

@@ -24,4 +24,5 @@
 import tester  # noqa
 import utils  # noqa
 from constants import *  # noqa
+from objects import *  # noqa
 from test_properties import *  # noqa

bandit/core/config.py

@@ -112,26 +112,33 @@ def _init_progress_increment(self):
     def _init_output_colors(self):
         '''Sets the settings colors
 
-        sets settings['color_xxx'] where xxx is DEFAULT, HEADER, INFO, WARN,
-        ERROR
+        sets settings['color_xxx'] where xxx is DEFAULT, HEADER, LOW, MEDIUM,
+        HIGH
         '''
-        colors = ['HEADER', 'DEFAULT', 'INFO', 'WARN', 'ERROR']
+        colors = ['HEADER', 'DEFAULT', 'LOW', 'MEDIUM', 'HIGH']
         color_settings = dict()
 
+        isatty = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
+
         for color in colors:
-            # grab the default color from constant
-            color_settings[color] = constants.color[color]
-
-            # check if the option has been set in config file
-            options_string = 'output_colors.' + color
-            if self.get_option(options_string):
-                color_string = self.get_option(options_string)
-                # some manipulation is needed because escape string doesn't
-                # come back from yaml correctly
-                if color_string.find('['):
-                    right_half = color_string[color_string.find('['):]
-                    left_half = '\033'
-                    color_settings[color] = left_half + right_half
+            # if not a TTY, overwrite color codes in configuration
+            if not isatty:
+                color_settings[color] = ""
+            # else read color codes in from the config
+            else:
+                # grab the default color from constant
+                color_settings[color] = constants.color[color]
+
+                # check if the option has been set in config file
+                options_string = 'output_colors.' + color
+                if self.get_option(options_string):
+                    color_string = self.get_option(options_string)
+                    # some manipulation is needed because escape string doesn't
+                    # come back from yaml correctly
+                    if color_string.find('['):
+                        right_half = color_string[color_string.find('['):]
+                        left_half = '\033'
+                        color_settings[color] = left_half + right_half
 
             # update the settings dict with the color value
             settings_string = 'color_' + color

bandit/core/constants.py

@@ -14,7 +14,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from collections import namedtuple
 from distutils.sysconfig import get_python_lib
 import os
 
@@ -23,9 +22,9 @@
 color = {
     'DEFAULT': '\033[0m',
     'HEADER': '\033[95m',
-    'INFO': '\033[94m',
-    'WARN': '\033[93m',
-    'ERROR': '\033[91m',
+    'LOW': '\033[94m',
+    'MEDIUM': '\033[93m',
+    'HIGH': '\033[91m',
 }
 
 # default plugin name pattern
@@ -40,21 +39,14 @@
 # flag/s used to mark lines where identified issues should not be reported
 SKIP_FLAGS = ['nosec', ]
 
-# list severities in ascending order
-SEVERITY = ['INFO', 'WARN', 'ERROR']
-SEVERITY_VALUES = {'INFO': 1, 'WARN': 5, 'ERROR': 10}
+RANKING = ['UNDEFINED', 'LOW', 'MEDIUM', 'HIGH']
+RANKING_VALUES = {'UNDEFINED': 1, 'LOW': 3, 'MEDIUM': 5, 'HIGH': 10}
 
-# add each severity to globals, to allow direct access in module name space
-for sev in SEVERITY:
-    globals()[sev] = sev
+# add each ranking to globals, to allow direct access in module name space
+for rank in RANKING:
+    globals()[rank] = rank
 
-# severity level constants for assignment to individual plugins
-severity_namedtuple = namedtuple('SeverityLevel', 'HIGH MEDIUM LOW')
-SEVERITY_LEVEL = severity_namedtuple(HIGH=10, MEDIUM=5, LOW=0)
-
-# confidence level constants for return from individual plugins
-confidence_namedtuple = namedtuple('ConfidenceLevel', 'HIGH MEDIUM LOW')
-CONFIDENCE_LEVEL = confidence_namedtuple(HIGH=10, MEDIUM=5, LOW=0)
+CONFIDENCE_DEFAULT = 'UNDEFINED'
 
 # A list of values Python considers to be False.
 # These can be useful in tests to check if a value is True or False.

bandit/core/node_visitor.py

@@ -161,7 +161,10 @@ def __init__(self, fname, logger, config, metaast, results, testset,
                  debug):
         self.debug = debug
         self.seen = 0
-        self.score = [0] * len(constants.SEVERITY)
+        self.scores = {
+            'SEVERITY': [0] * len(constants.RANKING),
+            'CONFIDENCE': [0] * len(constants.RANKING)
+        }
         self.fname = fname
         self.logger = logger
         self.config = config
@@ -223,7 +226,7 @@ def visit_FunctionDef(self, node):
         # For all child nodes and any tests run, add this function name to
         # current namespace
         self.namespace = b_utils.namespace_path_join(self.namespace, name)
-        self.update_score(self.tester.run_tests(self.context, 'FunctionDef'))
+        self.update_scores(self.tester.run_tests(self.context, 'FunctionDef'))
         super(BanditNodeVisitor, self).generic_visit(node)
         self.namespace = b_utils.namespace_path_split(self.namespace)[0]
 
@@ -246,7 +249,7 @@ def visit_Call(self, node):
         self.context['qualname'] = qualname
         self.context['name'] = name
 
-        self.update_score(self.tester.run_tests(self.context, 'Call'))
+        self.update_scores(self.tester.run_tests(self.context, 'Call'))
         super(BanditNodeVisitor, self).generic_visit(node)
 
     def visit_Import(self, node):
@@ -264,7 +267,7 @@ def visit_Import(self, node):
                 self.context['import_aliases'][nodename.asname] = nodename.name
             self.context['imports'].add(nodename.name)
             self.context['module'] = nodename.name
-        self.update_score(self.tester.run_tests(self.context, 'Import'))
+        self.update_scores(self.tester.run_tests(self.context, 'Import'))
         super(BanditNodeVisitor, self).generic_visit(node)
 
     def visit_ImportFrom(self, node):
@@ -300,7 +303,7 @@ def visit_ImportFrom(self, node):
             self.context['imports'].add(module + "." + nodename.name)
             self.context['module'] = module
             self.context['name'] = nodename.name
-        self.update_score(self.tester.run_tests(self.context, 'ImportFrom'))
+        self.update_scores(self.tester.run_tests(self.context, 'ImportFrom'))
         super(BanditNodeVisitor, self).generic_visit(node)
 
     def visit_Str(self, node):
@@ -328,14 +331,14 @@ def visit_Str(self, node):
             is_standalone_expr = False
         # if we don't have either one of those, run the test
         if not (is_str or is_standalone_expr):
-            self.update_score(self.tester.run_tests(self.context, 'Str'))
+            self.update_scores(self.tester.run_tests(self.context, 'Str'))
         super(BanditNodeVisitor, self).generic_visit(node)
 
     def visit_Exec(self, node):
         self.context['str'] = 'exec'
 
         self.logger.debug("visit_Exec called (%s)" % ast.dump(node))
-        self.update_score(self.tester.run_tests(self.context, 'Exec'))
+        self.update_scores(self.tester.run_tests(self.context, 'Exec'))
         super(BanditNodeVisitor, self).generic_visit(node)
 
     def visit(self, node):
@@ -364,9 +367,9 @@ def visit(self, node):
         super(BanditNodeVisitor, self).visit(node)
         self.depth -= 1
         self.logger.debug("%s\texiting : %s" % (self.depth, hex(id(node))))
-        return self.score
+        return self.scores
 
-    def update_score(self, score):
+    def update_scores(self, scores):
         '''Score updater
 
         Since we moved from a single score value to a map of scores per
@@ -375,7 +378,10 @@ def update_score(self, score):
         '''
         def add(x, y):
             return x + y
-        self.score = map(add, self.score, score)
+        for score_type in self.scores:
+            self.scores[score_type] = map(
+                add, self.scores[score_type], scores[score_type]
+            )
 
     def process(self, fdata):
         '''Main process loop
@@ -394,4 +400,4 @@ def process(self, fdata):
 
             self.visit(self.statement['node'])
             self.statement = self.stmt_buffer.get_next()
-        return self.score
+        return self.scores

bandit/core/objects.py

@@ -0,0 +1,19 @@
+# -*- coding:utf-8 -*-
+#
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from collections import namedtuple
+
+Issue = namedtuple('Issue', 'severity confidence text')