Merge branch 'master' of github.com:yelp/detect-secrets into add-audi…

…t-baseline-functionality

.coveragerc

@@ -11,6 +11,9 @@ exclude_lines =
     # Don't complain if non-runnable code isn't run:
     ^if __name__ == ['"]__main__['"]:$
 
+    # Don't complain if performing logic for cross-version functionality
+    ^\s*except ImportError:\b
+
     # Don't complain if tests don't hit defensive assertion code:
     ^\s*raise AssertionError\b
     ^\s*raise NotImplementedError\b

.pre-commit-config.yaml

@@ -9,10 +9,14 @@
     -   id: name-tests-test
         exclude: tests/util
     -   id: flake8
-        args: ['--ignore=E501']
+        args: ['--max-line-length', '100']
         exclude: ^test_data/
 -   repo: https://github.com/asottile/reorder_python_imports
     sha: v0.3.5
     hooks:
     -   id: reorder-python-imports
         language_version: python3
+-   repo: https://github.com/asottile/add-trailing-comma
+    sha: v0.6.4
+    hooks:
+    -   id: add-trailing-comma

detect_secrets/__init__.py

@@ -0,0 +1 @@
+VERSION = '0.8.8'

detect_secrets/core/baseline.py

@@ -34,7 +34,7 @@ def initialize(plugins, exclude_regex=None, rootdir='.'):
         regex = re.compile(exclude_regex, re.IGNORECASE)
         git_files = filter(
             lambda x: not regex.search(x),
-            git_files
+            git_files,
         )
 
     for file in git_files:

detect_secrets/core/potential_secret.py

@@ -57,7 +57,7 @@ def json(self):
             'type': self.type,
             'filename': self.filename,
             'line_number': self.lineno,
-            'hashed_secret': self.secret_hash
+            'hashed_secret': self.secret_hash,
         }
 
         return attributes
@@ -73,7 +73,7 @@ def __ne__(self, other):
 
     def __hash__(self):
         return hash(
-            tuple([getattr(self, x) for x in self.fields_to_compare])
+            tuple([getattr(self, x) for x in self.fields_to_compare]),
         )
 
     def __str__(self):  # pragma: no cover

detect_secrets/core/secrets_collection.py

@@ -10,8 +10,10 @@
 from unidiff import PatchSet
 from unidiff.errors import UnidiffParseError
 
+from detect_secrets import VERSION
 from detect_secrets.core.log import CustomLog
 from detect_secrets.core.potential_secret import PotentialSecret
+from detect_secrets.plugins import initialize_plugin
 
 
 CustomLogObj = CustomLog()
@@ -26,37 +28,15 @@ def __init__(self, plugins=(), exclude_regex=''):
 
         :type exclude_regex: str
         :param exclude_regex: for optional regex for ignored paths.
+
+        :type version: str
+        :param version: version of detect-secrets that SecretsCollection
+            is valid at.
         """
         self.data = {}
         self.plugins = plugins
         self.exclude_regex = exclude_regex
-
-    @classmethod
-    def load_baseline_from_file(cls, filename):
-        """Initialize a SecretsCollection object from file.
-
-        :param filename: string; name of file to load
-        :returns: SecretsCollection
-        :raises: IOError
-        """
-        return cls.load_baseline_from_string(
-            cls._get_baseline_string_from_file(filename)
-        )
-
-    @classmethod
-    def _get_baseline_string_from_file(cls, filename):
-        """Used for mocking, because we can't mock `open` (as it's also
-        used in `scan_file`."""
-        try:
-            with codecs.open(filename, encoding='utf-8') as f:
-                return f.read()
-
-        except (IOError, UnicodeDecodeError):
-            CustomLogObj.getLogger().error(
-                "Unable to open baseline file: %s.", filename
-            )
-
-            raise
+        self.version = VERSION
 
     @classmethod
     def load_baseline_from_string(cls, string):
@@ -85,9 +65,25 @@ def _load_baseline_from_dict(cls, data):
         :raises: IOError
         """
         result = SecretsCollection()
-        if 'results' not in data or 'exclude_regex' not in data:
+        if not all(key in data for key in (
+            'exclude_regex',
+            'plugins_used',
+            'results',
+            'version',
+        )):
             raise IOError
 
+        result.exclude_regex = data['exclude_regex']
+
+        plugins = []
+        for plugin in data['plugins_used']:
+            plugin_classname = plugin.pop('name')
+            plugins.append(initialize_plugin(
+                plugin_classname,
+                **plugin
+            ))
+        result.plugins = tuple(plugins)
+
         for filename in data['results']:
             result.data[filename] = {}
 
@@ -96,21 +92,21 @@ def _load_baseline_from_dict(cls, data):
                     item['type'],
                     filename,
                     item['line_number'],
-                    'will be replaced'
+                    'will be replaced',
                 )
                 secret.secret_hash = item['hashed_secret']
                 result.data[filename][secret] = secret
 
-        result.exclude_regex = data['exclude_regex']
+        result.version = data['version']
 
         return result
 
     def scan_diff(
-            self,
-            diff,
-            baseline_filename='',
-            last_commit_hash='',
-            repo_name=''
+        self,
+        diff,
+        baseline_filename='',
+        last_commit_hash='',
+        repo_name='',
     ):
         """For optimization purposes, our scanning strategy focuses on looking
         at incremental differences, rather than re-scanning the codebase every time.
@@ -160,7 +156,7 @@ def scan_diff(
                         patch_file,
                         plugin,
                         filename,
-                    )
+                    ),
                 )
 
     def scan_file(self, filename, filename_key=None):
@@ -245,6 +241,7 @@ def format_for_baseline_output(self):
             'exclude_regex': self.exclude_regex,
             'plugins_used': plugins_used,
             'results': results,
+            'version': self.version,
         }
 
     def _results_accumulator(self, filename):
@@ -305,7 +302,7 @@ def _extract_secrets_from_patch(self, f, plugin, filename):
                             line.value,
                             line.target_line_no,
                             filename,
-                        )
+                        ),
                     )
 
         return output
@@ -328,7 +325,7 @@ def __str__(self):  # pragma: no cover
         return json.dumps(
             self.json(),
             indent=2,
-            sort_keys=True
+            sort_keys=True,
         )
 
     def __getitem__(self, key):  # pragma: no cover

detect_secrets/core/usage.py

@@ -14,7 +14,8 @@ def __init__(self):
 
     def add_default_arguments(self):
         self.plugins_parser.add_arguments()
-        self._add_verbosity_argument()
+        self._add_verbosity_argument()\
+            ._add_version_argument()
 
     def add_pre_commit_arguments(self):
         return self._add_filenames_argument()\
@@ -30,6 +31,14 @@ def parse_args(self, argv):
 
         return output
 
+    def _add_version_argument(self):
+        self.parser.add_argument(
+            '--version',
+            action='store_true',
+            help='Display version information.',
+        )
+        return self
+
     def _add_verbosity_argument(self):
         self.parser.add_argument(
             '-v',
@@ -69,7 +78,7 @@ def _add_initialize_baseline_argument(self):
         self.parser.add_argument(
             '--exclude',
             nargs=1,
-            help='Pass in regex to specify ignored paths during initialization scan.'
+            help='Pass in regex to specify ignored paths during initialization scan.',
         )
 
         return self
@@ -112,7 +121,7 @@ class PluginDescriptor(namedtuple(
         # Therefore, only populate the default value upon consolidation
         # (rather than relying on argparse default).
         'related_args',
-    ]
+    ],
 )):
     def __new__(cls, related_args=None, **kwargs):
         if not related_args:
@@ -133,15 +142,15 @@ class PluginOptions(object):
             disable_flag_text='--no-hex-string-scan',
             disable_help_text='Disables scanning for hex high entropy strings',
             related_args=[
-                ('--hex-limit', [3],),
+                ('--hex-limit', 3,),
             ],
         ),
         PluginDescriptor(
             classname='Base64HighEntropyString',
             disable_flag_text='--no-base64-string-scan',
             disable_help_text='Disables scanning for base64 high entropy strings',
             related_args=[
-                ('--base64-limit', [4.5],),
+                ('--base64-limit', 4.5,),
             ],
         ),
         PluginDescriptor(
@@ -158,7 +167,7 @@ def __init__(self, parser):
                 'Configure settings for each secret scanning '
                 'ruleset. By default, all plugins are enabled '
                 'unless explicitly disabled.'
-            )
+            ),
         )
 
     def add_arguments(self):
@@ -181,7 +190,7 @@ def consolidate_args(args):
 
         for plugin in PluginOptions.all_plugins:
             arg_name = PluginOptions._convert_flag_text_to_argument_name(
-                plugin.disable_flag_text
+                plugin.disable_flag_text,
             )
 
             # Remove disabled plugins
@@ -200,7 +209,7 @@ def consolidate_args(args):
                     default_value = None
 
                 arg_name = PluginOptions._convert_flag_text_to_argument_name(
-                    flag_name
+                    flag_name,
                 )
 
                 related_args[arg_name] = getattr(args, arg_name)
@@ -210,7 +219,7 @@ def consolidate_args(args):
                     related_args[arg_name] = default_value
 
             active_plugins.update({
-                plugin.classname: related_args
+                plugin.classname: related_args,
             })
 
         args.plugins = active_plugins
@@ -224,13 +233,13 @@ def _add_custom_limits(self):
         self.parser.add_argument(
             '--base64-limit',
             type=self._argparse_minmax_type,
-            nargs=1,
+            nargs='?',
             help=high_entropy_help_text,
         )
         self.parser.add_argument(
             '--hex-limit',
             type=self._argparse_minmax_type,
-            nargs=1,
+            nargs='?',
             help=high_entropy_help_text,
         )
         return self
@@ -250,7 +259,8 @@ def _argparse_minmax_type(self, string):
         value = float(string)
         if value < 0 or value > 8:
             raise argparse.ArgumentTypeError(
-                '%s must be between 0.0 and 8.0' % string)
+                '%s must be between 0.0 and 8.0' % string,
+            )
 
         return value
 

detect_secrets/main.py

@@ -5,6 +5,7 @@
 import json
 import sys
 
+from detect_secrets import VERSION
 from detect_secrets.core import audit
 from detect_secrets.core import baseline
 from detect_secrets.core.log import CustomLog
@@ -25,6 +26,10 @@ def main(argv=None):
     if args.verbose:  # pragma: no cover
         CustomLog.enableDebug(args.verbose)
 
+    if args.version:
+        print(VERSION)
+        return
+
     plugins = initialize_plugins(args.plugins)
 
     if args.scan:
@@ -36,11 +41,11 @@ def main(argv=None):
                 baseline.initialize(
                     plugins,
                     args.exclude,
-                    args.scan
+                    args.scan,
                 ).format_for_baseline_output(),
                 indent=2,
-                sort_keys=True
-            )
+                sort_keys=True,
+            ),
         )
     elif args.audit:
         audit.audit_baseline(args.audit[0])