Reformat filesystem implementation

Clean up AWS code from output folder

ScoutSuite/__init__.py

@@ -2,3 +2,7 @@
 __version__ = '5.0.0-beta'
 
 ERRORS_LIST = []
+
+DEFAULT_REPORT_DIRECTORY = 'scoutsuite-report'
+DEFAULT_REPORT_RESULTS_DIRECTORY = 'scoutsuite-results'
+DEFAULT_INCLUDES_DIRECTORY = 'inc-scoutsuite'

ScoutSuite/__main__.py

@@ -12,7 +12,6 @@
 from ScoutSuite.core.ruleset import Ruleset
 from ScoutSuite.core.server import Server
 from ScoutSuite.output.html import ScoutReport
-from ScoutSuite.output.report_file import ReportFile
 from ScoutSuite.output.utils import get_filename
 from ScoutSuite.providers import get_provider
 from ScoutSuite.providers.base.authentication_strategy_factory import get_authentication_strategy
@@ -154,7 +153,7 @@ async def _run(provider,
                          result_format=result_format)
 
     if database_name:
-        database_file, _ = get_filename(ReportFile.results, report_name, report_dir, extension="db")
+        database_file, _ = get_filename('RESULTS', report_name, report_dir, file_extension="db")
         Server.init(database_file, host_ip, host_port)
         return
 
@@ -173,7 +172,7 @@ async def _run(provider,
         if update:
             print_info('Updating existing data')
             current_run_services = copy.deepcopy(cloud_provider.services)
-            last_run_dict = report.encoder.load_from_file(ReportFile.results)
+            last_run_dict = report.encoder.load_from_file('RESULTS')
             cloud_provider.services = last_run_dict['services']
             for service in cloud_provider.service_list:
                 cloud_provider.services[service] = current_run_services[service]
@@ -182,7 +181,7 @@ async def _run(provider,
     else:
         print_info('Using local data')
         # Reload to flatten everything into a python dictionary
-        last_run_dict = report.encoder.load_from_file(ReportFile.results)
+        last_run_dict = report.encoder.load_from_file('RESULTS')
         for key in last_run_dict:
             setattr(cloud_provider, key, last_run_dict[key])
 
@@ -213,7 +212,7 @@ async def _run(provider,
     if exceptions:
         print_info('Applying exceptions')
         try:
-            exceptions = RuleExceptions(profile, exceptions)
+            exceptions = RuleExceptions(exceptions)
             exceptions.process(cloud_provider)
             exceptions = exceptions.exceptions
         except Exception as e:

ScoutSuite/core/exceptions.py

@@ -4,18 +4,15 @@
 
 from ScoutSuite.core.console import print_debug
 
-from ScoutSuite.output.report_file import ReportFile
 from ScoutSuite.output.result_encoder import JavaScriptEncoder
 
 
 class RuleExceptions(object):
 
-    def __init__(self, profile, file_path=None):
-        self.profile = profile
-        self.file_path = file_path
-        self.jsrw = JavaScriptEncoder(self.profile)
-        self.exceptions = self.jsrw.load_from_file(config_type=ReportFile.exceptions,
-                                                   config_path=self.file_path,
+    def __init__(self, file_path=None):
+        self.jsrw = JavaScriptEncoder()
+        self.exceptions = self.jsrw.load_from_file(file_type='EXCEPTIONS',
+                                                   file_path=file_path,
                                                    first_line=True)
 
     def process(self, cloud_provider):

ScoutSuite/output/data/html/conditionals/json_format.html

@@ -5,5 +5,5 @@
 </script>
 
 <!-- Results that need to be read from js files -->
-<script src="scoutsuite-results/scoutsuite_results.js"></script>
-<script src="scoutsuite-results/scoutsuite_exceptions.js"></script>
+<script src="<!-- RESULTS PLACEHOLDER -->"></script>
+<script src="<!-- EXCEPTIONS PLACEHOLDER -->"></script>

ScoutSuite/output/data/html/report.html

@@ -88,7 +88,7 @@ <h2 id="section_title-h2"></h2>
                 <div class="col-sm-8 w-100 pr-0" id="double-column-right"></div>
             </div>
 
-            <!-- PLACEHOLDER -->
+            <!-- CONTENTS PLACEHOLDER -->
 
             <div class="modal-backdrop fade show" id="please-wait-backdrop"></div>
             <div class="modal fade show" tabindex="-1" role="dialog" id="please-wait-modal" style="padding-right: 17px; display: block;" aria-modal="true">

ScoutSuite/output/html.py

@@ -6,38 +6,37 @@
 import zipfile
 
 import dateutil.tz
-from ScoutSuite.core.console import print_info, print_exception
 
-from ScoutSuite.output.result_encoder import JavaScriptEncoder, SqlLiteEncoder
-from ScoutSuite.output.utils import get_filename, prompt_for_overwrite
+from ScoutSuite import DEFAULT_REPORT_DIRECTORY, DEFAULT_REPORT_RESULTS_DIRECTORY, DEFAULT_INCLUDES_DIRECTORY
 from ScoutSuite import ERRORS_LIST
-from ScoutSuite.output.js import JavaScriptReaderWriter
+from ScoutSuite.core.console import print_info, print_exception
+from ScoutSuite.output.result_encoder import JavaScriptEncoder, SqlLiteEncoder
 from ScoutSuite.output.utils import get_filename, prompt_for_overwrite
-from ScoutSuite.output.report_file import ReportFile
 
 
 class HTMLReport(object):
     """
     Base HTML report
     """
 
-    def __init__(self, profile, report_dir=None, timestamp=False, exceptions=None, result_format=None):
-        exceptions = {} if exceptions is None else exceptions
-        self.report_dir = report_dir if report_dir else ReportFile.directory.value
-        self.profile = profile.replace('/', '_').replace('\\', '_')  # Issue 111
+    def __init__(self, report_name=None, report_dir=None, timestamp=False, exceptions=None, result_format=None):
+
+        self.report_name = report_name
+        self.report_name = report_name.replace('/', '_').replace('\\', '_')  # Issue 111
+        self.report_dir = report_dir if report_dir else DEFAULT_REPORT_DIRECTORY
         self.current_time = datetime.datetime.now(dateutil.tz.tzlocal())
-        if timestamp:
-            self.timestamp = self.current_time.strftime("%Y-%m-%d_%Hh%M%z") if not timestamp else timestamp
-            self.profile = '%s-%s' % (self.profile, self.timestamp)
-        self.exceptions = exceptions
+        self.timestamp = self.current_time.strftime("%Y-%m-%d_%Hh%M%z") if not timestamp else timestamp
+
+        # exceptions = {} if exceptions is None else exceptions
+        self.exceptions = exceptions if exceptions else {}
         self.scout_report_data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
         self.html_data_path = os.path.join(self.scout_report_data_path, 'html')
-        self.exceptions_encoder = JavaScriptEncoder(self.profile, report_dir, timestamp)
+        self.exceptions_encoder = JavaScriptEncoder(self.report_name, report_dir, timestamp)
 
         if result_format == "sqlite":
-            self.encoder = SqlLiteEncoder(self.profile, report_dir, timestamp)
+            self.encoder = SqlLiteEncoder(self.report_name, report_dir, timestamp)
         else:
-            self.encoder = JavaScriptEncoder(self.profile, report_dir, timestamp)
+            self.encoder = JavaScriptEncoder(self.report_name, report_dir, timestamp)
 
     def get_content_from_folder(self, templates_type):
         contents = ''
@@ -53,20 +52,20 @@ def get_content_from_folder(self, templates_type):
         return contents
 
     def get_content_from_file(self, filename):
-            contents = ''
-            template_dir = os.path.join(self.html_data_path, 'conditionals')
-            filename = template_dir + filename
-            try:
-                with open('%s' % filename, 'rt') as f:
-                    contents = contents + f.read()
-            except Exception as e:
-                print_exception('Error reading filename %s: %s' % (filename, e))
-            return contents
+        contents = ''
+        template_dir = os.path.join(self.html_data_path, 'conditionals')
+        filename = template_dir + filename
+        try:
+            with open('%s' % filename, 'rt') as f:
+                contents = contents + f.read()
+        except Exception as e:
+            print_exception('Error reading filename %s: %s' % (filename, e))
+        return contents
 
     def prepare_html_report_dir(self):
         if not os.path.isdir(self.report_dir):
             os.makedirs(self.report_dir)
-        run_results_dir = os.path.join(self.report_dir, 'scoutsuite-results')
+        run_results_dir = os.path.join(self.report_dir, DEFAULT_REPORT_RESULTS_DIRECTORY)
         if not os.path.isdir(run_results_dir):
             os.makedirs(run_results_dir)
         # Copy static 3rd-party files
@@ -75,8 +74,8 @@ def prepare_html_report_dir(self):
         zip_ref.extractall(self.report_dir)
         zip_ref.close()
         # Copy static files
-        inc_scout_dir = os.path.join(self.report_dir, 'inc-scoutsuite')
-        src_inc_scout_dir = os.path.join(self.scout_report_data_path, 'inc-scoutsuite')
+        inc_scout_dir = os.path.join(self.report_dir, DEFAULT_INCLUDES_DIRECTORY)
+        src_inc_scout_dir = os.path.join(self.scout_report_data_path, DEFAULT_INCLUDES_DIRECTORY)
         if os.path.isdir(inc_scout_dir):
             shutil.rmtree(inc_scout_dir)
         shutil.copytree(src_inc_scout_dir, inc_scout_dir)
@@ -87,20 +86,20 @@ class ScoutReport(HTMLReport):
     Scout HTML report
     """
 
-    def __init__(self, provider, profile=None, report_dir=None, timestamp=False, exceptions=None, result_format='json'):
+    def __init__(self, provider, report_name=None, report_dir=None, timestamp=False, exceptions=None,
+                 result_format='json'):
         exceptions = {} if exceptions is None else exceptions
-        self.html_root = ReportFile.report.value
         self.provider = provider
         self.result_format = result_format
 
-        super(ScoutReport, self).__init__(profile, report_dir, timestamp, exceptions, result_format)
+        super(ScoutReport, self).__init__(report_name, report_dir, timestamp, exceptions, result_format)
 
     def save(self, config, exceptions, force_write=False, debug=False):
         self.prepare_html_report_dir()
-        self.encoder.save_to_file(config, ReportFile.results, force_write, debug)
-        self.exceptions_encoder.save_to_file(exceptions, ReportFile.exceptions, force_write, debug)
+        self.encoder.save_to_file(config, 'RESULTS', force_write, debug)
+        self.exceptions_encoder.save_to_file(exceptions, 'EXCEPTIONS', force_write, debug)
         if ERRORS_LIST:
-            self.exceptions_encoder.save_to_file(ERRORS_LIST, ReportFile.errors, force_write, debug=True)
+            self.exceptions_encoder.save_to_file(ERRORS_LIST, 'ERRORS', force_write, debug=True)
         return self.create_html_report(force_write)
 
     def create_html_report(self, force_write):
@@ -113,21 +112,25 @@ def create_html_report(self, force_write):
         # Use all scripts under html/summaries/
         contents += self.get_content_from_folder('summaries')
         contents += self.get_content_from_folder('summaries/%s' % self.provider)
-        new_file, first_line = get_filename(ReportFile.report, self.profile, self.report_dir)
+        new_file, first_line = get_filename('REPORT', self.report_name, self.report_dir)
         print_info('Creating %s' % new_file)
         if prompt_for_overwrite(new_file, force_write):
             if os.path.exists(new_file):
                 os.remove(new_file)
-            with open(os.path.join(self.html_data_path, self.html_root)) as f:
+            with open(os.path.join(self.html_data_path, 'report.html')) as f:
                 with open(new_file, 'wt') as nf:
                     for line in f:
                         newline = line
-                        newline = newline.replace('<!-- PLACEHOLDER -->', contents)
-                        if self.profile != 'default':
-                            newline = newline.replace(ReportFile.results.value,
-                                                      ReportFile.results.value.replace('.js', '-%s.js' % self.profile))
-                            newline = newline.replace(ReportFile.exceptions.value,
-                                                      ReportFile.exceptions.value.replace('.js', '-%s.js' % self.profile))
+                        newline = newline.replace('<!-- CONTENTS PLACEHOLDER -->', contents)
+                        newline = newline.replace('<!-- RESULTS PLACEHOLDER -->',
+                                                  get_filename('RESULTS',
+                                                               self.report_name,
+                                                               self.report_dir,
+                                                               relative_path=True)[0])
+                        newline = newline.replace('<!-- EXCEPTIONS PLACEHOLDER -->',
+                                                  get_filename('EXCEPTIONS',
+                                                               self.report_name,
+                                                               self.report_dir,
+                                                               relative_path=True)[0])
                         nf.write(newline)
         return new_file
-

ScoutSuite/output/js.py

@@ -6,8 +6,8 @@
 
 import dateutil
 
+from ScoutSuite import DEFAULT_REPORT_DIRECTORY
 from ScoutSuite.core.console import print_exception, print_info
-from ScoutSuite.output.report_file import ReportFile
 from ScoutSuite.output.utils import get_filename, prompt_for_overwrite
 
 
@@ -40,17 +40,17 @@ class JavaScriptReaderWriter(object):
     Reader/Writer for JS and JSON files
     """
 
-    def __init__(self, profile, report_dir=None, timestamp=None):
-        # self.metadata = {}
-        self.report_dir = report_dir if report_dir else ReportFile.directory
-        self.profile = profile.replace('/', '_').replace('\\', '_')  # Issue 111
+    def __init__(self, report_name=None, report_dir=None, timestamp=None):
+        self.report_name = report_name
+        if self.report_name:
+            self.report_name = report_name.replace('/', '_').replace('\\', '_')  # Issue 111
+        self.report_dir = report_dir if report_dir else DEFAULT_REPORT_DIRECTORY
         self.current_time = datetime.datetime.now(dateutil.tz.tzlocal())
-        if timestamp != False:
-            self.timestamp = self.current_time.strftime("%Y-%m-%d_%Hh%M%z") if not timestamp else timestamp
+        self.timestamp = self.current_time.strftime("%Y-%m-%d_%Hh%M%z") if not timestamp else timestamp
 
     def load_from_file(self, file_type, config_path=None, first_line=None):
         if not config_path:
-            config_path, first_line = get_filename(file_type, self.profile, self.report_dir)
+            config_path, first_line = get_filename(file_type, self.report_name, self.report_dir)
         with open(config_path, 'rt') as f:
             json_payload = f.readlines()
             if first_line:
@@ -59,7 +59,7 @@ def load_from_file(self, file_type, config_path=None, first_line=None):
         return json.loads(json_payload)
 
     def save_to_file(self, config, file_type, force_write, debug):
-        config_path, first_line = get_filename(file_type, self.profile, self.report_dir)
+        config_path, first_line = get_filename(file_type, self.report_name, self.report_dir)
         print_info('Saving data to %s' % config_path)
         try:
             with self.__open_file(config_path, force_write) as f:

ScoutSuite/output/result_encoder.py

@@ -8,7 +8,6 @@
 from sqlitedict import SqliteDict
 
 from ScoutSuite.core.console import print_exception, print_info
-from ScoutSuite.output.report_file import ReportFile
 from ScoutSuite.output.utils import get_filename, prompt_for_overwrite
 
 
@@ -37,12 +36,13 @@ def default(self, o):
 
 
 class ScoutResultEncoder(object):
-    def __init__(self, profile, report_dir=None, timestamp=None):
-        self.report_dir = report_dir if report_dir else ReportFile.directory.value
-        self.profile = profile.replace('/', '_').replace('\\', '_')  # Issue 111
+    def __init__(self, report_name=None, report_dir=None, timestamp=None):
+        self.report_name = report_name
+        if self.report_name:
+            self.report_name = report_name.replace('/', '_').replace('\\', '_')  # Issue 111
+        self.report_dir = report_dir
         self.current_time = datetime.datetime.now(dateutil.tz.tzlocal())
-        if timestamp:
-            self.timestamp = self.current_time.strftime("%Y-%m-%d_%Hh%M%z") if not timestamp else timestamp
+        self.timestamp = self.current_time.strftime("%Y-%m-%d_%Hh%M%z") if not timestamp else timestamp
 
     @staticmethod
     def to_dict(config):
@@ -52,11 +52,11 @@ def to_dict(config):
 class SqlLiteEncoder(ScoutResultEncoder):
     def load_from_file(self, config_type, config_path=None):
         if not config_path:
-            config_path, _ = get_filename(config_type, self.profile, self.report_dir)
+            config_path, _ = get_filename(config_type, self.report_name, self.report_dir)
         return SqliteDict(config_path, autocommit=True).data
 
     def save_to_file(self, config, config_type, force_write, _debug):
-        config_path, first_line = get_filename(config_type, self.profile, self.report_dir, extension="db")
+        config_path, first_line = get_filename(config_type, self.report_name, self.report_dir, file_extension="db")
         print_info('Saving data to %s' % config_path)
         try:
             with self.__open_file(config_path, force_write) as database:
@@ -95,24 +95,24 @@ class JavaScriptEncoder(ScoutResultEncoder):
     Reader/Writer for JS and JSON files
     """
 
-    def load_from_file(self, config_type, config_path=None, first_line=None):
-        if not config_path:
-            config_path, first_line = get_filename(config_type, self.profile, self.report_dir)
-        with open(config_path, 'rt') as f:
+    def load_from_file(self, file_type, file_path=None, first_line=None):
+        if not file_path:
+            file_path, first_line = get_filename(file_type, self.report_name, self.report_dir)
+        with open(file_path, 'rt') as f:
             json_payload = f.readlines()
             if first_line:
                 json_payload.pop(0)
             json_payload = ''.join(json_payload)
         return json.loads(json_payload)
 
-    def save_to_file(self, config, config_type, force_write, debug):
-        config_path, first_line = get_filename(config_type, self.profile, self.report_dir)
+    def save_to_file(self, content, file_type, force_write, debug):
+        config_path, first_line = get_filename(file_type, self.report_name, self.report_dir)
         print_info('Saving data to %s' % config_path)
         try:
             with self.__open_file(config_path, force_write) as f:
                 if first_line:
                     print('%s' % first_line, file=f)
-                print('%s' % json.dumps(config, indent=4 if debug else None, separators=(',', ': '), sort_keys=True,
+                print('%s' % json.dumps(content, indent=4 if debug else None, separators=(',', ': '), sort_keys=True,
                                         cls=ScoutJsonEncoder), file=f)
         except AttributeError as e:
             # __open_file returned None