Merge pull request greenbone#174 from jjnicola/finished-hosts

Extend OSP with finished_hosts to improve resume task.

CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 - Add clean_forgotten_scans(). [#171](https://github.com/greenbone/ospd/pull/171)
+- Extend OSP with finished_hosts to improve resume task.  [#177](https://github.com/greenbone/ospd/pull/177)
 
 ### Changed
 - Set loglevel to debug for some message. [#159](https://github.com/greenbone/ospd/pull/159)

doc/OSP.xml

@@ -178,6 +178,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
       <e>ports</e>
       <e>credentials</e>
       <e>exclude_hosts</e>
+      <e>finished_hosts</e>
     </pattern>
     <ele>
       <name>hosts</name>
@@ -198,7 +199,13 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
     </ele>
     <ele>
       <name>exclude_hosts</name>
-      <summary>One or many hosts to exclude. The list is comma-separated. Each entry can be a IP address, a CIDR notation, a hostname, a IP range. IPs can be v4 or v6. The listed hosts will be set as finished before starting the scan. Each wrapper must handle the exclude hosts.
+      <summary>One or many hosts to exclude. The list is comma-separated. Each entry can be a IP address, a CIDR notation, a hostname, a IP range. IPs can be v4 or v6. Each wrapper must handle the exclude hosts.
+      </summary>
+      <type>string</type>
+    </ele>
+    <ele>
+      <name>finished_hosts</name>
+      <summary>One or many finished hosts to exclude when resuming a task. The list is comma-separated. Each entry can be an IP address, a CIDR notation, a hostname, a IP range. IPs can be v4 or v6. The listed hosts will be set as finished before starting the scan. Each wrapper must handle the finished hosts.
       </summary>
       <type>string</type>
     </ele>
@@ -222,6 +229,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
             <credential>...</credential>
           </credentials>
           <exclude_hosts>192.168.1.10-15</exclude_hosts>
+          <finished_hosts>192.168.1.1-3</finished_hosts>
         </target>
       </e>
     </example>
@@ -1216,6 +1224,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
                 ...
               </credentials>
               <exclude_hosts>192.168.1.10-15</exclude_hosts>
+              <finished_hosts>192.168.1.1-3</finished_hosts>
             </target>
           </targets>
         </start_scan>

ospd/misc.py

@@ -101,6 +101,21 @@ def add_result(
         # Set scan_info's results to propagate results to parent process.
         self.scans_table[scan_id]['results'] = results
 
+    def remove_hosts_from_target_progress(self, scan_id, target, hosts):
+        """Remove a list of hosts from the main scan progress table to avoid
+        the hosts to be included in the calculation of the scan progress"""
+        if not hosts:
+            return
+
+        targets = self.scans_table[scan_id]['target_progress']
+        for host in hosts:
+            if host in targets[target]:
+                del targets[target][host]
+
+        # Set scan_info's target_progress to propagate progresses
+        # to parent process.
+        self.scans_table[scan_id]['target_progress'] = targets
+
     def set_progress(self, scan_id, progress):
         """ Sets scan_id scan's progress. """
 
@@ -225,11 +240,11 @@ def create_scan(self, scan_id='', targets='', options=None, vts=''):
         scan_info = self.data_manager.dict()
         scan_info['results'] = list()
         scan_info['finished_hosts'] = dict(
-            [[target, []] for target, _, _, _ in targets]
+            [[target, []] for target, _, _, _, _ in targets]
         )
         scan_info['progress'] = 0
         scan_info['target_progress'] = dict(
-            [[target, {}] for target, _, _, _ in targets]
+            [[target, {}] for target, _, _, _, _ in targets]
         )
         scan_info['targets'] = targets
         scan_info['vts'] = vts
@@ -275,11 +290,15 @@ def get_target_progress(self, scan_id, target):
         in the target."""
 
         total_hosts = len(target_str_to_list(target))
+        exc_hosts_list = target_str_to_list(
+            self.get_exclude_hosts(scan_id, target)
+        )
+        exc_hosts = len(exc_hosts_list) if exc_hosts_list else 0
         host_progresses = self.scans_table[scan_id]['target_progress'].get(
             target
         )
         try:
-            t_prog = sum(host_progresses.values()) / total_hosts
+            t_prog = sum(host_progresses.values()) / (total_hosts - exc_hosts)
         except ZeroDivisionError:
             LOGGER.error(
                 "Zero division error in %s", self.get_target_progress.__name__
@@ -301,7 +320,7 @@ def get_target_list(self, scan_id):
         """ Get a scan's target list. """
 
         target_list = []
-        for target, _, _, _ in self.scans_table[scan_id]['targets']:
+        for target, _, _, _, _ in self.scans_table[scan_id]['targets']:
             target_list.append(target)
         return target_list
 

ospd/ospd.py

@@ -541,19 +541,24 @@ def process_targets_element(cls, scanner_target):
         target_list = []
         for target in scanner_target:
             exclude_hosts = ''
+            finished_hosts = ''
             ports = ''
             credentials = {}
             for child in target:
                 if child.tag == 'hosts':
                     hosts = child.text
                 if child.tag == 'exclude_hosts':
                     exclude_hosts = child.text
+                if child.tag == 'finished_hosts':
+                    finished_hosts = child.text
                 if child.tag == 'ports':
                     ports = child.text
                 if child.tag == 'credentials':
                     credentials = cls.process_credentials_elements(child)
             if hosts:
-                target_list.append([hosts, ports, credentials, exclude_hosts])
+                target_list.append(
+                    [hosts, ports, credentials, exclude_hosts, finished_hosts]
+                )
             else:
                 raise OspdCommandError('No target to scan', 'start_scan')
 
@@ -578,7 +583,7 @@ def handle_start_scan_command(self, scan_et):
         else:
             scan_targets = []
             for single_target in target_str_to_list(target_str):
-                scan_targets.append([single_target, ports_str, '', ''])
+                scan_targets.append([single_target, ports_str, '', '', ''])
 
         scan_id = scan_et.attrib.get('scan_id')
         if scan_id is not None and scan_id != '' and not valid_uuid(scan_id):
@@ -846,15 +851,28 @@ def calculate_progress(self, scan_id):
         return sum(t_prog.values()) / len(t_prog)
 
     def process_exclude_hosts(self, scan_id, target_list):
-        """ Process the exclude hosts before launching the scans.
-        Set exclude hosts as finished with 100% to calculate
-        the scan progress."""
+        """ Process the exclude hosts before launching the scans."""
 
-        for target, _, _, exclude_hosts in target_list:
+        for target, _, _, exclude_hosts, _ in target_list:
             exc_hosts_list = ''
             if not exclude_hosts:
                 continue
             exc_hosts_list = target_str_to_list(exclude_hosts)
+            self.remove_scan_hosts_from_target_progress(
+                scan_id, target, exc_hosts_list
+            )
+
+    def process_finished_hosts(self, scan_id, target_list):
+        """ Process the finished hosts before launching the scans.
+        Set finished hosts as finished with 100% to calculate
+        the scan progress."""
+
+        for target, _, _, _, finished_hosts in target_list:
+            exc_hosts_list = ''
+            if not finished_hosts:
+                continue
+            exc_hosts_list = target_str_to_list(finished_hosts)
+
             for host in exc_hosts_list:
                 self.set_scan_host_finished(scan_id, target, host)
                 self.set_scan_host_progress(scan_id, target, host, 100)
@@ -870,6 +888,7 @@ def start_scan(self, scan_id, targets, parallel=1):
             raise OspdCommandError('Erroneous targets list', 'start_scan')
 
         self.process_exclude_hosts(scan_id, target_list)
+        self.process_finished_hosts(scan_id, target_list)
 
         for _index, target in enumerate(target_list):
             while len(multiscan_proc) >= parallel:
@@ -932,6 +951,14 @@ def handle_timeout(self, scan_id, host):
             value="{0} exec timeout.".format(self.get_scanner_name()),
         )
 
+    def remove_scan_hosts_from_target_progress(
+        self, scan_id, target, exc_hosts_list
+    ):
+        """ Remove a list of hosts from the main scan progress table."""
+        self.scan_collection.remove_hosts_from_target_progress(
+            scan_id, target, exc_hosts_list
+        )
+
     def set_scan_host_finished(self, scan_id, target, host):
         """ Add the host in a list of finished hosts """
         self.scan_collection.set_host_finished(scan_id, target, host)
@@ -1704,10 +1731,10 @@ def clean_forgotten_scans(self):
             end_time = int(self.get_scan_end_time(scan_id))
             scan_status = self.get_scan_status(scan_id)
 
-            if ((scan_status == ScanStatus.STOPPED or
-                scan_status == ScanStatus.FINISHED) and
-                end_time
-            ):
+            if (
+                scan_status == ScanStatus.STOPPED
+                or scan_status == ScanStatus.FINISHED
+            ) and end_time:
                 stored_time = int(time.time()) - end_time
                 if stored_time > self.scaninfo_store_time * 3600:
                     logger.debug(
@@ -1719,7 +1746,6 @@ def clean_forgotten_scans(self):
                     )
                     self.delete_scan(scan_id)
 
-
     def check_scan_process(self, scan_id):
         """ Check the scan's process, and terminate the scan if not alive. """
         scan_process = self.scan_processes[scan_id]

tests/test_scan_and_result.py

@@ -984,7 +984,7 @@ def test_scan_get_target(self):
         scan_res = response.find('scan')
         self.assertEqual(scan_res.get('target'), 'localhosts,192.168.0.0/24')
 
-    def test_scan_get_exclude_hosts(self):
+    def test_scan_get_finished_hosts(self):
         daemon = DummyWrapper([])
         response = secET.fromstring(
             daemon.handle_command(
@@ -994,8 +994,8 @@ def test_scan_get_exclude_hosts(self):
                 '<targets><target>'
                 '<hosts>192.168.10.20-25</hosts>'
                 '<ports>80,443</ports>'
-                '<exclude_hosts>192.168.10.23-24'
-                '</exclude_hosts>'
+                '<finished_hosts>192.168.10.23-24'
+                '</finished_hosts>'
                 '</target>'
                 '<target><hosts>192.168.0.0/24</hosts>'
                 '<ports>22</ports></target>'

tests/test_ssh_daemon.py

@@ -83,7 +83,7 @@ def test_run_command(self):
         daemon = OSPDaemonSimpleSSH('cert', 'key', 'ca', '10')
         scanid = daemon.create_scan(
             None,
-            [['host.example.com', '80, 443', '', '']],
+            [['host.example.com', '80, 443', '', '', '']],
             dict(port=5, ssh_timeout=15, username_password='dummy:pw'),
             '',
         )
@@ -99,7 +99,7 @@ def test_run_command_legacy_credential(self):
         daemon = OSPDaemonSimpleSSH('cert', 'key', 'ca', '10')
         scanid = daemon.create_scan(
             None,
-            [['host.example.com', '80, 443', '', '']],
+            [['host.example.com', '80, 443', '', '', '']],
             dict(port=5, ssh_timeout=15, username='dummy', password='pw'),
             '',
         )
@@ -126,7 +126,7 @@ def test_run_command_new_credential(self):
 
         scanid = daemon.create_scan(
             None,
-            [['host.example.com', '80, 443', cred_dict, '']],
+            [['host.example.com', '80, 443', cred_dict, '', '']],
             dict(port=5, ssh_timeout=15),
             '',
         )
@@ -141,7 +141,7 @@ def test_run_command_no_credential(self):
         daemon = OSPDaemonSimpleSSH('cert', 'key', 'ca', '10')
         scanid = daemon.create_scan(
             None,
-            [['host.example.com', '80, 443', '', '']],
+            [['host.example.com', '80, 443', '', '', '']],
             dict(port=5, ssh_timeout=15),
             '',
         )