sos remove -s only removes workflows under current directory

src/sos/__main__.py

@@ -1405,10 +1405,16 @@ def cmd_remove(args, unknown_args):
         # a special case where all file and runtime signatures are removed.
         # no other options are allowed.
         from .signatures import target_signatures, step_signatures, workflow_signatures
-        target_signatures.remove(workflow_signatures.files())
-        step_signatures.remove(workflow_signatures.steps(), global_sig=False)
-        step_signatures.remove(workflow_signatures.steps(), global_sig=True)
-        env.logger.info('All runtime signatures are removed')
+        sig_files = workflow_signatures.files()
+        if sig_files:
+            files = list(set([x[1] for x in sig_files]))
+            num_removed_files = target_signatures.remove_many(files)
+            sig_ids = list(set([x[0] for x in sig_files]))
+            num_removed_local_steps = step_signatures.remove_many(sig_ids, global_sig=False)
+            num_removed_global_steps = step_signatures.remove_many(sig_ids, global_sig=True)
+            env.logger.info(f'Signatures of {num_removed_files} files from {num_removed_local_steps + num_removed_global_steps} substeps are removed.')
+        else:
+            env.logger.info('No signatures are found from workflows executed under the current directory.')
         return
     #
     tracked_files = {os.path.abspath(os.path.expanduser(x))

src/sos/signatures.py

@@ -11,7 +11,6 @@
 
 from .utils import env
 
-
 class TargetSignatures:
     TargetSig = namedtuple('TargetSig', 'mtime size md5')
 
@@ -57,15 +56,36 @@ def set(self, target, mtime: float, size: str, md5: str):
         except sqlite3.DatabaseError as e:
             env.logger.warning(f'Failed to save signature for target {target}: {e}')
 
+    def _num_records(self, cur):
+        try:
+            cur.execute('SELECT COUNT(rowid) FROM targets')
+            return cur.fetchone()[0]
+        except sqlite3.DatabaseError as e:
+            env.logger.warning(f'Failed to number of records for target: {e}')
+            return 0
+
     def remove(self, target):
         try:
             cur = self.conn.cursor()
             cur.execute(
-                'DELETE FROM targets WHERE target=?', (target.target_name(),))
+                    'DELETE FROM targets WHERE target=?', (target.target_name(),))
             self.conn.commit()
         except sqlite3.DatabaseError as e:
             env.logger.warning(f'Failed to remove signature for target {target}: {e}')
 
+    def remove_many(self, targets):
+        try:
+            cur = self.conn.cursor()
+            cnt = self._num_records(cur)
+            cur.executemany(
+                    'DELETE FROM targets WHERE target=?',
+                        [(x,) for x in targets])
+            self.conn.commit()
+            return cnt - self._num_records(cur)
+        except sqlite3.DatabaseError as e:
+            env.logger.warning(f'Failed to remove signature for {len(targets)} targets: {e}')
+            return 0
+
     def clear(self):
         try:
             cur = self.conn.cursor()
@@ -139,6 +159,28 @@ def set(self, step_id: str, signature: dict, global_sig: bool):
         except sqlite3.DatabaseError as e:
             env.logger.warning(f'Failed to set step signature for step {step_id}: {e}')
 
+    def _num_records(self, cur):
+        try:
+            cur.execute('SELECT COUNT(rowid) FROM steps')
+            return cur.fetchone()[0]
+        except sqlite3.DatabaseError as e:
+            env.logger.warning(f'Failed to number of records for steps: {e}')
+            return 0
+
+    def remove_many(self, steps: list, global_sig: bool):
+        try:
+            conn = self.get_conn(global_sig)
+            cur = conn.cursor()
+            cnt = self._num_records(cur)
+            cur.executemany(
+                    'DELETE FROM steps WHERE step_id=?',
+                    [(x,) for x in steps])
+            conn.commit()
+            return cnt - self._num_records(cur)
+        except sqlite3.DatabaseError as e:
+            env.logger.warning(f'Failed to remove signature for {len(steps)} substeps: {e}')
+            return 0
+
     def clear(self, global_sig: bool):
         try:
             conn = self.get_conn(global_sig)
@@ -211,8 +253,8 @@ def files(self):
         '''Listing files related to workflows related to current directory'''
         try:
             cur = self.conn.cursor()
-            cur.execute('SELECT DISTINCT item FROM workflows WHERE entry_type LIKE "%_file"')
-            return [eval(x[0])['filename'] for x in cur.fetchall()]
+            cur.execute('SELECT id, item FROM workflows WHERE entry_type LIKE "%_file"')
+            return [(x[0], eval(x[1])['filename']) for x in cur.fetchall()]
         except sqlite3.DatabaseError as e:
             env.logger.warning(f'Failed to get files from signature database: {e}')
             return []

src/sos/targets.py

@@ -1100,15 +1100,15 @@ def write(self, rebuild=False):
         # successfully write signature, write in workflow runtime info
         for f in self.input_files:
             if isinstance(f, file_target):
-                workflow_signatures.write('input_file', self.step_md5,
+                workflow_signatures.write('input_file', self.sig_id,
                                           f'{{"filename":{str(f)!r},"size":{f.size()},"md5":{f.target_signature()!r}}}')
         for f in self.dependent_files:
             if isinstance(f, file_target):
-                workflow_signatures.write('dependent_file', self.step_md5,
+                workflow_signatures.write('dependent_file', self.sig_id,
                                           f'{{"filename":{str(f)!r},"size":{f.size()},"md5":{f.target_signature()!r}}}')
         for f in self.output_files:
             if isinstance(f, file_target):
-                workflow_signatures.write('output_file', self.step_md5,
+                workflow_signatures.write('output_file', self.sig_id,
                                           f'{{"filename":{str(f)!r},"size":{f.size()},"md5":{f.target_signature()!r}}}')
         return True