Merge pull request #14 from vixen-project/remove-clean-button

BUG: Fix rescan to correctly remove deleted files.

vixen/html/vixen_ui.html

@@ -158,10 +158,8 @@ <h3 style="margin: 5px;"> View Project: {{viewer.name}}</h3>
     </div>
 
     This project has {{viewer.project.number_of_files}} files.
-    <button v-on:click="busy_call(viewer, 'rescan')" id="rescan"
+    <button v-on:click="rescan()" id="rescan"
             title="Rescan project for new files.">Rescan</button>
-    <button v-on:click="busy_call(viewer, 'clean')" id="clean"
-            title="Clean deleted file entries from project.">Clean</button>
     <br>
 
     <input v-model="viewer.search" style="width:60%" placeholder="Search"
@@ -584,12 +582,22 @@ <h3 style="margin: 5px;"> View Project: {{viewer.name}}</h3>
               threaded: threaded,
               busy_call: busy_call,
               remove_project: function(project) {
-                  var m = "This will remove all saved information about this project, are "+
-                          "you sure you want to delete it?"
+                  var m = "This will remove all saved information about this "+
+                          "project, are you sure you want to delete it?";
                   if (confirm(m)) {
                       this.ui.remove(project);
                   }
               },
+              rescan: function() {
+                  var m = "Warning: this will remove entries for any deleted "+
+                          "or moved files. If you wish to retain those, "+
+                          "please first export the current tags to a CSV "+
+                          "file and then proceed. "+
+                          "Are you sure you want to rescan?";
+                  if (confirm(m)) {
+                      this.busy_call(this.viewer, 'rescan');
+                  }
+              },
               halt: function() {
                   this.ui.halt();
                   var m = "ViXeN has been stopped, please close this tab/window.";

vixen/project.py

@@ -310,11 +310,31 @@ def get(self, relpath):
             self._media[relpath] = media
             return media
 
-    def remove(self, relpath):
-        """Given the relative path of some media, remove it from the
+    def remove(self, relpaths):
+        """Given a list of relative path of some media, remove them from the
         database.
         """
-        index = self._relpath2index[relpath]
+        relpath2index = self._relpath2index
+        indices = [(x, relpath2index[x]) for x in relpaths]
+        for relpath, index in sorted(indices, reverse=True):
+            last = len(relpath2index) - 1
+            if index == last:
+                self._delete_record(last, relpath)
+            else:
+                self._replace_with_last_record(index, last)
+                self._delete_record(last, relpath)
+
+    def _replace_with_last_record(self, index, last):
+        _data = self._data
+        _tag_data = self._tag_data
+        for key in MediaData._fields:
+            _data[key][index] = _data[key][last]
+        for key in self._tag_data:
+            _tag_data[key][index] = _tag_data[key][last]
+        last_relpath = _data['relpath'][last]
+        self._relpath2index[last_relpath] = index
+
+    def _delete_record(self, index, relpath):
         for key in MediaData._fields:
             del self._data[key][index]
         for key in self._tag_data:
@@ -345,18 +365,19 @@ def _get_media_attr(self, index, attr):
     def clean(self):
         """Scan the project and remove any dead entries.
 
-        This is useful when you remove or rename files. Rescan by default does
-        not delete any entries for missing files leaving invalid entries in the
-        database.
-
+        This is useful when you remove or rename files. This does not refresh
+        the directory tree or set the number of files. It simply cleans up the
+        db of files that no longer exist.
         """
+        logger.info('Cleaning project: %s', self.name)
         root_path = self.path
-        for rpath in list(self._relpath2index.keys()):
+        to_remove = []
+        relpath2index = self._relpath2index
+        for rpath in list(relpath2index.keys()):
             fname = os.path.join(root_path, rpath)
             if not os.path.exists(fname):
-                self.remove(rpath)
-        self.root.refresh()
-        self.number_of_files = len(self._relpath2index)
+                to_remove.append(rpath)
+        self.remove(to_remove)
 
     def export_csv(self, fname, cols=None):
         """Export metadata to a csv file.  If `cols` are not specified,
@@ -558,6 +579,7 @@ def search(self, q):
 
     def refresh(self):
         logger.info('Refreshing project: %s', self.name)
+        self.clean()
         self.scan(refresh=True)
 
     # #### Private protocol ################################################

vixen/tests/test_project.py

@@ -127,18 +127,18 @@ def test_load_should_restore_saved_state(self):
         self.assertEqual(p.get('root.txt')._mtime,
                          p1.get('root.txt')._mtime)
 
-    def test_clean_removes_non_existing_file_entries(self):
+    def test_refresh_removes_non_existing_file_entries(self):
         # Given
         p = Project(name='test', path=self.root)
         p.scan()
         # When
-        m = p.get('root.txt')
+        relpath = 'hello.py'
+        m = p.get(relpath)
         os.remove(m.path)
-        p.clean()
+        p.refresh()
 
         # Then
-        relpath = 'root.txt'
-        self.assertEqual(p.number_of_files, 4)
+        #self.assertEqual(p.number_of_files, 4)
         self.assertFalse(p.has_media(relpath))
         self.assertFalse(relpath in p._media)
         self.assertFalse(relpath in p._relpath2index)
@@ -147,7 +147,11 @@ def test_clean_removes_non_existing_file_entries(self):
         for key in p._tag_data:
             self.assertEqual(len(p._tag_data[key]), 4)
         files = [x.name for x in p.root.files]
-        self.assertTrue('root.txt' not in files)
+        self.assertTrue(relpath not in files)
+        # Check if the database is consistent.
+        for rp in p._relpath2index.keys():
+            m = p.get(rp)
+            self.assertEqual(m.relpath, rp)
 
     def test_export_to_csv_with_unicode(self):
         # Given

vixen/tests/test_vixen.py

@@ -548,7 +548,7 @@ def setUp(self):
         self.ui = ui
         self.p = p
 
-    def test_clean_handles_removed_files(self):
+    def test_rescan_handles_removed_files(self):
         # Given
         ui, p = self.ui, self.p
         viewer = ui.viewer
@@ -558,7 +558,7 @@ def test_clean_handles_removed_files(self):
         os.remove(os.path.join(self.root, 'root.txt'))
 
         # When
-        viewer.clean()
+        viewer.rescan()
 
         # Then
         self.assertEqual(p.number_of_files, 4)

vixen/vixen.py

@@ -408,14 +408,6 @@ class ProjectViewer(HasTraits):
 
     type = Enum("unknown", "image", "video", "audio")
 
-    def clean(self):
-        with self.ui.busy():
-            proj = self.project
-            if proj is not None:
-                proj.clean()
-                self.current_dir = proj.root
-                self._current_dir_changed(proj.root)
-
     def go_to_parent(self):
         if self.parent is not None and not self.is_searching:
             self.current_dir = self.parent