New index diff to prevent tree recreation

apply_index_local.py

@@ -1,15 +1,13 @@
 #!/usr/bin/python
 import json
 import os
-import shutil
 import settings
 
 from raven import Client
 
 client = Client(settings.SENTRY_URL)
 
 from mii_common import tools
-from time import sleep
 
 
 def apply_index(path, json_file_name):
@@ -21,20 +19,11 @@ def apply_index(path, json_file_name):
         dict_index = json.loads(input_json)
     client.captureMessage(u'Opened the json file')
     index_path = os.path.join(path, 'Movies', 'Index')
-    if os.path.exists(index_path):
-        shutil.rmtree(index_path)
-    retry = 0
-    while os.path.exists(index_path):
-        retry += 1
-        client.captureMessage(u'Sleeping until deleted')
-        sleep(10)
-        if retry == 5:
-            return
-
     current_path_root = tools.make_dir(index_path)
     tools.dict_apply(current_path_root, dict_index, symlink_method=os.symlink)
     os.remove(json_file_path)
 
+
 if __name__ == '__main__':
     try:
         apply_index(settings.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)

mii_common/tools.py

@@ -1,5 +1,8 @@
 import logging
 import os
+from errno import ENOTDIR
+
+import shutil
 
 logger = logging.getLogger(__name__)
 
@@ -42,6 +45,25 @@ def listdir_abs(parent):
     return [os.path.join(parent, child) for child in os.listdir(parent)]
 
 
+def get_size(file_name):
+    return os.path.getsize(os.path.abspath(file_name))
+
+
+def get_dir_size(dir_name):
+    # TODO : Write unite test for that method
+    return sum([get_size(os.path.join(dir_name, x)) for x in os.listdir(dir_name)]) if os.path.exists(dir_name) else 0
+
+
+def safe_delete(path):
+    if os.path.exists(path):
+        if os.path.islink(path):
+            os.unlink(path)
+        elif os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+
 def dict_apply(path, dictionnary, symlink_method=None):
     '''
     This method expect a dict with any depth where leaf are a list of tuple (name, path) where a symlink is going to be created
@@ -52,22 +74,52 @@ def dict_apply(path, dictionnary, symlink_method=None):
     '''
     if not dictionnary:
         return
+    path_content = set(os.listdir(path))
+    dictionarry_keys = set(dictionnary.keys())
+    to_remove = path_content - dictionarry_keys
+    for remove in to_remove:
+        full_remove = os.path.join(path, remove)
+        safe_delete(full_remove)
+
     for root, leaf in dictionnary.items():
+        full_leaf = os.path.join(path, root)
         if not leaf:
+            safe_delete(full_leaf)
             continue
-
         current_path = make_dir(os.path.join(path, root))
+        current_path_content = set(os.listdir(current_path))
         if isinstance(leaf, list):
             for name, abs_path_to_name in leaf:
-                try:
-                    if not symlink_method:
-                        os.symlink(abs_path_to_name, os.path.join(current_path, name))
-                    else:
-                        symlink_method(abs_path_to_name, os.path.join(current_path, name))
-                except OSError as e:
-                    logger.error(u'Tried to symlink: "%s" to "%s/%s"' % (abs_path_to_name,
-                                                                        current_path,
-                                                                        name))
-                    logger.error(u'Error: %s' % e)
+                new_one = os.path.join(current_path, name)
+                if name not in current_path_content:
+                    try:
+                        if not symlink_method:
+                            os.symlink(abs_path_to_name, new_one)
+                        else:
+                            symlink_method(abs_path_to_name, new_one)
+                    except OSError as e:
+                        logger.error(u'Tried to symlink: "%s" to "%s/%s"' % (abs_path_to_name,
+                                                                             current_path,
+                                                                             name))
+                        logger.error(u'Error: %s' % e)
+                else:
+                    current_path_content = current_path_content.remove(name)
+                    if get_dir_size(abs_path_to_name) != get_dir_size(new_one):
+                        safe_delete(new_one)
+                        try:
+                            if not symlink_method:
+                                os.symlink(abs_path_to_name, new_one)
+                            else:
+                                symlink_method(abs_path_to_name, new_one)
+                        except OSError as e:
+                            logger.error(u'Tried to symlink: "%s" to "%s/%s"' % (abs_path_to_name,
+                                                                                 current_path,
+                                                                                 name))
+                            logger.error(u'Error: %s' % e)
+            if current_path_content:
+                for content in current_path_content:
+                    full_content = os.path.join(current_path, content)
+                    safe_delete(full_content)
+
         else:
             dict_apply(current_path, leaf, symlink_method=symlink_method)

mii_sorter/logic.py

@@ -10,6 +10,7 @@
 from django.conf import settings
 from middleware import mii_cache_wrapper
 from mii_common import tools
+from mii_common.tools import get_size
 from mii_sorter.models import WhatsNew, get_serie_episode, insert_serie_episode, get_movie, insert_movie, insert_report, \
     RegexRenaming, update_whatsnew, SpecialHandling
 from movieinfo import hash_tool as ht
@@ -393,15 +394,6 @@ def create_dir_and_move_movie(self, movie_name, year, imdb_id, filename):
         return False
 
 
-def get_size(file_name):
-    return os.path.getsize(os.path.abspath(file_name))
-
-
-def get_dir_size(dir_name):
-    # TODO : Write unite test for that method
-    return sum([get_size(os.path.join(dir_name, x)) for x in os.listdir(dir_name)]) if os.path.exists(dir_name) else 0
-
-
 def get_info(name):
     regex_res = re.match('^(.+)(20[0-2][0-9]|19[5-9][0-9])', name)
     if regex_res:

tests/test_apply_index.py

@@ -12,28 +12,98 @@
 
 @override_settings(DUMP_INDEX_JSON_FILE_NAME='lol.json')
 class TestApplyIndex(TestMiilibrary):
+    def setUp(self):
+        super().setUp()
+
+    def _dump_index(self, dict_index):
+        index_path = os.path.join(self.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)
+        self.fs.CreateFile(index_path,
+                           contents=json.dumps(dict_index))
+        return index_path
+
     @mock.patch('apply_index_local.json.loads')
     def test_return_immediately(self, loads):
         apply_index(self.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)
         assert not loads.called
 
-    @mock.patch('apply_index_local.tools.dict_apply')
-    def test_apply_index(self, dict_apply):
+    def test_apply_index(self):
         fake_index = {
             'Actor': {
-                'Michael Douglas':
+                'Michal':
                     [
-                        ('Thor.(2011).mkv', self.SOURCE_FOLDER + 'Thor.(2011).mkv'),
-                        ('Thor.(2011).720p.mkv', self.SOURCE_FOLDER + 'Thor.(2011).720p.mkv'),
+                        ('Thor.(2011)', self.SOURCE_FOLDER),
+                        ('Thor.2.(2016)', self.SOURCE_FOLDER),
                     ]
             }
         }
-        index_path = os.path.join(self.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)
-        self.fs.CreateFile(index_path,
-                           contents=json.dumps(fake_index))
+        index_path = self._dump_index(fake_index)
+
         movies = tools.make_dir(os.path.join(self.DESTINATION_FOLDER, 'Movies'))
         tools.make_dir(os.path.join(movies, 'Index'))
         assert os.path.exists(index_path)
         apply_index(self.DESTINATION_FOLDER, 'lol.json')
-        assert dict_apply.called
         assert not os.path.exists(index_path)
+        root = os.path.join(self.DESTINATION_FOLDER, 'Movies', 'Index')
+        assert os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.(2011)'))
+        assert os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.2.(2016)'))
+
+    def test_apply_index_diff(self):
+        fake_index = {
+            'Actor': {
+                'Michal':
+                    [
+                        ('Thor.(2011)', self.SOURCE_FOLDER),
+                        ('Thor.2.(2016)', self.SOURCE_FOLDER),
+                    ]
+            }
+        }
+        self._dump_index(fake_index)
+
+        movies = tools.make_dir(os.path.join(self.DESTINATION_FOLDER, 'Movies'))
+        tools.make_dir(os.path.join(movies, 'Index'))
+
+        apply_index(self.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)
+        root = os.path.join(self.DESTINATION_FOLDER, 'Movies', 'Index')
+
+        assert os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.(2011)'))
+        assert os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.2.(2016)'))
+
+        fake_index = {
+            'Actor': {
+                'Michal':
+                    [
+                        ('Thor.3.(2011)', self.SOURCE_FOLDER),
+                    ]
+            }
+        }
+        self._dump_index(fake_index)
+        apply_index(self.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)
+        assert os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.3.(2011)'))
+        assert not os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.(2011)'))
+        assert not os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.2.(2016)'))
+
+        fake_index = {
+            'Actor': {
+                'Michol':
+                    [
+                        ('Thor.3.(2011)', self.SOURCE_FOLDER),
+                    ]
+            }
+        }
+        self._dump_index(fake_index)
+        apply_index(self.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)
+        assert not os.path.exists(os.path.join(root, 'Actor', 'Michal', 'Thor.3.(2011)'))
+        assert os.path.exists(os.path.join(root, 'Actor', 'Michol', 'Thor.3.(2011)'))
+
+        fake_index = {
+            'Genre': {
+                'Drama':
+                    [
+                        ('Thor.3.(2011)', self.SOURCE_FOLDER),
+                    ]
+            }
+        }
+        self._dump_index(fake_index)
+        apply_index(self.DESTINATION_FOLDER, settings.DUMP_INDEX_JSON_FILE_NAME)
+        assert not os.path.exists(os.path.join(root, 'Actor', 'Michol', 'Thor.3.(2011)'))
+        assert os.path.exists(os.path.join(root, 'Genre', 'Drama', 'Thor.3.(2011)'))

tests/test_common.py

@@ -25,6 +25,7 @@ class TestWithFakeFS(FakeFSTestCase):
     def setUp(self):
         self.setUpPyfakefs()
         self.fs.CreateFile('/t/blah.mkv')
+        os.mkdir('/test/')
 
     def test_delete_dir(self):
         self.fs.CreateFile('/t/e/s/t/blah.mkv')
@@ -44,35 +45,35 @@ def test_listdirabs(self):
         assert listdir_abs('/t') == ['/t/blah.mkv']
 
     def test_dict_apply(self):
-        dict_apply('/', {'a': {'b': [('blah', '/t/blah.mkv')]}})
-        assert os.path.exists('/a/b/blah')
+        dict_apply('/test', {'a': {'b': [('blah', '/t/blah.mkv')]}})
+        assert os.path.exists('/test/a/b/blah')
 
     def test_dict_apply_empty_leaf(self):
-        dict_apply('/', {'a': {'b': [('blah', '/t/blah.mkv')], 'c': []}})
-        assert os.path.exists('/a/b/blah')
-        assert not os.path.exists('/a/c/')
+        dict_apply('/test', {'a': {'b': [('blah', '/t/blah.mkv')], 'c': []}})
+        assert os.path.exists('/test/a/b/blah')
+        assert not os.path.exists('/test/a/c/')
 
     def test_dict_apply_custom_symlink(self):
         def custom_symlink(source, destination):
             destination += '.symlink'
             os.symlink(source, destination)
 
-        dict_apply('/', {'a': {'b': [('blah', '/t/blah.mkv')]}}, symlink_method=custom_symlink)
-        assert os.path.exists('/a/b/blah.symlink')
+        dict_apply('/test/', {'a': {'b': [('blah', '/t/blah.mkv')]}}, symlink_method=custom_symlink)
+        assert os.path.exists('/test/a/b/blah.symlink')
 
     @mock.patch('mii_common.tools.logger')
     def test_dict_apply_custom_symlink_raises_error(self, logger):
         def custom_symlink(source, destination):
             destination += '.symlink'
             raise OSError()
 
-        dict_apply('/', {'a': {'b': [('blah', '/t/blah.mkv')]}}, symlink_method=custom_symlink)
-        assert not os.path.exists('/a/b/blah.symlink')
+        dict_apply('/test', {'a': {'b': [('blah', '/t/blah.mkv')]}}, symlink_method=custom_symlink)
+        assert not os.path.exists('/test/a/b/blah.symlink')
         assert logger.error.called
         assert logger.error.call_count == 2
 
     def test_dict_apply_empty_dict(self):
-        before = os.listdir('/')
-        dict_apply('/', {})
-        after = os.listdir('/')
+        before = os.listdir('/test')
+        dict_apply('/test', {})
+        after = os.listdir('/test')
         assert before == after

tests/test_functionnality.py

@@ -11,7 +11,8 @@
 from mii_indexer.models import MovieRelation
 from mii_indexer.models import MovieTagging, Person
 from mii_sorter.factories import MovieFactory, EpisodeFactory
-from mii_sorter.logic import Sorter, get_dir_size, get_size
+from mii_sorter.logic import Sorter
+from mii_common.tools import get_size, get_dir_size
 from mii_sorter.models import Movie, Episode, Serie, Season, get_serie_episode, WhatsNew
 from mii_unpacker.factories import UnpackedFactory
 from mii_unpacker.logic import RecursiveUnrarer