Merge pull request #5 from prabhuramachandran/fix-perf

Improve load/save performance significantly.

vixen/directory.py

@@ -1,57 +1,79 @@
 import os
+from os.path import basename, join
 
-from traits.api import HasTraits, Instance, List, Property, Str
+from traits.api import Any, HasTraits, Instance, List, Property, Str
 
-from .media import Media
 
 class File(HasTraits):
     path = Str
     parent = Instance('Directory')
-    name = Property(Str)
-    relpath = Property(Str)
-
-    media = Instance(Media)
+    name = Str
+    relpath = Str
+
+    def __init__(self, path, parent, relpath=None, name=None):
+        self.path = path
+        self.parent = parent
+        self.name = basename(path) if name is None else name
+        if relpath is None:
+            self.relpath = join(parent.relpath, self.name)
+        else:
+            self.relpath = relpath
 
     def __repr__(self):
-        return self.name
-
-    def _get_relpath(self):
-        return os.path.join(self.parent.relpath, self.name)
-
-    def _get_name(self):
-        return os.path.basename(self.path)
+        return 'File(path=%s)' % self.path
 
 
 class Directory(HasTraits):
     path = Str
-    name = Property(Str)
-    relpath = Property(Str)
+    name = Str
+    relpath = Str
     parent = Instance('Directory')
-    directories = List(Instance('Directory'))
+    directories = Property(List(Instance('Directory')))
     files = List(Instance(File))
 
     extensions = List(Str)
 
+    _directories = List(Instance('Directory'))
+    _directory_state = Any
+
     def __getstate__(self):
-        files = [x.path for x in self.files]
-        dirs = [d.__getstate__() for d in self.directories]
-        d = dict(path=self.path, files=files, directories=dirs,
-                 extensions=self.extensions)
-        return d
+        files = [(x.path, x.relpath, x.name) for x in self.files]
+        if self._directory_state is None:
+            dirs = [d.__getstate__() for d in self.directories]
+        else:
+            dirs = self._directory_state
+        result = dict(path=self.path, files=files, directories=dirs,
+                      extensions=self.extensions, relpath=self.relpath,
+                      name=self.name)
+        return result
 
     def __setstate__(self, state):
-        self.trait_setq(path=state['path'])
-        self.files = [File(path=x, parent=self) for x in state['files']]
-        self.extensions = state.get('extensions', [])
-        dirs = []
-        for dir_state in state['directories']:
-            d = Directory(parent=self, extensions=self.extensions)
-            d.__setstate__(dir_state)
-            dirs.append(d)
-        self.directories = dirs
+        extensions = state.get('extensions', [])
+        path = state['path']
+        if 'relpath' in state:
+            self.__dict__.update(dict(
+                relpath=state['relpath'], path=path,
+                name=state['name'], extensions=extensions
+            ))
+            self.files = [
+                File(path=x[0], parent=self, relpath=x[1], name=x[2])
+                for x in state['files']
+            ]
+        else:
+            name = basename(path)
+            if self.parent:
+                relpath = join(self.parent.relpath, self.name)
+            else:
+                relpath = ''
+            self.__dict__.update(dict(
+                path=path, extensions=extensions, name=name, relpath=relpath
+            ))
+            self.files = [File(path=x, parent=self) for x in state['files']]
+
+        self._directory_state = state['directories']
 
     def __repr__(self):
-        return 'Directory(path=%r)'%self.path
+        return 'Directory(path=%r)' % self.path
 
     def refresh(self):
         self._path_changed(self.path)
@@ -60,6 +82,11 @@ def _path_changed(self, new):
         dirs = []
         files = []
         extensions = self.extensions
+        self.name = basename(new)
+        if self.parent is None:
+            self.relpath = ''
+        else:
+            self.relpath = join(self.parent.relpath, self.name)
         try:
             for pth in os.listdir(new):
                 full_path = os.path.join(new, pth)
@@ -75,18 +102,10 @@ def _path_changed(self, new):
 
         except IOError:
             pass
-        self.directories = dirs
+        self._directory_state = None
+        self._directories = dirs
         self.files = files
 
-    def _get_relpath(self):
-        if self.parent is None:
-            return ''
-        else:
-            return os.path.join(self.parent.relpath, self.name)
-
-    def _get_name(self):
-        return os.path.basename(self.path)
-
     def _extensions_changed(self, new, old):
         if len(self.path) > 0:
             if set(new) != set(old):
@@ -95,3 +114,15 @@ def _extensions_changed(self, new, old):
     def _extensions_items_changed(self):
         if len(self.path) > 0:
             self._path_changed(self.path)
+
+    def _get_directories(self):
+        if self._directory_state is not None:
+            dirs = []
+            for dir_state in self._directory_state:
+                d = Directory(parent=self)
+                d.__setstate__(dir_state)
+                dirs.append(d)
+            self._directories = dirs
+            self._directory_state = None
+
+        return self._directories

vixen/html/vixen_ui.html

@@ -230,9 +230,9 @@ <h3 style="margin: 5px;"> View Project: {{viewer.name}}</h3>
    <label v-on:click="pager.select($index)"
    v-bind:class="{'current-index': pager.rel_index == $index,
                                'selected': pager.selected == media}"
-       title="{{media.path}}"
+       title="{{media[1]}}"
        >
-   {{media.file_name}}
+   {{media[0]}}
    </label>
    </div>
    </div>
@@ -431,7 +431,7 @@ <h3 style="margin: 5px;"> View Project: {{viewer.name}}</h3>
       </div>
     </div>
     <div v-else v-if="job_status">
-      <div v-if="project && project.media.length == 0">
+      <div v-if="project && project.number_of_files == 0">
         You need to apply changes once to scan the project before you
         start processing.  Currently no files are available.
       </div>

vixen/media.py

@@ -1,19 +1,78 @@
+from collections import namedtuple
 import datetime
 import os
 
 from jigna.core.wsgi import guess_type
-from traits.api import Date, Dict, HasTraits, Int, Property, Str
+from traits.api import Dict, HasTraits, Int, Long, Property, Str
+from whoosh.util.times import datetime_to_long
+
 
 # Some pre-defined file extensions.
 IMAGE = ['.bmp', '.png', '.gif', '.jpg', '.jpeg', '.svg']
 VIDEO = ['.avi', '.mp4', '.ogv', '.webm', '.flv']
 AUDIO = ['.mp3', '.wav', '.ogg', '.m4a']
 HTML = ['.html', '.htm']
 PDF = ['.pdf']
-## Only add the ones that mimetypes does not detect correctly here.
+# Only add the ones that mimetypes does not detect correctly here.
 TEXT = ['.md', '.rst', '.pyx']
 
 
+MediaData = namedtuple(
+    'MediaData',
+    ['path', 'relpath', 'file_name', 'size', 'ctime', 'ctime_',
+     'mtime', 'mtime_', 'type']
+)
+
+
+def find_type(path):
+    ext = os.path.splitext(path)[1].lower()
+    result = "unknown"
+    if ext in IMAGE:
+        result = "image"
+    elif ext in VIDEO:
+        result = "video"
+    elif ext in AUDIO:
+        result = "audio"
+    elif ext in HTML:
+        result = "html"
+    elif ext in PDF:
+        result = "pdf"
+    elif ext in TEXT:
+        result = "text"
+    else:
+        type, encoding = guess_type(path)
+        if len(type) > 0:
+            if type.startswith('text'):
+                result = 'text'
+            elif type.startswith('video'):
+                result = 'video'
+            elif type.startswith('audio'):
+                result = 'audio'
+            elif type.startswith('image'):
+                result = 'image'
+    return result
+
+
+def get_media_data(path, relpath):
+    if os.path.exists(path):
+        stat = os.stat(path)
+        _mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
+        _ctime = datetime.datetime.fromtimestamp(stat.st_ctime)
+        size = stat.st_size
+        mtime = _mtime.strftime('%d %b %Y %H:%M:%S')
+        ctime = _ctime.strftime('%d %b %Y %H:%M:%S')
+        fname = os.path.basename(path)
+        _ctime = datetime_to_long(_ctime)
+        _mtime = datetime_to_long(_mtime)
+        type = find_type(path)
+        return MediaData(
+            path, relpath, fname, size, ctime, _ctime, mtime,
+            _mtime, type
+        )
+    else:
+        return None
+
+
 class Media(HasTraits):
 
     # The type of the media, typically ("video", "image" or whatever).  This
@@ -48,10 +107,10 @@ class Media(HasTraits):
     # The created time of the file. This and the _mtime are private as we
     # cannot send this to the HTML UI as it is not JSON serializable. However
     # they are is useful for searching through the media.
-    _ctime = Date
+    _ctime = Long
 
     # The modified time of the file.
-    _mtime = Date
+    _mtime = Long
 
     @classmethod
     def from_path(cls, path, relpath):
@@ -60,6 +119,13 @@ def from_path(cls, path, relpath):
         obj.update()
         return obj
 
+    @classmethod
+    def from_data(cls, data, tags):
+        obj = cls()
+        obj.update(data)
+        obj.tags.update(tags)
+        return obj
+
     def to_dict(self):
         return self.__dict__
 
@@ -71,42 +137,22 @@ def flatten(self):
         data.update(tags)
         return data
 
-    def update(self):
-        """Update the metadata from the file.
+    def update(self, data=None, tags=None):
+        """Update the metadata from the file or from the data given.
         """
-        path = self.path
-        if os.path.exists(path):
-            stat = os.stat(path)
-            self._mtime = datetime.datetime.fromtimestamp(stat.st_mtime)
-            self._ctime = datetime.datetime.fromtimestamp(stat.st_ctime)
-            self.size = stat.st_size
-            self.mtime = self._mtime.strftime('%d %b %Y %H:%M:%S')
-            self.ctime = self._ctime.strftime('%d %b %Y %H:%M:%S')
+        if data is None:
+            data = get_media_data(self.path, self.relpath)
+        if data is not None:
+            self.path = data.path
+            self._mtime = data.mtime_
+            self.mtime = data.mtime
+            self._ctime = data.ctime_
+            self.ctime = data.ctime
+            self.size = data.size
+            self.relpath = data.relpath
+            self.type = data.type
+        if tags is not None:
+            self.tags.update(tags)
 
     def _get_file_name(self):
         return os.path.basename(self.path)
-
-    def _path_changed(self, path):
-        ext = os.path.splitext(path)[1].lower()
-        if ext in IMAGE:
-            self.type = "image"
-        elif ext in VIDEO:
-            self.type = "video"
-        elif ext in AUDIO:
-            self.type = "audio"
-        elif ext in HTML:
-            self.type = "html"
-        elif ext in PDF:
-            self.type = "pdf"
-        else:
-            type, encoding = guess_type(path)
-            if len(type) > 0:
-                if type.startswith('text'):
-                    self.type = 'text'
-                else:
-                    self.type = 'unknown'
-            else:
-                if ext in TEXT:
-                    self.type = 'text'
-                else:
-                    self.type = "unknown"

vixen/processor.py

@@ -154,13 +154,14 @@ class CommandFactory(FactoryBase):
 
     name = 'CommandFactory'
 
-    def make_jobs(self, media_seq, project):
+    def make_jobs(self, media_keys, project):
         jobs = []
         ext = self.output_extension
         if len(ext) > 0:
             ext = ext if '.' in ext else '.' + ext
 
-        for media in media_seq:
+        for key in media_keys:
+            media = project.get(key)
             relpath = media.relpath
             if os.path.splitext(relpath.lower())[1] != self.input_extension:
                 continue
@@ -227,10 +228,11 @@ class PythonFunctionFactory(FactoryBase):
 
     name = 'PythonFunctionFactory'
 
-    def make_jobs(self, media_seq, project):
+    def make_jobs(self, media_keys, project):
         self._setup_func()
         jobs = []
-        for media in media_seq:
+        for key in media_keys:
+            media = project.get(key)
             relpath = media.relpath
             if not self._done.get(media.path, False):
                 info = "Processing %s" % media.path
@@ -262,11 +264,12 @@ class TaggerFactory(FactoryBase):
 
     name = 'TaggerFactory'
 
-    def make_jobs(self, media_seq, project):
+    def make_jobs(self, media_keys, project):
         self._setup_tag_types(project)
         jobs = []
 
-        for media in media_seq:
+        for key in media_keys:
+            media = project.get(key)
             path = media.path
             if not os.path.exists(media.path):
                 continue