Version 204

help/changelog.html

@@ -8,10 +8,26 @@
 		<div class="content">
 			<h3>changelog</h3>
 			<ul>
+				<li><h3>version 204</h3></li>
+				<ul>
+					<li>current, deleted, pending, and petitioned mappings are now stored on service-separated dynamic tables</li>
+					<li>any tag service deletion or reset should now only take a few seconds</li>
+					<li>fixed a physical file deletion bug that seems to have affected (some?) versions of linux and os x</li>
+					<li>reintroduced a revamped 'clear orphans' database->maintenance routine for the client, which will delete or move orphaned files and delete orphaned thumbnails. run this if you are on os x or linux</li>
+					<li>improved misc file deletion code</li>
+					<li>fixed some multiplatform path conversion, and generally improved how paths are stored and retrieved</li>
+					<li>the network application of a parent tag to all files with the child tag now happens serverside, and only on petitioner-approval of a tag parent relationship. it still occurs locally to the uploader, but is now wrapped up in the parent commit</li>
+					<li>fixed some numtags search logic</li>
+					<li>fixed an issue when mixing system:age with any tag-based predicate</li>
+					<li>media_results db calls no longer require a file domain</li>
+					<li>'hide inbox and archive preds if either has no files' now defaults to off</li>
+					<li>added an option to disable OpenCV for gif rendering</li>
+					<li>the selection tags panel now initialises with the correct tag domain (it was previously always starting as 'all known tags', which is not always true for session pages)</li>
+				</ul>
 				<li><h3>version 203</h3></li>
 				<ul>
 					<li>thumbnail resize now happens on the fly--feel free to change it as often as you like, it takes no time at all</li>
-					<li>added 'lexicopgrahic (grouped by namespace)' tag sorting to the selection tags box</li>
+					<li>added 'lexicographic (grouped by namespace)' tag sorting to the selection tags box</li>
 					<li>added an option to disable OpenCV for static images under the media options page</li>
 					<li>panning with the shortcut keys now pans by a twelfth of the media size or canvas size, whichever is smaller</li>
 					<li>cleared the analyze timestamp cache for both client and server, which will force a reanalyze of the new db files on the next db maintenance run</li>

include/ClientCaches.py

@@ -695,7 +695,7 @@ def _GetInfo( self, share_key ):
 
             info[ 'hashes_set' ] = set( hashes )
 
-            media_results = self._controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
+            media_results = self._controller.Read( 'media_results', hashes )
 
             info[ 'media_results' ] = media_results
 

include/ClientController.py

@@ -1009,15 +1009,13 @@ def THREADDoFileQuery( self, query_key, search_context ):
 
         query_hash_ids = self.Read( 'file_query_ids', search_context )
 
-        service_key = search_context.GetFileServiceKey()
-
         media_results = []
 
         for sub_query_hash_ids in HydrusData.SplitListIntoChunks( query_hash_ids, 256 ):
 
             if query_key.IsCancelled(): return
 
-            more_media_results = self.Read( 'media_results_from_ids', service_key, sub_query_hash_ids )
+            more_media_results = self.Read( 'media_results_from_ids', sub_query_hash_ids )
 
             media_results.extend( more_media_results )
 

include/ClientDaemons.py

@@ -92,7 +92,7 @@ def DAEMONDownloadFiles( controller ):
 
             job_key.SetVariable( 'popup_text_1', 'downloading ' + HydrusData.ConvertIntToPrettyString( num_downloads - len( successful_hashes ) ) + ' files from repositories' )
 
-            ( media_result, ) = controller.Read( 'media_results', CC.COMBINED_FILE_SERVICE_KEY, ( hash, ) )
+            ( media_result, ) = controller.Read( 'media_results', ( hash, ) )
 
             service_keys = list( media_result.GetLocationsManager().GetCurrent() )
 

include/ClientData.py

@@ -347,7 +347,16 @@ def key( tag ):
 
             if ':' in tag:
 
-                return tag.split( ':', 1 )
+                ( namespace, subtag ) = tag.split( ':', 1 )
+
+                if namespace == '':
+
+                    return ( '{', subtag )
+
+                else:
+
+                    return ( namespace, subtag )
+
 
             else:
 
@@ -436,13 +445,14 @@ def _InitialiseDefaults( self ):
 
         self._dictionary[ 'booleans' ][ 'apply_all_parents_to_all_services' ] = False
         self._dictionary[ 'booleans' ][ 'apply_all_siblings_to_all_services' ] = False
-        self._dictionary[ 'booleans' ][ 'filter_inbox_and_archive_predicates' ] = True
+        self._dictionary[ 'booleans' ][ 'filter_inbox_and_archive_predicates' ] = False
         self._dictionary[ 'booleans' ][ 'waiting_politely_text' ] = False
 
         self._dictionary[ 'booleans' ][ 'show_thumbnail_title_banner' ] = True
         self._dictionary[ 'booleans' ][ 'show_thumbnail_page' ] = True
 
         self._dictionary[ 'booleans' ][ 'disable_cv_for_static_images' ] = False
+        self._dictionary[ 'booleans' ][ 'disable_cv_for_gifs' ] = False
 
         self._dictionary[ 'noneable_integers' ] = {}
 

include/ClientFiles.py

@@ -124,9 +124,9 @@ def GetExportPath():
 
     options = HydrusGlobals.client_controller.GetOptions()
 
-    path = options[ 'export_path' ]
+    portable_path = options[ 'export_path' ]
 
-    if path is None:
+    if portable_path is None:
 
         path = os.path.join( os.path.expanduser( '~' ), 'hydrus_export' )
 
@@ -135,10 +135,10 @@ def GetExportPath():
             os.makedirs( path )
 
 
-
-    path = os.path.normpath( path ) # converts slashes to backslashes for windows
-
-    path = HydrusPaths.ConvertPortablePathToAbsPath( path )
+    else:
+        
+        path = HydrusPaths.ConvertPortablePathToAbsPath( portable_path )
+        
 
     return path
 
@@ -466,7 +466,7 @@ def DoWork( self ):
 
                     sub_query_hash_ids = query_hash_ids[ last_i : i ]
 
-                    more_media_results = HydrusGlobals.client_controller.Read( 'media_results_from_ids', CC.LOCAL_FILE_SERVICE_KEY, sub_query_hash_ids )
+                    more_media_results = HydrusGlobals.client_controller.Read( 'media_results_from_ids', sub_query_hash_ids )
 
                     media_results.extend( more_media_results )
 

include/ClientGUI.py

@@ -515,6 +515,41 @@ def _CheckFileIntegrity( self ):
 
 
 
+    def _ClearOrphans( self ):
+
+        text = 'This will iterate through every file in your database\'s file storage, removing any it does not expect to be there. It may take some time.'
+
+        with ClientGUIDialogs.DialogYesNo( self, text, yes_label = 'do it', no_label = 'forget it' ) as dlg:
+
+            if dlg.ShowModal() == wx.ID_YES:
+
+                text = 'What would you like to do with the orphaned files? Note that all orphaned thumbnails will be deleted.'
+
+                with ClientGUIDialogs.DialogYesNo( self, text, title = 'Choose what do to with the orphans.', yes_label = 'move them somewhere', no_label = 'delete them' ) as dlg_2:
+
+                    result = dlg_2.ShowModal()
+
+                    if result == wx.ID_YES:
+
+                        with wx.DirDialog( self, 'Select location.' ) as dlg_3:
+
+                            if dlg_3.ShowModal() == wx.ID_OK:
+
+                                path = HydrusData.ToUnicode( dlg_3.GetPath() )
+
+                                self._controller.Write( 'clear_orphans', path )
+
+
+
+                    elif result == wx.ID_NO:
+
+                        self._controller.Write( 'clear_orphans' )
+
+
+
+
+
+
     def _CloseCurrentPage( self, polite = True ):
 
         selection = self._notebook.GetSelection()
@@ -890,6 +925,7 @@ def database():
             submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'rebalance_client_files' ), p( '&Rebalance File Storage' ), p( 'Move your files around your chosen storage directories until they satisfy the weights you have set in the options.' ) )
             submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'regenerate_thumbnails' ), p( '&Regenerate All Thumbnails' ), p( 'Delete all thumbnails and regenerate from original files.' ) )
             submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'file_integrity' ), p( '&Check File Integrity' ), p( 'Review and fix all local file records.' ) )
+            submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'clear_orphans' ), p( '&Clear Orphans' ), p( 'Clear out surplus files that have found their way into the database.' ) )
             submenu.Append( ClientCaches.MENU_EVENT_ID_TO_ACTION_CACHE.GetPermanentId( 'check_db_integrity' ), p( 'Check Database Integrity' ) )
 
             menu.AppendMenu( CC.ID_NULL, p( '&Maintenance' ), submenu )
@@ -1241,13 +1277,11 @@ def do_it():
 
                         if len( initial_hashes ) > 0:
 
-                            file_service_key = management_controller.GetKey( 'file_service' )
-
                             initial_media_results = []
 
                             for group_of_inital_hashes in HydrusData.SplitListIntoChunks( initial_hashes, 256 ):
 
-                                more_media_results = self._controller.Read( 'media_results', file_service_key, group_of_inital_hashes )
+                                more_media_results = self._controller.Read( 'media_results', group_of_inital_hashes )
 
                                 initial_media_results.extend( more_media_results )
 
@@ -2304,6 +2338,7 @@ def EventMenu( self, event ):
             elif command == 'backup_service': self._BackupService( data )
             elif command == 'check_db_integrity': self._CheckDBIntegrity()
             elif command == 'clear_caches': self._controller.ClearCaches()
+            elif command == 'clear_orphans': self._ClearOrphans()
             elif command == 'close_page': self._CloseCurrentPage()
             elif command == 'db_profile_mode':
 
@@ -3473,7 +3508,7 @@ def EventBooruOpenSearch( self, event ):
 
             for ( name, text, timeout, ( num_hashes, hashes, share_key ) ) in self._booru_shares.GetSelectedClientData():
 
-                media_results = self._controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
+                media_results = self._controller.Read( 'media_results', hashes )
 
                 self._controller.pub( 'new_page_query', CC.LOCAL_FILE_SERVICE_KEY, initial_media_results = media_results )
 

include/ClientGUICommon.py

@@ -4527,7 +4527,7 @@ def EventShowFilesButton( self, event ):
 
         hashes = self._job_key.GetVariable( 'popup_files' )
 
-        media_results = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, hashes )
+        media_results = HydrusGlobals.client_controller.Read( 'media_results', hashes )
 
         HydrusGlobals.client_controller.pub( 'new_page_query', CC.LOCAL_FILE_SERVICE_KEY, initial_media_results = media_results )
 

include/ClientGUIDialogsManage.py

@@ -4363,6 +4363,7 @@ def __init__( self, parent ):
             self._animation_start_position = wx.SpinCtrl( self, min = 0, max = 100 )
 
             self._disable_cv_for_static_images = wx.CheckBox( self, label = '' )
+            self._disable_cv_for_gifs = wx.CheckBox( self, label = '' )
 
             self._mime_media_viewer_panel = ClientGUICommon.StaticBox( self, 'media viewer mime handling' )
 
@@ -4391,6 +4392,7 @@ def __init__( self, parent ):
             self._fit_to_canvas.SetValue( HC.options[ 'fit_to_canvas' ] )
             self._animation_start_position.SetValue( int( HC.options[ 'animation_start_position' ] * 100.0 ) )
             self._disable_cv_for_static_images.SetValue( self._new_options.GetBoolean( 'disable_cv_for_static_images' ) )
+            self._disable_cv_for_gifs.SetValue( self._new_options.GetBoolean( 'disable_cv_for_gifs' ) )
 
             gridbox = wx.FlexGridSizer( 0, 2 )
 
@@ -4421,6 +4423,9 @@ def __init__( self, parent ):
             gridbox.AddF( wx.StaticText( self, label = 'Disable OpenCV for static images: ' ), CC.FLAGS_MIXED )
             gridbox.AddF( self._disable_cv_for_static_images, CC.FLAGS_MIXED )
 
+            gridbox.AddF( wx.StaticText( self, label = 'Disable OpenCV for gifs: ' ), CC.FLAGS_MIXED )
+            gridbox.AddF( self._disable_cv_for_gifs, CC.FLAGS_MIXED )
+
             vbox.AddF( gridbox, CC.FLAGS_EXPAND_PERPENDICULAR )
             vbox.AddF( self._mime_media_viewer_panel, CC.FLAGS_EXPAND_PERPENDICULAR )
 
@@ -4442,6 +4447,7 @@ def UpdateOptions( self ):
             HC.options[ 'mime_media_viewer_actions' ] = mime_media_viewer_actions
 
             self._new_options.SetBoolean( 'disable_cv_for_static_images', self._disable_cv_for_static_images.GetValue() )
+            self._new_options.SetBoolean( 'disable_cv_for_gifs', self._disable_cv_for_gifs.GetValue() )
 
 
 
@@ -9797,7 +9803,7 @@ def SetMedia( self, media ):
 
             hashes = { hash for hash in itertools.chain.from_iterable( ( m.GetHashes() for m in media ) ) }
 
-            if len( hashes ) > 0: media_results = HydrusGlobals.client_controller.Read( 'media_results', self._file_service_key, hashes )
+            if len( hashes ) > 0: media_results = HydrusGlobals.client_controller.Read( 'media_results', hashes )
             else: media_results = []
 
             # this should now be a nice clean copy of the original media

include/ClientGUIManagement.py

@@ -2485,7 +2485,7 @@ def _ShowHashes( self, hashes ):
 
         file_service_key = self._management_controller.GetKey( 'file_service' )
 
-        with wx.BusyCursor(): media_results = self._controller.Read( 'media_results', file_service_key, hashes )
+        with wx.BusyCursor(): media_results = self._controller.Read( 'media_results', hashes )
 
         panel = ClientGUIMedia.MediaPanelThumbnails( self._page, self._page_key, file_service_key, media_results )
 
@@ -2724,6 +2724,12 @@ def _MakeCurrentSelectionTagsBox( self, sizer ):
 
             t = ClientGUICommon.ListBoxTagsSelectionManagementPanel( tags_box, self._page_key, predicates_callable = self._current_predicates_box.GetPredicates )
 
+            file_search_context = self._management_controller.GetVariable( 'file_search_context' )
+
+            tag_service_key = file_search_context.GetTagServiceKey()
+
+            t.ChangeTagService( tag_service_key )
+
         else:
 
             t = ClientGUICommon.ListBoxTagsSelectionManagementPanel( tags_box, self._page_key )

include/ClientGUIMedia.py

@@ -1017,7 +1017,7 @@ def _ShowSelectionInNewQueryPage( self ):
 
         if hashes is not None and len( hashes ) > 0:
 
-            media_results = HydrusGlobals.client_controller.Read( 'media_results', self._file_service_key, hashes )
+            media_results = HydrusGlobals.client_controller.Read( 'media_results', hashes )
 
             hashes_to_media_results = { media_result.GetHash() : media_result for media_result in media_results }
 

include/ClientImporting.py

@@ -235,7 +235,7 @@ def _WorkOnFiles( self, page_key ):
                     HydrusGlobals.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
 
 
-                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
+                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
 
                 HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )
 
@@ -700,7 +700,7 @@ def _WorkOnFiles( self, page_key ):
                     HydrusGlobals.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
 
 
-                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
+                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
 
                 HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )
 
@@ -1338,7 +1338,7 @@ def _WorkOnFiles( self, page_key ):
 
             if status in ( CC.STATUS_SUCCESSFUL, CC.STATUS_REDUNDANT ):
 
-                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
+                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
 
                 HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )
 
@@ -2630,7 +2630,7 @@ def _WorkOnFiles( self, page_key ):
                     HydrusGlobals.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )
 
 
-                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', CC.LOCAL_FILE_SERVICE_KEY, ( hash, ) )
+                ( media_result, ) = HydrusGlobals.client_controller.Read( 'media_results', ( hash, ) )
 
                 HydrusGlobals.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )
 

include/ClientVideoHandling.py

@@ -2,6 +2,7 @@
 import cv2
 import ClientImageHandling
 import HydrusExceptions
+import HydrusGlobals
 import HydrusImageHandling
 
 if cv2.__version__.startswith( '2' ):
@@ -61,7 +62,9 @@ def __init__( self, path, num_frames, target_resolution ):
         self._num_frames = num_frames
         self._target_resolution = target_resolution
 
-        if cv2.__version__.startswith( '2' ):
+        new_options = HydrusGlobals.client_controller.GetNewOptions()
+
+        if new_options.GetBoolean( 'disable_cv_for_gifs' ) or cv2.__version__.startswith( '2' ):
 
             self._InitialisePIL()
 

include/HydrusConstants.py

@@ -53,7 +53,7 @@
 # Misc
 
 NETWORK_VERSION = 17
-SOFTWARE_VERSION = 203
+SOFTWARE_VERSION = 204
 
 UNSCALED_THUMBNAIL_DIMENSIONS = ( 200, 200 )