Requery thumbnail generation cursors as needed

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Changes
 - Log more when saving annotations ([#1525](../../pull/1525))
-- Thumbnail generation jobs are less blocking ([#1528](../../pull/1528))
+- Thumbnail generation jobs are less blocking ([#1528](../../pull/1528), [#1530](../../pull/1530))
 
 ### Bug Fixes
 - Annotations are sometimes paged when they shouldn't be ([#1529](../../pull/1529))

girder/girder_large_image/rest/large_image_resource.py

@@ -26,6 +26,7 @@
 import time
 
 import cherrypy
+import pymongo
 from girder_jobs.constants import JobStatus
 from girder_jobs.models.job import Job
 
@@ -34,7 +35,7 @@
 from girder.api import access
 from girder.api.describe import Description, autoDescribeRoute, describeRoute
 from girder.api.rest import Resource
-from girder.constants import TokenScope
+from girder.constants import SortDir, TokenScope
 from girder.exceptions import RestException
 from girder.models.file import File
 from girder.models.item import Item
@@ -122,7 +123,7 @@ def createThumbnailsJob(job):
     thread.start()
 
 
-def createThumbnailsJobThread(job):
+def createThumbnailsJobThread(job):  # noqa
     """
     Create thumbnails for all of the large image items.
 
@@ -160,7 +161,9 @@ def createThumbnailsJobThread(job):
     pool = concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
     try:
         # Get a cursor with the list of images
-        items = Item().find({'largeImage.fileId': {'$exists': True}})
+        query = {'largeImage.fileId': {'$exists': True}}
+        sort = [('_id', SortDir.ASCENDING)]
+        items = Item().find(query, sort=sort)
         if hasattr(items, 'count'):
             status['items'] = items.count()
         status['specs'] = len(spec)
@@ -173,7 +176,14 @@ def createThumbnailsJobThread(job):
             # be exhausted before we are done.
             while len(tasks) < concurrency * 4 and nextitem is not None:
                 tasks.append(pool.submit(createThumbnailsJobTask, nextitem, spec))
-                nextitem = cursorNextOrNone(items)
+                try:
+                    nextitem = cursorNextOrNone(items)
+                except pymongo.CursorNotFound:
+                    # If the process takes long enough, the cursor is removed.
+                    # In this case, redo the query and keep going.
+                    items = Item().find(query, sort=sort)
+                if nextitem is not None:
+                    query['_id'] = {'$gt': nextitem['_id']}
             # Wait a short time or until the oldest task is complete
             try:
                 tasks[0].result(0.1)