(#885) Add endpoint to queue generation of all page thumbnails with p…

…rogress reporting

lib/LANraragi/Controller/Api/Archive.pm

@@ -89,6 +89,12 @@ sub update_thumbnail {
     LANraragi::Model::Archive::update_thumbnail( $self, $id );
 }
 
+sub generate_page_thumbnails {
+    my $self = shift;
+    my $id   = check_id_parameter( $self, "generate_page_thumbnails" ) || return;
+    LANraragi::Model::Archive::generate_page_thumbnails( $self, $id );
+}
+
 # Use RenderFile to get the file of the provided id to the client.
 sub serve_file {
 
@@ -162,11 +168,11 @@ sub update_metadata {
     my $self = shift;
     my $id   = check_id_parameter( $self, "update_metadata" ) || return;
 
-    my $title = $self->req->param('title');
-    my $tags  = $self->req->param('tags');
+    my $title   = $self->req->param('title');
+    my $tags    = $self->req->param('tags');
     my $summary = $self->req->param('summary');
 
-    my $res = LANraragi::Model::Archive::update_metadata( $id, $title, $tags, $summary);
+    my $res = LANraragi::Model::Archive::update_metadata( $id, $title, $tags, $summary );
 
     if ( $res eq "" ) {
         render_api_response( $self, "update_metadata" );

lib/LANraragi/Model/Archive.pm

@@ -92,6 +92,88 @@ sub update_thumbnail {
 
 }
 
+sub generate_page_thumbnails {
+
+    my ( $self, $id ) = @_;
+
+    my $force = $self->req->param('force');
+    $force = ( $force && $force eq "true" ) || "0";    # Prevent undef warnings by checking the variable first
+
+    my $logger   = get_logger( "Archives", "lanraragi" );
+    my $thumbdir = LANraragi::Model::Config->get_thumbdir;
+    my $use_hq   = LANraragi::Model::Config->get_hqthumbpages;
+    my $use_jxl  = LANraragi::Model::Config->get_jxlthumbpages;
+    my $format   = $use_jxl ? 'jxl' : 'jpg';
+
+    # Get the number of pages in the archive
+    my $redis = LANraragi::Model::Config->get_redis;
+    my $pages = $redis->hget( $id, "pagecount" );
+
+    my $subfolder = substr( $id, 0, 2 );
+    my $thumbname = "$thumbdir/$subfolder/$id.$format";
+
+    my $should_queue_job = 0;
+
+    for ( my $page = 1; $page <= $pages; $page++ ) {
+        my $thumbname = ( $page - 1 > 0 ) ? "$thumbdir/$subfolder/$id/$page.$format" : "$thumbdir/$subfolder/$id.$format";
+
+        unless ( $force == 0 && -e $thumbname ) {
+            $logger->debug("Thumbnail for page $page doesn't exist (path: $thumbname or force=$force), queueing job.");
+            $should_queue_job = 1;
+            last;
+        }
+    }
+
+    if ($should_queue_job) {
+
+        # Check if a job is already queued for this archive
+        if ( $redis->hexists( $id, "thumbjob" ) ) {
+
+            my $job_id = $redis->hget( $id, "thumbjob" );
+
+            # If the job is pending or running, don't queue a new job and just return this one
+            my $job_state = $self->minion->job($job_id)->info->{state};
+            if ( $job_state eq "active" || $job_state eq "inactive" ) {
+                $self->render(
+                    json => {
+                        operation => "generate_page_thumbnails",
+                        success   => 1,
+                        job       => $job_id
+                    },
+                    status => 202    # 202 Accepted
+                );
+                $redis->quit;
+                return;
+            }
+        }
+
+        # Queue a minion job to generate the thumbnails. Clients can check on its progress through the job ID.
+        my $job_id = $self->minion->enqueue( page_thumbnails => [ $id, $force ] => { priority => 0, attempts => 3 } );
+
+        # Save job in Redis so we can check on it if this endpoint is called again
+        $redis->hset( $id, "thumbjob", $job_id );
+        $self->render(
+            json => {
+                operation => "generate_page_thumbnails",
+                success   => 1,
+                job       => $job_id
+            },
+            status => 202    # 202 Accepted
+        );
+    } else {
+        $self->render(
+            json => {
+                operation => "generate_page_thumbnails",
+                success   => 1,
+                message   => "No job queued, all thumbnails already exist."
+            },
+            status => 200    # 200 OK
+        );
+    }
+
+    $redis->quit;
+}
+
 sub serve_thumbnail {
 
     my ( $self, $id ) = @_;

lib/LANraragi/Utils/Minion.pm

@@ -7,12 +7,12 @@ use Encode;
 use Mojo::UserAgent;
 use Parallel::Loops;
 
-use LANraragi::Utils::Logging qw(get_logger);
-use LANraragi::Utils::Database qw(redis_decode);
-use LANraragi::Utils::Archive qw(extract_thumbnail extract_archive);
-use LANraragi::Utils::Plugins qw(get_downloader_for_url get_plugin get_plugin_parameters use_plugin);
-use LANraragi::Utils::Generic qw(split_workload_by_cpu);
-use LANraragi::Utils::String qw(trim_url);
+use LANraragi::Utils::Logging    qw(get_logger);
+use LANraragi::Utils::Database   qw(redis_decode);
+use LANraragi::Utils::Archive    qw(extract_thumbnail extract_archive);
+use LANraragi::Utils::Plugins    qw(get_downloader_for_url get_plugin get_plugin_parameters use_plugin);
+use LANraragi::Utils::Generic    qw(split_workload_by_cpu);
+use LANraragi::Utils::String     qw(trim_url);
 use LANraragi::Utils::TempFolder qw(get_temp);
 
 use LANraragi::Model::Upload;
@@ -46,6 +46,48 @@ sub add_tasks {
         }
     );
 
+    $minion->add_task(
+        page_thumbnails => sub {
+
+            my ( $job, @args )  = @_;
+            my ( $id,  $force ) = @args;
+
+            my $logger = get_logger( "Minion", "minion" );
+            $logger->debug("Generating page thumbnails for archive $id...");
+
+            # Get the number of pages in the archive
+            my $redis = LANraragi::Model::Config->get_redis;
+            my $pages = $redis->hget( $id, "pagecount" );
+
+            my $use_hq   = LANraragi::Model::Config->get_hqthumbpages;
+            my $thumbdir = LANraragi::Model::Config->get_thumbdir;
+
+            my $use_jxl   = LANraragi::Model::Config->get_jxlthumbpages;
+            my $format    = $use_jxl ? 'jxl' : 'jpg';
+            my $subfolder = substr( $id, 0, 2 );
+
+            # Generate thumbnails for all pages -- Cover should already be handled
+            for ( my $i = 2; $i <= $pages; $i++ ) {
+
+                my $thumbname = "$thumbdir/$subfolder/$id/$i.$format";
+                unless ( $force == 0 && -e $thumbname ) {
+                    $logger->debug("Generating thumbnail for page $i... ($thumbname)");
+                    eval { $thumbname = extract_thumbnail( $thumbdir, $id, $i, $use_hq ); };
+                    if ($@) {
+                        $logger->warn("Error while generating thumbnail: $@");
+                    }
+                }
+
+                # Notify progress so it can be checked via API
+                $job->note( progress => $i, pages => $pages );
+            }
+
+            $redis->hdel( $id, "thumbjob" );
+            $redis->quit;
+            $job->finish;
+        }
+    );
+
     $minion->add_task(
         regen_all_thumbnails => sub {
             my ( $job,      @args )  = @_;

lib/LANraragi/Utils/Routing.pm

@@ -109,6 +109,7 @@ sub apply_routes {
     $public_api->get('/api/archives/:id/download')->to('api-archive#serve_file');
     $public_api->get('/api/archives/:id/page')->to('api-archive#serve_page');
     $public_api->get('/api/archives/:id/files')->to('api-archive#get_file_list');
+    $public_api->post('/api/archives/:id/files/thumbnails')->to('api-archive#generate_page_thumbnails');
     $public_api->post('/api/archives/:id/extract')->to('api-archive#get_file_list');    # Deprecated
     $public_api->put('/api/archives/:id/progress/:page')->to('api-archive#update_progress');
     $public_api->delete('/api/archives/:id/isnew')->to('api-archive#clear_new');

public/js/reader.js

@@ -710,49 +710,56 @@ Reader.initializeArchiveOverlay = function () {
         const page = index + 1;
 
         const thumbCss = (localStorage.cropthumbs === "true") ? "id3" : "id3 nocrop";
+        const thumbnailUrl = `./api/archives/${Reader.id}/thumbnail?page=${page}`;
         const thumbnail = `
             <div class='${thumbCss} quick-thumbnail' page='${index}' style='display: inline-block; cursor: pointer'>
                 <span class='page-number'>Page ${page}</span>
-                <img src="./img/wait_warmly.jpg" id="${index}_thumb" />
+                <img src="${thumbnailUrl}" id="${index}_thumb" />
                 <i id="${index}_spinner" class="fa fa-4x fa-circle-notch fa-spin ttspinner" style="display:flex;justify-content: center; align-items: center;"></i>
             </div>`;
 
-        // Try to load the thumbnail and see if we have to wait for a Minion job (202 vs 200)
-        const thumbnailUrl = `/api/archives/${Reader.id}/thumbnail?page=${page}`;
-
-        const thumbSuccess = function () {
-            // Set image source to the thumbnail
-            $(`#${index}_thumb`).attr("src", thumbnailUrl);
-            $(`#${index}_spinner`).hide();
-        };
-
-        const thumbFail = function () {
-            // If we fail to load the thumbnail, then we'll just show a placeholder
-            $(`#${index}_thumb`).attr("src", "/img/noThumb.png");
-            $(`#${index}_spinner`).hide();
-        };
-
-        fetch(`${thumbnailUrl}&no_fallback=true`, { method: "GET" })
-            .then((response) => {
-                if (response.status === 200) {
-                    thumbSuccess();
-                } else if (response.status === 202) {
-                    // Wait for Minion job to finish
-                    response.json().then((data) => Server.checkJobStatus(
-                        data.job,
-                        false,
-                        () => thumbSuccess(),
-                        () => thumbFail(),
-                    ));
-                } else {
-                    // We don't have a thumbnail for this page
-                    thumbFail();
-                }
-            });
-
         $("#archivePagesOverlay").append(thumbnail);
     }
     $("#archivePagesOverlay").attr("loaded", "true");
+
+    // Queue a single minion job for thumbnails and check on its progress regularly
+    const thumbProgress = function (notes) {
+
+        if (notes.progress === undefined) { return; }
+        for (let index = 0; index < notes.progress; ++index) {
+
+            const page = index + 1;
+            // If the spinner is still visible, update the thumbnail
+            if ($(`#${index}_spinner`).attr("loaded") !== true) {
+
+                // Set image source to the thumbnail
+                const thumbnailUrl = `./api/archives/${Reader.id}/thumbnail?page=${page}&cachebust=${Date.now()}`;
+                $(`#${index}_thumb`).attr("src", thumbnailUrl);
+                $(`#${index}_spinner`).attr("loaded", true);
+                $(`#${index}_spinner`).hide();
+            }
+        }
+
+    };
+
+    fetch(`/api/archives/${Reader.id}/files/thumbnails`, { method: "POST" })
+        .then((response) => {
+            if (response.status === 200) {
+                // Thumbnails are already generated, there's nothing to do. Very nice!
+                $(".ttspinner").hide();
+                return;
+            }
+            if (response.status === 202) { 
+                // Check status and update progress 
+                response.json().then((data) => Server.checkJobStatus(
+                    data.job,
+                    false,
+                    (data) => thumbProgress(data.notes), // call progress callback one last time to ensure all thumbs are loaded
+                    () => LRR.showErrorToast("The page thumbnailing job didn't conclude properly. Your archive might be corrupted."),
+                    thumbProgress
+                ));
+            }
+        });
 };
 
 Reader.changePage = function (targetPage) {

tools/Documentation/api-documentation/archive-api.md

@@ -223,6 +223,51 @@ This parameter does nothing if the image already exists. (You will get the image
 {% endswagger-response %}
 {% endswagger %}
 
+{% swagger baseUrl="http://lrr.tvc-16.science" path="/api/archives/:id/files/thumbnails" method="post" summary="Queue extraction of page thumbnails" %}
+{% swagger-description %}
+Create thumbnails for every page of a given Archive. This endpoint will queue generation of the thumbnails in the background.  
+If all thumbnails are detected as already existing, the call will return HTTP code 200.  
+This endpoint can be called multiple times -- If a thumbnailing job is already in progress for the given ID, it'll just give you the ID for that ongoing job.
+{% endswagger-description %}
+
+{% swagger-parameter name="id" type="string" required="true" in="path" %}
+ID of the Archive to process.
+{% endswagger-parameter %}
+{% swagger-parameter name="force" type="boolean" required="false" in="query" %}
+Whether to force regeneration of all thumbnails even if they already exist.  
+{% endswagger-parameter %}
+
+{% swagger-response status="202" description="The thumbnails are queued for extraction. You can use `/api/minion/:jobid` to track progress, by looking at `notes->progress` and `notes->pages`." %}
+```javascript
+{
+  "job": 2429,
+  "operation": "generate_page_thumbnails",
+  "success": 1
+}
+```
+{% endswagger-response %}
+
+{% swagger-response status="200" description="If the thumbnails were already extracted and force=0." %}
+```javascript
+{
+  "message": "No job queued, all thumbnails already exist.",
+  "operation": "generate_page_thumbnails",
+  "success": 1
+}
+```
+{% endswagger-response %}
+
+{% swagger-response status="400" description="" %}
+```javascript
+{
+    "operation": "generate_page_thumbnails",
+    "error": "No archive ID specified.",
+    "success": 0
+}
+```
+{% endswagger-response %}
+{% endswagger %}
+
 {% swagger baseUrl="http://lrr.tvc-16.science" path="/api/archives/:id/download" method="get" summary="Download an Archive" %}
 {% swagger-description %}
 Download an Archive from the server.

tools/Documentation/api-documentation/minion-api.md

@@ -8,6 +8,7 @@ description: Control the built-in Minion Job Queue.
 {% swagger-description %}
 For a given Minion job ID, check whether it succeeded or failed.  
 Minion jobs are ran for various occasions like thumbnails, cache warmup and handling incoming files.  
+For some jobs, you can check the `notes` field for progress information. Look at https://docs.mojolicious.org/Minion/Guide#Job-progress for more information.  
 {% endswagger-description %}
 
 {% swagger-parameter name="id" type="string" required="true" in="path" %}
@@ -19,6 +20,9 @@ ID of the Job.
 {
   "state": "finished",
   "task": "handle_upload",
+  "notes": {
+    "example_note": "This note could contain the name of the upload, for example."
+  },
   "error": null
 }