Improve 'all' tab pagination to handle edge cases

app/controllers/search_controller.rb

@@ -88,48 +88,29 @@ def load_timdex_results
   end
 
   def load_all_results
-    # Fetch results from both APIs in parallel
-    primo_data, timdex_data = fetch_all_data
-
-    # Combine errors from both APIs
-    @errors = combine_errors(primo_data[:errors], timdex_data[:errors])
-
-    # Zipper merge results from both APIs
-    @results = merge_results(primo_data[:results], timdex_data[:results])
-
-    # Use Analyzer for combined pagination calculation
-    @pagination = Analyzer.new(@enhanced_query, timdex_data[:hits], :all,
-                               primo_data[:hits]).pagination
-
-    # Handle primo continuation for high page numbers
-    @show_primo_continuation = primo_data[:show_continuation] || false
-  end
-
-  def fetch_all_data
-    # Parallel fetching from both APIs
-    primo_thread = Thread.new { fetch_primo_data }
-    timdex_thread = Thread.new { fetch_timdex_data }
+    current_page = @enhanced_query[:page] || 1
+    per_page = ENV.fetch('RESULTS_PER_PAGE', '20').to_i
 
-    [primo_thread.value, timdex_thread.value]
-  end
+    service = MergedSearchService.new(
+      enhanced_query: @enhanced_query,
+      active_tab: @active_tab,
+      cache: Rails.cache,
+      primo_fetcher: ->(offset:, per_page:, query: nil) { fetch_primo_data(offset: offset, per_page: per_page) },
+      timdex_fetcher: ->(offset:, per_page:, query: nil) { fetch_timdex_data(offset: offset, per_page: per_page) }
+    )
+    data = service.fetch(page: current_page, per_page: per_page)
 
-  def combine_errors(*error_arrays)
-    all_errors = error_arrays.compact.flatten
-    all_errors.any? ? all_errors : nil
-  end
-
-  def merge_results(primo_results, timdex_results)
-    (primo_results || []).zip(timdex_results || []).flatten.compact
+    @results = data[:results]
+    @errors = data[:errors]
+    @pagination = data[:pagination]
+    @show_primo_continuation = data[:show_primo_continuation]
   end
 
-  def fetch_primo_data
+  def fetch_primo_data(offset: nil, per_page: nil)
+    # Default to current page if not provided
     current_page = @enhanced_query[:page] || 1
-    per_page = if @active_tab == 'all'
-                 ENV.fetch('RESULTS_PER_PAGE', '20').to_i / 2
-               else
-                 ENV.fetch('RESULTS_PER_PAGE', '20').to_i
-               end
-    offset = (current_page - 1) * per_page
+    per_page ||= ENV.fetch('RESULTS_PER_PAGE', '20').to_i
+    offset ||= (current_page - 1) * per_page
 
     # Check if we're beyond Primo API limits before making the request.
     if offset >= Analyzer::PRIMO_MAX_OFFSET
@@ -151,8 +132,9 @@ def fetch_primo_data
     if results.empty?
       docs = primo_response['docs'] if primo_response.is_a?(Hash)
       if docs.nil? || docs.empty?
-        # Only show continuation for pagination scenarios (page > 1), not for searches with no results
-        show_continuation = true if current_page > 1
+        # Only show continuation for pagination scenarios (where offset is present), not for
+        # searches with no results
+        show_continuation = true if offset > 0
       else
         errors = [{ 'message' => 'No more results available at this page number.' }]
       end
@@ -164,19 +146,10 @@ def fetch_primo_data
     { results: [], pagination: {}, errors: handle_primo_errors(e), show_continuation: false, hits: 0 }
   end
 
-  def fetch_timdex_data
-    # For all tab, modify query to use half page size
-    if @active_tab == 'all'
-      per_page = ENV.fetch('RESULTS_PER_PAGE', '20').to_i / 2
-      page = @enhanced_query[:page] || 1
-      from_offset = ((page - 1) * per_page).to_s
-
-      query_builder = QueryBuilder.new(@enhanced_query)
-      query = query_builder.query
-      query['from'] = from_offset
-    else
-      query = QueryBuilder.new(@enhanced_query).query
-    end
+  def fetch_timdex_data(offset: nil, per_page: nil)
+    query = QueryBuilder.new(@enhanced_query).query
+    query['from'] = offset.to_s if offset
+    query['size'] = per_page.to_s if per_page
 
     response = query_timdex(query)
     errors = extract_errors(response)
@@ -223,7 +196,8 @@ def query_timdex(query)
 
   def query_primo(per_page, offset)
     # We generate unique cache keys to avoid naming collisions.
-    cache_key = generate_cache_key(@enhanced_query)
+    # Include per_page and offset in the cache key to ensure pagination works correctly.
+    cache_key = generate_cache_key(@enhanced_query.merge(per_page: per_page, offset: offset))
 
     Rails.cache.fetch("#{cache_key}/primo", expires_in: 12.hours) do
       primo_search = PrimoSearch.new(@enhanced_query[:tab])

app/models/merged_search_paginator.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# MergedSearchPaginator encapsulates stateless merged pagination logic for combining two API result sets.
+# It calculates the merge plan, API offsets, and merges the results for a given page.
+class MergedSearchPaginator
+  attr_reader :primo_total, :timdex_total, :current_page, :per_page
+
+  def initialize(primo_total:, timdex_total:, current_page:, per_page:)
+    @primo_total = primo_total
+    @timdex_total = timdex_total
+    @current_page = current_page
+    @per_page = per_page
+  end
+
+  # Returns an array of :primo and :timdex symbols for the merged result order on this page
+  def merge_plan
+    total_results = primo_total + timdex_total
+    start_index = (current_page - 1) * per_page
+    end_index = [start_index + per_page, total_results].min
+    plan = []
+    primo_used = 0
+    timdex_used = 0
+    i = 0
+    while i < end_index
+      if primo_used < primo_total && (timdex_used >= timdex_total || primo_used <= timdex_used)
+        source = :primo
+        primo_used += 1
+      elsif timdex_used < timdex_total
+        source = :timdex
+        timdex_used += 1
+      end
+      plan << source if i >= start_index
+      i += 1
+    end
+    plan
+  end
+
+  # Returns [primo_offset, timdex_offset] for the start of this page
+  def api_offsets
+    start_index = (current_page - 1) * per_page
+    primo_offset = 0
+    timdex_offset = 0
+    i = 0
+    while i < start_index
+      if primo_offset < primo_total && (timdex_offset >= timdex_total || primo_offset <= timdex_offset)
+        primo_offset += 1
+      elsif timdex_offset < timdex_total
+        timdex_offset += 1
+      else
+        break
+      end
+      i += 1
+    end
+    [primo_offset, timdex_offset]
+  end
+
+  # Merges two result arrays according to the merge plan
+  def merge_results(primo_results, timdex_results)
+    merged = []
+    primo_idx = 0
+    timdex_idx = 0
+    merge_plan.each do |source|
+      if source == :primo
+        merged << primo_results[primo_idx] if primo_idx < primo_results.length
+        primo_idx += 1
+      else
+        merged << timdex_results[timdex_idx] if timdex_idx < timdex_results.length
+        timdex_idx += 1
+      end
+    end
+    merged
+  end
+end