Sparse rindex readers: fixing query resume on TileDB cloud. (#2900)

There are two types of incompletes on TileDB Cloud. One where the user's
buffers are not full, but the server side buffers are and one where the
user side buffers are full. In the first case, the query is 'kept alive'
with it's current state. In the second the query is recreated on the
server and all state is lost. This fixes an issue in the second case
where the all ranges tile overlap gets recomputed when the query is
resumed. In this case, the computation needs to take into account the
tile index deserialized from the client.

---
TYPE: IMPROVEMENT
DESC: Sparse rindex readers: fixing query resume on TileDB cloud.

tiledb/sm/query/sparse_index_reader_base.cc

@@ -209,8 +209,6 @@ Status SparseIndexReaderBase::load_initial_data() {
   auto fragment_num = fragment_metadata_.size();
 
   // Make sure there is enough space for tiles data.
-  read_state_.frag_tile_idx_.clear();
-  all_tiles_loaded_.clear();
   read_state_.frag_tile_idx_.resize(fragment_num);
   all_tiles_loaded_.resize(fragment_num);
 
@@ -226,7 +224,9 @@ Status SparseIndexReaderBase::load_initial_data() {
     // This is ok as it is a soft limit and will be taken into consideration
     // later.
     RETURN_NOT_OK(subarray_.precompute_all_ranges_tile_overlap(
-        storage_manager_->compute_tp(), &result_tile_ranges_));
+        storage_manager_->compute_tp(),
+        read_state_.frag_tile_idx_,
+        &result_tile_ranges_));
 
     for (auto frag_result_tile_ranges : result_tile_ranges_) {
       memory_used_result_tile_ranges_ += frag_result_tile_ranges.size() *

tiledb/sm/subarray/subarray.cc

@@ -2159,6 +2159,7 @@ Status Subarray::precompute_tile_overlap(
 
 Status Subarray::precompute_all_ranges_tile_overlap(
     ThreadPool* const compute_tp,
+    std::vector<std::pair<uint64_t, uint64_t>>& frag_tile_idx,
     std::vector<std::vector<std::pair<uint64_t, uint64_t>>>*
         result_tile_ranges) {
   auto timer_se = stats_->start_timer("read_compute_simple_tile_overlap");
@@ -2236,7 +2237,8 @@ Status Subarray::precompute_all_ranges_tile_overlap(
         // contiguity, push a new result tile range.
         uint64_t end = tile_bitmaps[0].size() - 1;
         uint64_t length = 0;
-        for (int64_t t = tile_bitmaps[0].size() - 1; t >= 0; t--) {
+        int64_t min = static_cast<int64_t>(frag_tile_idx[f].first);
+        for (int64_t t = tile_bitmaps[0].size() - 1; t >= min; t--) {
           bool comb = true;
           for (unsigned d = 0; d < dim_num; d++) {
             comb &= (bool)tile_bitmaps[d][t];

tiledb/sm/subarray/subarray.h

@@ -304,10 +304,12 @@ class Subarray {
    * Precomputes the tile overlap with all subarray ranges for all fragments.
    *
    * @param compute_tp The compute thread pool.
+   * @param frag_tile_idx The current tile index, per fragment.
    * @param result_tile_ranges The resulting tile ranges.
    */
   Status precompute_all_ranges_tile_overlap(
       ThreadPool* const compute_tp,
+      std::vector<std::pair<uint64_t, uint64_t>>& frag_tile_idx,
       std::vector<std::vector<std::pair<uint64_t, uint64_t>>>*
           result_tile_ranges);