Fix DBSCAN allocates rbc index even if deactivated (#5859)

This fix addresses DBSCAN always reserving memory for the RBC index no matter whether the approach is activated or not. With the latest RBC index optimizations the memory consumption has increased which makes this more painful.

* Keep RBC index empty when not used
* Confirm index type sufficiently large to address data in RBC index

FYI, @tfeher

Authors:
  - Malte Förster (https://github.com/mfoerste4)

Approvers:
  - Dante Gama Dessavre (https://github.com/dantegd)

URL: #5859

cpp/src/dbscan/runner.cuh

@@ -178,6 +178,15 @@ std::size_t run(const raft::handle_t& handle,
     return size;
   }
 
+  if (sparse_rbc_mode && (std::size_t)D > static_cast<std::size_t>(MAX_LABEL / N)) {
+    CUML_LOG_WARN(
+      "You are using an index type of size (%d bytes) which is not sufficient "
+      "to represent the input dimensions in the RBC index. "
+      "Consider using a larger index type. Falling back to BRUTE_FORCE strategy.",
+      (int)sizeof(Index_));
+    sparse_rbc_mode = false;
+  }
+
   // partition the temporary workspace needed for different stages of dbscan.
 
   std::vector<Index_> batchadjlen(n_batches);
@@ -211,7 +220,7 @@ std::size_t run(const raft::handle_t& handle,
   raft::neighbors::ball_cover::BallCoverIndex<Index_, Type_f, Index_, Index_>* rbc_index_ptr =
     nullptr;
   raft::neighbors::ball_cover::BallCoverIndex<Index_, Type_f, Index_, Index_> rbc_index(
-    handle, x, N, D, metric);
+    handle, x, sparse_rbc_mode ? N : 0, sparse_rbc_mode ? D : 0, metric);
 
   if (sparse_rbc_mode) {
     raft::neighbors::ball_cover::build_index(handle, rbc_index);