[Python][ML] Fix and re-enable rdataloader xgboost tutorial

tree/ml/inc/ROOT/ML/RClusterLoader.hxx

@@ -172,6 +172,7 @@ private:
    bool fIsFiltered{false};
    bool fSplitDiscovered{false};
    std::size_t fAccumulatedFilteredForTrain{0};
+   std::size_t fAccumulatedFilteredForVal{0};
 
 public:
    RClusterLoader(std::vector<ROOT::RDF::RNode> &rdfs, const std::vector<std::string> &cols,
@@ -232,9 +233,19 @@ public:
          std::mt19937 g(fSetSeed);
          std::uniform_int_distribution<int> coin(0, 1);
 
+         std::size_t cumulativeEntries = 0;
+         std::size_t currentCumulativeTrain = 0;
+         // We iterate over clusters and accumulate the entry counts to assign training and validation sizes
+         // proportionally to the cluster size. Filtered clusters have varying sizes, so instead of calculating
+         // the training size as a fraction of each cluster's size independently, we take into account
+         // the cumulative counts of previous clusters in each calculation.
          for (const RClusterRange &c : fAllClusters) {
             const std::size_t sz = c.GetNumEntries();
-            const std::size_t trainSz = static_cast<std::size_t>((1.0f - fValidationSplit) * sz);
+            cumulativeEntries += sz;
+            const std::size_t targetCumulativeTrain =
+               static_cast<std::size_t>(cumulativeEntries * (1.0f - fValidationSplit));
+            const std::size_t trainSz = targetCumulativeTrain - currentCumulativeTrain;
+            currentCumulativeTrain = targetCumulativeTrain;
             const std::size_t valSz = sz - trainSz;
 
             // Randomly assign prefix or suffix to training
@@ -369,25 +380,11 @@ public:
 
          ROOT::RDF::RNode &rdf = fRdfs[rdfIdx];
 
-         // Fill data and collect raw entry indices that pass the filter
          std::vector<ULong64_t> rdfEntries;
          rdfEntries.reserve(endRow - startRow);
 
-         RClusterLoaderFunctor<Args...> loader(dest, fNumChunkCols, fVecSizes, fVecPadding, 0, rowOffset);
          ROOT::Internal::RDF::ChangeBeginAndEndEntries(rdf, startRow, endRow);
-
-         std::vector<std::string> colsWithEntry;
-         colsWithEntry.reserve(fCols.size() + 1);
-         colsWithEntry.push_back("rdfentry_");
-         colsWithEntry.insert(colsWithEntry.end(), fCols.begin(), fCols.end());
-
-         rdf.Foreach(
-            [&](ULong64_t entry, const Args &...cols) {
-               rdfEntries.push_back(entry);
-               loader(cols...);
-            },
-            colsWithEntry);
-
+         rdf.Foreach([&](ULong64_t entry) { rdfEntries.push_back(entry); }, {"rdfentry_"});
          ROOT::Internal::RDF::ChangeBeginAndEndEntries(rdf, 0, fRdfSizes[rdfIdx]);
 
          const std::size_t totalFiltered = rdfEntries.size();
@@ -396,9 +393,11 @@ public:
          }
          std::sort(rdfEntries.begin(), rdfEntries.end());
 
-         const std::size_t trainRemaining = fNumTrainingEntries - fAccumulatedFilteredForTrain;
-         const std::size_t trainCount =
-            std::min(static_cast<std::size_t>(totalFiltered * (1.0f - fValidationSplit)), trainRemaining);
+         const std::size_t cumulativeFiltered =
+            fAccumulatedFilteredForTrain + fAccumulatedFilteredForVal + totalFiltered;
+         const std::size_t targetCumulativeTrain =
+            std::min(static_cast<std::size_t>(cumulativeFiltered * (1.0f - fValidationSplit)), fNumTrainingEntries);
+         const std::size_t trainCount = targetCumulativeTrain - fAccumulatedFilteredForTrain;
          const std::size_t valCount = totalFiltered - trainCount;
 
          bool trainIsPrefix = true;
@@ -436,6 +435,11 @@ public:
             fValidationClusters.push_back({rdfIdx, valStart, valEnd, valCount});
 
          fAccumulatedFilteredForTrain += trainCount;
+         fAccumulatedFilteredForVal += valCount;
+
+         if (trainCount > 0)
+            LoadClusterInto(dest, rdfIdx, trainStart, trainEnd, rowOffset);
+
          return trainCount;
       }
 

tree/ml/inc/ROOT/ML/RFlat2DMatrix.hxx

@@ -23,6 +23,13 @@ struct RFlat2DMatrix {
 
    const float *GetData() const { return fRVec.data(); }
 
+   ROOT::RVecF ReleaseData()
+   {
+      fRows = 0;
+      fCols = 0;
+      return std::move(fRVec);
+   }
+
    // Used in the pythonization
    std::pair<std::size_t, std::size_t> GetShape() const { return {fRows, fCols}; }
 

tutorials/CMakeLists.txt

@@ -118,12 +118,6 @@ endif()
 # functionality is also covered by rf617 (the multidimensional case).
 list(APPEND roofit_veto roofit/roofit/rf615_simulation_based_inference.py)
 
-# TODO: fix the problem and re-enable the tutorial test.
-# This tutorial intermittently produces corrupted (zero/negative) sample weights
-# when RDataLoader is used with a filtered multi-cluster RDataFrame causing
-# XGBoost's positive-weight assertion to fail sporadically in the CI.
-list(APPEND dataframe_veto machine_learning/ml_dataloader_XGBoost.py)
-
 if (NOT dataframe)
     # RDataFrame
     list(APPEND dataframe_veto analysis/dataframe/*.C analysis/dataframe/*.py)

tutorials/machine_learning/ml_dataloader_Higgs_Classification.py

@@ -142,7 +142,7 @@ def forward(self, x):
 
 # num_features must be calculated manually since the train.training_columns includes condensed vector columns.
 # Vector columns are lazily expanded while receiving batches, unless eager_loading is enabled.
-num_features = sum(max_vec_sizes.values()) + len([0 for i in train.train_columns if i not in max_vec_sizes])
+num_features = sum(max_vec_sizes.values()) + len([0 for i in train.feature_columns if i not in max_vec_sizes])
 
 torch.manual_seed(set_seed)
 hidden_layers = [60, 60]

tutorials/machine_learning/ml_dataloader_PyTorch.py

@@ -33,7 +33,7 @@
 gen_train, gen_validation = dl.train_test_split(test_size=0.3)
 
 # Get a list of the columns used for training
-input_columns = gen_train.train_columns
+input_columns = gen_train.feature_columns
 num_features = len(input_columns)
 
 

tutorials/machine_learning/ml_dataloader_TensorFlow.py

@@ -46,7 +46,7 @@
 validation_batches_per_epoch = ds_valid.num_batches
 
 # Get a list of the columns used for training
-input_columns = ds_train.train_columns
+input_columns = ds_train.feature_columns
 num_features = len(input_columns)
 
 ##############################################################################

tutorials/machine_learning/ml_dataloader_XGBoost.py

@@ -57,7 +57,7 @@ def load_data():
 
     loader = ROOT.Experimental.ML.RDataLoader(
         [rdf_sig, rdf_bkg],
-        columns=variables + ["label", "weight"],
+        columns=variables,
         target="label",
         weights="weight",
         batch_size=num_all,  # Load all data in one batch