First version of the new parameter "tree_interaction_constraints""

include/LightGBM/config.h

@@ -571,6 +571,14 @@ struct Config {
   // desc = any two features can only appear in the same branch only if there exists a constraint containing both features
   std::string interaction_constraints = "";
 
+  // desc = controls which features can appear in the same tree
+  // desc = by default interaction constraints are disabled, to enable them you can specify
+  // descl2 = for CLI, lists separated by commas, e.g. ``[0,1,2],[2,3]``
+  // descl2 = for Python-package, list of lists, e.g. ``[[0, 1, 2], [2, 3]]``
+  // descl2 = for R-package, list of character or numeric vectors, e.g. ``list(c("var1", "var2", "var3"), c("var3", "var4"))`` or ``list(c(1L, 2L, 3L), c(3L, 4L))``. Numeric vectors should use 1-based indexing, where ``1L`` is the first feature, ``2L`` is the second feature, etc
+  // desc = any two features can only appear in the same tree only if there exists a constraint containing both features
+  std::string tree_interaction_constraints = "";
+
   // alias = verbose
   // desc = controls the level of LightGBM's verbosity
   // desc = ``< 0``: Fatal, ``= 0``: Error (Warning), ``= 1``: Info, ``> 1``: Debug
@@ -1126,6 +1134,7 @@ struct Config {
   static const std::unordered_set<std::string>& parameter_set();
   std::vector<std::vector<double>> auc_mu_weights_matrix;
   std::vector<std::vector<int>> interaction_constraints_vector;
+  std::vector<std::vector<int>> tree_interaction_constraints_vector;
   static const std::unordered_map<std::string, std::string>& ParameterTypes();
   static const std::string DumpAliases();
 
@@ -1135,6 +1144,7 @@ struct Config {
   std::string SaveMembersToString() const;
   void GetAucMuWeights();
   void GetInteractionConstraints();
+  void GetTreeInteractionConstraints();
 };
 
 inline bool Config::GetString(

include/LightGBM/tree.h

@@ -158,6 +158,10 @@ class Tree {
   /*! \brief Get features on leaf's branch*/
   inline std::vector<int> branch_features(int leaf) const { return branch_features_[leaf]; }
 
+  std::set<int> tree_features() const {
+    return tree_features_;
+  }
+
   inline double split_gain(int split_idx) const { return split_gain_[split_idx]; }
 
   inline double internal_value(int node_idx) const {
@@ -520,6 +524,10 @@ class Tree {
   bool track_branch_features_;
   /*! \brief Features on leaf's branch, original index */
   std::vector<std::vector<int>> branch_features_;
+
+  /*! \brief Features used by the tree, original index */
+  std::set<int> tree_features_;
+
   double shrinkage_;
   int max_depth_;
   /*! \brief Tree has linear model at each leaf */
@@ -579,7 +587,9 @@ inline void Tree::Split(int leaf, int feature, int real_feature,
     branch_features_[num_leaves_] = branch_features_[leaf];
     branch_features_[num_leaves_].push_back(split_feature_[new_node_idx]);
     branch_features_[leaf].push_back(split_feature_[new_node_idx]);
+    tree_features_.insert(split_feature_[new_node_idx]);
   }
+
 }
 
 inline double Tree::Predict(const double* feature_values) const {

src/io/config.cpp

@@ -232,13 +232,21 @@ void Config::GetAucMuWeights() {
 }
 
 void Config::GetInteractionConstraints() {
-  if (interaction_constraints == "") {
+  if (interaction_constraints.empty()) {
     interaction_constraints_vector = std::vector<std::vector<int>>();
   } else {
     interaction_constraints_vector = Common::StringToArrayofArrays<int>(interaction_constraints, '[', ']', ',');
   }
 }
 
+void Config::GetTreeInteractionConstraints() {
+  if (tree_interaction_constraints.empty()) {
+    tree_interaction_constraints_vector = std::vector<std::vector<int>>();
+  } else {
+    tree_interaction_constraints_vector = Common::StringToArrayofArrays<int>(tree_interaction_constraints, '[', ']', ',');
+  }
+}
+
 void Config::Set(const std::unordered_map<std::string, std::string>& params) {
   // generate seeds by seed.
   if (GetInt(params, "seed", &seed)) {
@@ -269,6 +277,8 @@ void Config::Set(const std::unordered_map<std::string, std::string>& params) {
 
   GetInteractionConstraints();
 
+  GetTreeInteractionConstraints();
+
   // sort eval_at
   std::sort(eval_at.begin(), eval_at.end());
 

src/io/config_auto.cpp

@@ -246,6 +246,7 @@ const std::unordered_set<std::string>& Config::parameter_set() {
   "cegb_penalty_feature_coupled",
   "path_smooth",
   "interaction_constraints",
+  "tree_interaction_constraints",
   "verbosity",
   "input_model",
   "output_model",
@@ -488,6 +489,8 @@ void Config::GetMembersFromString(const std::unordered_map<std::string, std::str
 
   GetString(params, "interaction_constraints", &interaction_constraints);
 
+  GetString(params, "tree_interaction_constraints", &tree_interaction_constraints);
+
   GetInt(params, "verbosity", &verbosity);
 
   GetString(params, "input_model", &input_model);
@@ -722,6 +725,7 @@ std::string Config::SaveMembersToString() const {
   str_buf << "[cegb_penalty_feature_coupled: " << Common::Join(cegb_penalty_feature_coupled, ",") << "]\n";
   str_buf << "[path_smooth: " << path_smooth << "]\n";
   str_buf << "[interaction_constraints: " << interaction_constraints << "]\n";
+  str_buf << "[tree_interaction_constraints: " << tree_interaction_constraints << "]\n";
   str_buf << "[verbosity: " << verbosity << "]\n";
   str_buf << "[saved_feature_importance_type: " << saved_feature_importance_type << "]\n";
   str_buf << "[use_quantized_grad: " << use_quantized_grad << "]\n";
@@ -922,6 +926,144 @@ const std::unordered_map<std::string, std::vector<std::string>>& Config::paramet
     {"num_gpu", {}},
   });
   return map;
+const std::string Config::DumpAliases() {
+  std::stringstream str_buf;
+  str_buf << "{";
+  str_buf << "\"config\": [\"config_file\"], ";
+  str_buf << "\"task\": [\"task_type\"], ";
+  str_buf << "\"objective\": [\"objective_type\", \"app\", \"application\", \"loss\"], ";
+  str_buf << "\"boosting\": [\"boosting_type\", \"boost\"], ";
+  str_buf << "\"data\": [\"train\", \"train_data\", \"train_data_file\", \"data_filename\"], ";
+  str_buf << "\"valid\": [\"test\", \"valid_data\", \"valid_data_file\", \"test_data\", \"test_data_file\", \"valid_filenames\"], ";
+  str_buf << "\"num_iterations\": [\"num_iteration\", \"n_iter\", \"num_tree\", \"num_trees\", \"num_round\", \"num_rounds\", \"nrounds\", \"num_boost_round\", \"n_estimators\", \"max_iter\"], ";
+  str_buf << "\"learning_rate\": [\"shrinkage_rate\", \"eta\"], ";
+  str_buf << "\"num_leaves\": [\"num_leaf\", \"max_leaves\", \"max_leaf\", \"max_leaf_nodes\"], ";
+  str_buf << "\"tree_learner\": [\"tree\", \"tree_type\", \"tree_learner_type\"], ";
+  str_buf << "\"num_threads\": [\"num_thread\", \"nthread\", \"nthreads\", \"n_jobs\"], ";
+  str_buf << "\"device_type\": [\"device\"], ";
+  str_buf << "\"seed\": [\"random_seed\", \"random_state\"], ";
+  str_buf << "\"deterministic\": [], ";
+  str_buf << "\"force_col_wise\": [], ";
+  str_buf << "\"force_row_wise\": [], ";
+  str_buf << "\"histogram_pool_size\": [\"hist_pool_size\"], ";
+  str_buf << "\"max_depth\": [], ";
+  str_buf << "\"min_data_in_leaf\": [\"min_data_per_leaf\", \"min_data\", \"min_child_samples\", \"min_samples_leaf\"], ";
+  str_buf << "\"min_sum_hessian_in_leaf\": [\"min_sum_hessian_per_leaf\", \"min_sum_hessian\", \"min_hessian\", \"min_child_weight\"], ";
+  str_buf << "\"bagging_fraction\": [\"sub_row\", \"subsample\", \"bagging\"], ";
+  str_buf << "\"pos_bagging_fraction\": [\"pos_sub_row\", \"pos_subsample\", \"pos_bagging\"], ";
+  str_buf << "\"neg_bagging_fraction\": [\"neg_sub_row\", \"neg_subsample\", \"neg_bagging\"], ";
+  str_buf << "\"bagging_freq\": [\"subsample_freq\"], ";
+  str_buf << "\"bagging_seed\": [\"bagging_fraction_seed\"], ";
+  str_buf << "\"feature_fraction\": [\"sub_feature\", \"colsample_bytree\"], ";
+  str_buf << "\"feature_fraction_bynode\": [\"sub_feature_bynode\", \"colsample_bynode\"], ";
+  str_buf << "\"feature_fraction_seed\": [], ";
+  str_buf << "\"extra_trees\": [\"extra_tree\"], ";
+  str_buf << "\"extra_seed\": [], ";
+  str_buf << "\"early_stopping_round\": [\"early_stopping_rounds\", \"early_stopping\", \"n_iter_no_change\"], ";
+  str_buf << "\"first_metric_only\": [], ";
+  str_buf << "\"max_delta_step\": [\"max_tree_output\", \"max_leaf_output\"], ";
+  str_buf << "\"lambda_l1\": [\"reg_alpha\", \"l1_regularization\"], ";
+  str_buf << "\"lambda_l2\": [\"reg_lambda\", \"lambda\", \"l2_regularization\"], ";
+  str_buf << "\"linear_lambda\": [], ";
+  str_buf << "\"min_gain_to_split\": [\"min_split_gain\"], ";
+  str_buf << "\"drop_rate\": [\"rate_drop\"], ";
+  str_buf << "\"max_drop\": [], ";
+  str_buf << "\"skip_drop\": [], ";
+  str_buf << "\"xgboost_dart_mode\": [], ";
+  str_buf << "\"uniform_drop\": [], ";
+  str_buf << "\"drop_seed\": [], ";
+  str_buf << "\"top_rate\": [], ";
+  str_buf << "\"other_rate\": [], ";
+  str_buf << "\"min_data_per_group\": [], ";
+  str_buf << "\"max_cat_threshold\": [], ";
+  str_buf << "\"cat_l2\": [], ";
+  str_buf << "\"cat_smooth\": [], ";
+  str_buf << "\"max_cat_to_onehot\": [], ";
+  str_buf << "\"top_k\": [\"topk\"], ";
+  str_buf << "\"monotone_constraints\": [\"mc\", \"monotone_constraint\", \"monotonic_cst\"], ";
+  str_buf << "\"monotone_constraints_method\": [\"monotone_constraining_method\", \"mc_method\"], ";
+  str_buf << "\"monotone_penalty\": [\"monotone_splits_penalty\", \"ms_penalty\", \"mc_penalty\"], ";
+  str_buf << "\"feature_contri\": [\"feature_contrib\", \"fc\", \"fp\", \"feature_penalty\"], ";
+  str_buf << "\"forcedsplits_filename\": [\"fs\", \"forced_splits_filename\", \"forced_splits_file\", \"forced_splits\"], ";
+  str_buf << "\"refit_decay_rate\": [], ";
+  str_buf << "\"cegb_tradeoff\": [], ";
+  str_buf << "\"cegb_penalty_split\": [], ";
+  str_buf << "\"cegb_penalty_feature_lazy\": [], ";
+  str_buf << "\"cegb_penalty_feature_coupled\": [], ";
+  str_buf << "\"path_smooth\": [], ";
+  str_buf << "\"interaction_constraints\": [], ";
+  str_buf << "\"tree_interaction_constraints\": [], ";
+  str_buf << "\"verbosity\": [\"verbose\"], ";
+  str_buf << "\"input_model\": [\"model_input\", \"model_in\"], ";
+  str_buf << "\"output_model\": [\"model_output\", \"model_out\"], ";
+  str_buf << "\"saved_feature_importance_type\": [], ";
+  str_buf << "\"snapshot_freq\": [\"save_period\"], ";
+  str_buf << "\"linear_tree\": [\"linear_trees\"], ";
+  str_buf << "\"max_bin\": [\"max_bins\"], ";
+  str_buf << "\"max_bin_by_feature\": [], ";
+  str_buf << "\"min_data_in_bin\": [], ";
+  str_buf << "\"bin_construct_sample_cnt\": [\"subsample_for_bin\"], ";
+  str_buf << "\"data_random_seed\": [\"data_seed\"], ";
+  str_buf << "\"is_enable_sparse\": [\"is_sparse\", \"enable_sparse\", \"sparse\"], ";
+  str_buf << "\"enable_bundle\": [\"is_enable_bundle\", \"bundle\"], ";
+  str_buf << "\"use_missing\": [], ";
+  str_buf << "\"zero_as_missing\": [], ";
+  str_buf << "\"feature_pre_filter\": [], ";
+  str_buf << "\"pre_partition\": [\"is_pre_partition\"], ";
+  str_buf << "\"two_round\": [\"two_round_loading\", \"use_two_round_loading\"], ";
+  str_buf << "\"header\": [\"has_header\"], ";
+  str_buf << "\"label_column\": [\"label\"], ";
+  str_buf << "\"weight_column\": [\"weight\"], ";
+  str_buf << "\"group_column\": [\"group\", \"group_id\", \"query_column\", \"query\", \"query_id\"], ";
+  str_buf << "\"ignore_column\": [\"ignore_feature\", \"blacklist\"], ";
+  str_buf << "\"categorical_feature\": [\"cat_feature\", \"categorical_column\", \"cat_column\", \"categorical_features\"], ";
+  str_buf << "\"forcedbins_filename\": [], ";
+  str_buf << "\"save_binary\": [\"is_save_binary\", \"is_save_binary_file\"], ";
+  str_buf << "\"precise_float_parser\": [], ";
+  str_buf << "\"parser_config_file\": [], ";
+  str_buf << "\"start_iteration_predict\": [], ";
+  str_buf << "\"num_iteration_predict\": [], ";
+  str_buf << "\"predict_raw_score\": [\"is_predict_raw_score\", \"predict_rawscore\", \"raw_score\"], ";
+  str_buf << "\"predict_leaf_index\": [\"is_predict_leaf_index\", \"leaf_index\"], ";
+  str_buf << "\"predict_contrib\": [\"is_predict_contrib\", \"contrib\"], ";
+  str_buf << "\"predict_disable_shape_check\": [], ";
+  str_buf << "\"pred_early_stop\": [], ";
+  str_buf << "\"pred_early_stop_freq\": [], ";
+  str_buf << "\"pred_early_stop_margin\": [], ";
+  str_buf << "\"output_result\": [\"predict_result\", \"prediction_result\", \"predict_name\", \"prediction_name\", \"pred_name\", \"name_pred\"], ";
+  str_buf << "\"convert_model_language\": [], ";
+  str_buf << "\"convert_model\": [\"convert_model_file\"], ";
+  str_buf << "\"objective_seed\": [], ";
+  str_buf << "\"num_class\": [\"num_classes\"], ";
+  str_buf << "\"is_unbalance\": [\"unbalance\", \"unbalanced_sets\"], ";
+  str_buf << "\"scale_pos_weight\": [], ";
+  str_buf << "\"sigmoid\": [], ";
+  str_buf << "\"boost_from_average\": [], ";
+  str_buf << "\"reg_sqrt\": [], ";
+  str_buf << "\"alpha\": [], ";
+  str_buf << "\"fair_c\": [], ";
+  str_buf << "\"poisson_max_delta_step\": [], ";
+  str_buf << "\"tweedie_variance_power\": [], ";
+  str_buf << "\"lambdarank_truncation_level\": [], ";
+  str_buf << "\"lambdarank_norm\": [], ";
+  str_buf << "\"label_gain\": [], ";
+  str_buf << "\"metric\": [\"metrics\", \"metric_types\"], ";
+  str_buf << "\"metric_freq\": [\"output_freq\"], ";
+  str_buf << "\"is_provide_training_metric\": [\"training_metric\", \"is_training_metric\", \"train_metric\"], ";
+  str_buf << "\"eval_at\": [\"ndcg_eval_at\", \"ndcg_at\", \"map_eval_at\", \"map_at\"], ";
+  str_buf << "\"multi_error_top_k\": [], ";
+  str_buf << "\"auc_mu_weights\": [], ";
+  str_buf << "\"num_machines\": [\"num_machine\"], ";
+  str_buf << "\"local_listen_port\": [\"local_port\", \"port\"], ";
+  str_buf << "\"time_out\": [], ";
+  str_buf << "\"machine_list_filename\": [\"machine_list_file\", \"machine_list\", \"mlist\"], ";
+  str_buf << "\"machines\": [\"workers\", \"nodes\"], ";
+  str_buf << "\"gpu_platform_id\": [], ";
+  str_buf << "\"gpu_device_id\": [], ";
+  str_buf << "\"gpu_use_dp\": [], ";
+  str_buf << "\"num_gpu\": []";
+  str_buf << "}";
+  return str_buf.str();
 }
 
 const std::unordered_map<std::string, std::string>& Config::ParameterTypes() {

src/treelearner/col_sampler.hpp

@@ -28,6 +28,10 @@ class ColSampler {
       std::unordered_set<int> constraint_set(constraint.begin(), constraint.end());
       interaction_constraints_.push_back(constraint_set);
     }
+    for (auto constraint : config->tree_interaction_constraints_vector) {
+      std::unordered_set<int> constraint_set(constraint.begin(), constraint.end());
+      tree_interaction_constraints_.push_back(constraint_set);
+    }
   }
 
   static int GetCnt(size_t total_cnt, double fraction) {
@@ -89,30 +93,67 @@ class ColSampler {
   }
 
   std::vector<int8_t> GetByNode(const Tree* tree, int leaf) {
+    std::unordered_set<int> tree_allowed_features;
+    if (!tree_interaction_constraints_.empty()) {
+      std::set<int> tree_features = tree->tree_features();
+      tree_allowed_features.insert(tree_features.begin(), tree_features.end());
+      for (auto constraint : tree_interaction_constraints_) {
+        int num_feat_found = 0;
+
+        if (tree_features.empty()) {
+          tree_allowed_features.insert(constraint.begin(), constraint.end());
+        }
+
+        for (int feat : tree_features) {
+          if (constraint.count(feat) == 0) { break; }
+          ++num_feat_found;
+          if (num_feat_found == static_cast<int>(tree_features.size())) {
+            tree_allowed_features.insert(constraint.begin(), constraint.end());
+            break;
+          }
+        }
+      }
+    }
+
     // get interaction constraints for current branch
-    std::unordered_set<int> allowed_features;
+    std::unordered_set<int> branch_allowed_features;
     if (!interaction_constraints_.empty()) {
       std::vector<int> branch_features = tree->branch_features(leaf);
-      allowed_features.insert(branch_features.begin(), branch_features.end());
       for (auto constraint : interaction_constraints_) {
         int num_feat_found = 0;
-        if (branch_features.size() == 0) {
-          allowed_features.insert(constraint.begin(), constraint.end());
+        if (branch_features.empty()) {
+          branch_allowed_features.insert(constraint.begin(), constraint.end());
         }
         for (int feat : branch_features) {
           if (constraint.count(feat) == 0) { break; }
           ++num_feat_found;
           if (num_feat_found == static_cast<int>(branch_features.size())) {
-            allowed_features.insert(constraint.begin(), constraint.end());
+            branch_allowed_features.insert(constraint.begin(), constraint.end());
             break;
           }
         }
       }
     }
 
+    // intersect allowed features for branch and tree
+    std::unordered_set<int> allowed_features;
+
+    if(tree_interaction_constraints_.empty() && !interaction_constraints_.empty()) {
+      allowed_features.insert(branch_allowed_features.begin(), branch_allowed_features.end());
+    } else if(!tree_interaction_constraints_.empty() && interaction_constraints_.empty()){
+      allowed_features.insert(tree_allowed_features.begin(), tree_allowed_features.end());
+    } else {
+      for (int element : tree_allowed_features) {
+        if (branch_allowed_features.count(element) > 0) {
+          allowed_features.insert(element);
+        }
+      }
+    }
+
+
     std::vector<int8_t> ret(train_data_->num_features(), 0);
     if (fraction_bynode_ >= 1.0f) {
-      if (interaction_constraints_.empty()) {
+      if (interaction_constraints_.empty() && tree_interaction_constraints_.empty()) {
         return std::vector<int8_t>(train_data_->num_features(), 1);
       } else {
         for (int feat : allowed_features) {
@@ -128,7 +169,7 @@ class ColSampler {
       auto used_feature_cnt = GetCnt(used_feature_indices_.size(), fraction_bynode_);
       std::vector<int>* allowed_used_feature_indices;
       std::vector<int> filtered_feature_indices;
-      if (interaction_constraints_.empty()) {
+      if (interaction_constraints_.empty() && tree_interaction_constraints_.empty()) {
         allowed_used_feature_indices = &used_feature_indices_;
       } else {
         for (int feat_ind : used_feature_indices_) {
@@ -154,7 +195,7 @@ class ColSampler {
           GetCnt(valid_feature_indices_.size(), fraction_bynode_);
       std::vector<int>* allowed_valid_feature_indices;
       std::vector<int> filtered_feature_indices;
-      if (interaction_constraints_.empty()) {
+      if (interaction_constraints_.empty() && tree_interaction_constraints_.empty()) {
         allowed_valid_feature_indices = &valid_feature_indices_;
       } else {
         for (int feat : valid_feature_indices_) {
@@ -199,6 +240,8 @@ class ColSampler {
   std::vector<int> valid_feature_indices_;
   /*! \brief interaction constraints index in original (raw data) features */
   std::vector<std::unordered_set<int>> interaction_constraints_;
+  /*! \brief tree nteraction constraints index in original (raw data) features */
+  std::vector<std::unordered_set<int>> tree_interaction_constraints_;
 };
 
 }  // namespace LightGBM