Explainable boosting parameters

docs/Parameters.rst

@@ -618,6 +618,32 @@ Learning Control Parameters
 
    -  any two features can only appear in the same branch only if there exists a constraint containing both features
 
+-  ``tree_interaction_constraints`` :raw-html:`<a id="tree_interaction_constraints" title="Permalink to this parameter" href="#tree_interaction_constraints">&#x1F517;&#xFE0E;</a>`, default = ``""``, type = string
+
+   -  controls which features can appear in the same tree
+
+   -  by default interaction constraints are disabled, to enable them you can specify
+
+      -  for CLI, lists separated by commas, e.g. ``[0,1,2],[2,3]``
+
+      -  for Python-package, list of lists, e.g. ``[[0, 1, 2], [2, 3]]``
+
+      -  for R-package, list of character or numeric vectors, e.g. ``list(c("var1", "var2", "var3"), c("var3", "var4"))`` or ``list(c(1L, 2L, 3L), c(3L, 4L))``. Numeric vectors should use 1-based indexing, where ``1L`` is the first feature, ``2L`` is the second feature, etc
+
+   -  any two features can only appear in the same tree only if there exists a constraint containing both features
+
+-  ``max_tree_interactions`` :raw-html:`<a id="max_tree_interactions" title="Permalink to this parameter" href="#max_tree_interactions">&#x1F517;&#xFE0E;</a>`, default = ``0``, type = int, constraints: ``max_tree_interactions >=  0.0``
+
+   -  controls how many features can appear in the same tree
+
+   -  by default (max_tree_interactions = 0) interaction constraints are disabled
+
+-  ``max_interactions`` :raw-html:`<a id="max_interactions" title="Permalink to this parameter" href="#max_interactions">&#x1F517;&#xFE0E;</a>`, default = ``0``, type = int, constraints: ``max_interactions >=  0.0``
+
+   -  controls how many features interactions can be added to the final model
+
+   -  by default no limit is imposed on the interaction with max_interactions = 0
+
 -  ``verbosity`` :raw-html:`<a id="verbosity" title="Permalink to this parameter" href="#verbosity">&#x1F517;&#xFE0E;</a>`, default = ``1``, type = int, aliases: ``verbose``
 
    -  controls the level of LightGBM's verbosity

include/LightGBM/config.h

@@ -571,6 +571,24 @@ struct Config {
   // desc = any two features can only appear in the same branch only if there exists a constraint containing both features
   std::string interaction_constraints = "";
 
+  // desc = controls which features can appear in the same tree
+  // desc = by default interaction constraints are disabled, to enable them you can specify
+  // descl2 = for CLI, lists separated by commas, e.g. ``[0,1,2],[2,3]``
+  // descl2 = for Python-package, list of lists, e.g. ``[[0, 1, 2], [2, 3]]``
+  // descl2 = for R-package, list of character or numeric vectors, e.g. ``list(c("var1", "var2", "var3"), c("var3", "var4"))`` or ``list(c(1L, 2L, 3L), c(3L, 4L))``. Numeric vectors should use 1-based indexing, where ``1L`` is the first feature, ``2L`` is the second feature, etc
+  // desc = any two features can only appear in the same tree only if there exists a constraint containing both features
+  std::string tree_interaction_constraints = "";
+
+  // check = >= 0.0
+  // desc = controls how many features can appear in the same tree
+  // desc = by default (max_tree_interactions = 0) interaction constraints are disabled
+  int max_tree_interactions = 0;
+
+  // check = >= 0.0
+  // desc = controls how many features interactions can be added to the final model
+  // desc = by default no limit is imposed on the interaction with max_interactions = 0
+  int max_interactions = 0;
+
   // alias = verbose
   // desc = controls the level of LightGBM's verbosity
   // desc = ``< 0``: Fatal, ``= 0``: Error (Warning), ``= 1``: Info, ``> 1``: Debug
@@ -1126,6 +1144,7 @@ struct Config {
   static const std::unordered_set<std::string>& parameter_set();
   std::vector<std::vector<double>> auc_mu_weights_matrix;
   std::vector<std::vector<int>> interaction_constraints_vector;
+  std::vector<std::vector<int>> tree_interaction_constraints_vector;
   static const std::unordered_map<std::string, std::string>& ParameterTypes();
   static const std::string DumpAliases();
 
@@ -1135,6 +1154,7 @@ struct Config {
   std::string SaveMembersToString() const;
   void GetAucMuWeights();
   void GetInteractionConstraints();
+  void GetTreeInteractionConstraints();
 };
 
 inline bool Config::GetString(

include/LightGBM/tree.h

@@ -13,6 +13,7 @@
 #include <memory>
 #include <unordered_map>
 #include <vector>
+#include <set>
 
 namespace LightGBM {
 
@@ -158,6 +159,11 @@ class Tree {
   /*! \brief Get features on leaf's branch*/
   inline std::vector<int> branch_features(int leaf) const { return branch_features_[leaf]; }
 
+  /*! \brief Get unique features used by the current tree*/
+  std::set<int> tree_features() const {
+     return tree_features_;
+  }
+
   inline double split_gain(int split_idx) const { return split_gain_[split_idx]; }
 
   inline double internal_value(int node_idx) const {
@@ -319,6 +325,8 @@ class Tree {
 
   inline bool is_linear() const { return is_linear_; }
 
+  inline bool is_tracking_branch_features() const { return track_branch_features_; }
+
   #ifdef USE_CUDA
   inline bool is_cuda_tree() const { return is_cuda_tree_; }
   #endif  // USE_CUDA
@@ -520,6 +528,10 @@ class Tree {
   bool track_branch_features_;
   /*! \brief Features on leaf's branch, original index */
   std::vector<std::vector<int>> branch_features_;
+
+  /*! \brief Features used by the tree, original index */
+  std::set<int> tree_features_;
+
   double shrinkage_;
   int max_depth_;
   /*! \brief Tree has linear model at each leaf */
@@ -579,6 +591,7 @@ inline void Tree::Split(int leaf, int feature, int real_feature,
     branch_features_[num_leaves_] = branch_features_[leaf];
     branch_features_[num_leaves_].push_back(split_feature_[new_node_idx]);
     branch_features_[leaf].push_back(split_feature_[new_node_idx]);
+    tree_features_.insert(split_feature_[new_node_idx]);
   }
 }
 

python-package/lightgbm/engine.py

@@ -275,6 +275,7 @@ def train(
     booster.best_iteration = 0
 
     # start training
+    interactions_used = set()
     for i in range(init_iteration, init_iteration + num_boost_round):
         for cb in callbacks_before_iter:
             cb(callback.CallbackEnv(model=booster,
@@ -287,6 +288,15 @@ def train(
         booster.update(fobj=fobj)
 
         evaluation_result_list: List[_LGBM_BoosterEvalMethodResultType] = []
+        if params.get("max_interactions", 0) > 0:
+            interaction_used = booster.dump_model(num_iteration=1, start_iteration=i)["tree_info"][0]["tree_features"]
+            interaction_used.sort()
+            interactions_used.add(tuple(interaction_used))
+
+            if len(interactions_used) == params["max_interactions"]:
+                params["tree_interaction_constraints"] = [list(feats) for feats in interactions_used]
+                params["max_interactions"] = 0
+                booster.reset_parameter(params)
         # check evaluation result.
         if valid_sets is not None:
             if is_valid_contain_train:

src/boosting/gbdt.cpp

@@ -244,6 +244,20 @@ void GBDT::Train(int snapshot_freq, const std::string& model_output_path) {
       std::string snapshot_out = model_output_path + ".snapshot_iter_" + std::to_string(iter + 1);
       SaveModelToFile(0, -1, config_->saved_feature_importance_type, snapshot_out.c_str());
     }
+
+    if (config_->max_interactions != 0) {
+      interactions_used.insert(models_[models_.size() - 1]->tree_features());
+    }
+
+    if (config_->max_interactions != 0 && static_cast<int>(interactions_used.size()) >= config_->max_interactions) {
+      auto new_config = std::unique_ptr<Config>(new Config(*config_));
+      new_config->tree_interaction_constraints_vector.clear();
+      for (auto &inter_set : interactions_used) {
+        new_config->tree_interaction_constraints_vector.emplace_back(inter_set.begin(), inter_set.end());
+      }
+      new_config->max_interactions = 0;
+      ResetConfig(new_config.release());
+    }
   }
 }
 

src/boosting/gbdt.h

@@ -23,6 +23,7 @@
 #include <unordered_map>
 #include <utility>
 #include <vector>
+#include <set>
 
 #include "cuda/cuda_score_updater.hpp"
 #include "score_updater.hpp"
@@ -542,6 +543,8 @@ class GBDT : public GBDTBase {
   std::vector<std::vector<std::string>> best_msg_;
   /*! \brief Trained models(trees) */
   std::vector<std::unique_ptr<Tree>> models_;
+  /*! \brief Set of set of features used in all the models */
+  std::set<std::set<int>> interactions_used;
   /*! \brief Max feature index of training data*/
   int max_feature_idx_;
   /*! \brief Parser config file content */

src/io/config.cpp

@@ -232,13 +232,21 @@ void Config::GetAucMuWeights() {
 }
 
 void Config::GetInteractionConstraints() {
-  if (interaction_constraints == "") {
+  if (interaction_constraints.empty()) {
     interaction_constraints_vector = std::vector<std::vector<int>>();
   } else {
     interaction_constraints_vector = Common::StringToArrayofArrays<int>(interaction_constraints, '[', ']', ',');
   }
 }
 
+void Config::GetTreeInteractionConstraints() {
+  if (tree_interaction_constraints.empty()) {
+    tree_interaction_constraints_vector = std::vector<std::vector<int>>();
+  } else {
+    tree_interaction_constraints_vector = Common::StringToArrayofArrays<int>(tree_interaction_constraints, '[', ']', ',');
+  }
+}
+
 void Config::Set(const std::unordered_map<std::string, std::string>& params) {
   // generate seeds by seed.
   if (GetInt(params, "seed", &seed)) {
@@ -269,6 +277,8 @@ void Config::Set(const std::unordered_map<std::string, std::string>& params) {
 
   GetInteractionConstraints();
 
+  GetTreeInteractionConstraints();
+
   // sort eval_at
   std::sort(eval_at.begin(), eval_at.end());
 

src/io/config_auto.cpp

@@ -246,6 +246,9 @@ const std::unordered_set<std::string>& Config::parameter_set() {
   "cegb_penalty_feature_coupled",
   "path_smooth",
   "interaction_constraints",
+  "tree_interaction_constraints",
+  "max_tree_interactions",
+  "max_interactions",
   "verbosity",
   "input_model",
   "output_model",
@@ -488,6 +491,14 @@ void Config::GetMembersFromString(const std::unordered_map<std::string, std::str
 
   GetString(params, "interaction_constraints", &interaction_constraints);
 
+  GetString(params, "tree_interaction_constraints", &tree_interaction_constraints);
+
+  GetInt(params, "max_tree_interactions", &max_tree_interactions);
+  CHECK_GE(max_tree_interactions,  0.0);
+
+  GetInt(params, "max_interactions", &max_interactions);
+  CHECK_GE(max_interactions,  0.0);
+
   GetInt(params, "verbosity", &verbosity);
 
   GetString(params, "input_model", &input_model);
@@ -722,6 +733,9 @@ std::string Config::SaveMembersToString() const {
   str_buf << "[cegb_penalty_feature_coupled: " << Common::Join(cegb_penalty_feature_coupled, ",") << "]\n";
   str_buf << "[path_smooth: " << path_smooth << "]\n";
   str_buf << "[interaction_constraints: " << interaction_constraints << "]\n";
+  str_buf << "[tree_interaction_constraints: " << tree_interaction_constraints << "]\n";
+  str_buf << "[max_tree_interactions: " << max_tree_interactions << "]\n";
+  str_buf << "[max_interactions: " << max_interactions << "]\n";
   str_buf << "[verbosity: " << verbosity << "]\n";
   str_buf << "[saved_feature_importance_type: " << saved_feature_importance_type << "]\n";
   str_buf << "[use_quantized_grad: " << use_quantized_grad << "]\n";
@@ -846,6 +860,9 @@ const std::unordered_map<std::string, std::vector<std::string>>& Config::paramet
     {"cegb_penalty_feature_coupled", {}},
     {"path_smooth", {}},
     {"interaction_constraints", {}},
+    {"tree_interaction_constraints", {}},
+    {"max_tree_interactions", {}},
+    {"max_interactions", {}},
     {"verbosity", {"verbose"}},
     {"input_model", {"model_input", "model_in"}},
     {"output_model", {"model_output", "model_out"}},
@@ -989,6 +1006,9 @@ const std::unordered_map<std::string, std::string>& Config::ParameterTypes() {
     {"cegb_penalty_feature_coupled", "vector<double>"},
     {"path_smooth", "double"},
     {"interaction_constraints", "vector<vector<int>>"},
+    {"tree_interaction_constraints", "string"},
+    {"max_tree_interactions", "int"},
+    {"max_interactions", "int"},
     {"verbosity", "int"},
     {"input_model", "string"},
     {"output_model", "string"},

src/io/tree.cpp

@@ -415,6 +415,18 @@ std::string Tree::ToJSON() const {
   str_buf << "\"num_leaves\":" << num_leaves_ << "," << '\n';
   str_buf << "\"num_cat\":" << num_cat_ << "," << '\n';
   str_buf << "\"shrinkage\":" << shrinkage_ << "," << '\n';
+
+  auto feats_used = tree_features();
+  size_t i = 0;
+  str_buf << "\"tree_features\":[";
+  for (int feat : feats_used) {
+    str_buf << feat;
+    if (i != feats_used.size() - 1) {
+      str_buf << ",";
+    }
+    ++i;
+  }
+  str_buf <<    "]," << '\n';
   if (num_leaves_ == 1) {
     if (is_linear_) {
       str_buf << "\"tree_structure\":{" << "\"leaf_value\":" << leaf_value_[0] << ", " << "\n";