Introduction
This short tutorial shows the current cross-validation API for greedy concentration-index trees and concentration-index forests. The core workflow is the same for both model classes:
- Build a small control grid.
- Run cross-validation with
tune_ci_tree()ortune_ci_forest(). - Select settings with
ci_select_best(). - Collect metrics with
ci_collect_metrics(). - Build a report-ready table with
ci_fit_summary_table().
The examples use the simulated Kenya child-survival data included
with ineqTrees.
library(ineqTrees)
library(data.table)
#>
#> Attaching package: 'data.table'
#> The following object is masked from 'package:base':
#>
#> %notin%
library(knitr)
data(kenya, package = "ineqTrees")
setDT(kenya)
tuning_vars <- c(
"wealth",
"deadu5_num",
"rural",
"ed",
"reg",
"unskilled",
"sample_weight"
)
tuning_data <- kenya[
complete.cases(kenya[, ..tuning_vars]),
..tuning_vars
]
set.seed(20260521)
tuning_data <- tuning_data[
sample.int(.N, min(700L, .N))
]
ci_formula <- cbind(wealth, deadu5_num) ~ rural + ed + reg + unskilled
criterion_types <- c("CI", "CIg")
tuning_metrics <- c(
"validation_gain",
"relative_validation_gain",
"brier"
)validation_gain is the reduction in validation-fold
concentration-index impurity after observations are assigned to terminal
nodes. The relative metric scales that gain by the absolute validation
root impurity:
relative_validation_gain = validation_gain / abs(root_impurity)The raw gain is useful for selecting within a criterion. The relative gain and the percentage gain columns are useful for interpreting how much validation root impurity the model recovers.
Cross-Validate Trees
Start with a grid of greedy tree controls. These are split-search controls, not conditional-inference test controls.
tree_grid <- ci_tree_control_grid(
minsplit = c(120L, 180L),
minbucket = 60L,
minprob = 0.05,
maxdepth = 2:3
)
tree_grid
#> minsplit minbucket minprob maxdepth min_gain min_relative_gain
#> <int> <int> <num> <int> <num> <num>
#> 1: 120 60 0.05 2 0 0
#> 2: 180 60 0.05 2 0 0
#> 3: 120 60 0.05 3 0 0
#> 4: 180 60 0.05 3 0 0Run cross-validation with tune_ci_tree(). The first
metric is the selection metric used for
tree_tuning$best_fit when refit = TRUE.
tree_tuning <- tune_ci_tree(
formula = ci_formula,
data = tuning_data,
rank_name = "wealth",
outcome_name = "deadu5_num",
weights = tuning_data$sample_weight,
type = criterion_types,
control_grid = tree_grid,
v = 3L,
strata = "deadu5_num",
seed = 20260521,
metrics = tuning_metrics,
control = control_ci_tune(save_pred = TRUE),
refit = TRUE
)Use ci_collect_metrics(format = "wide") when you want
one row per setting with metric columns.
tree_metrics_wide <- ci_collect_metrics(
tree_tuning,
metric = tuning_metrics,
format = "wide"
)
knitr::kable(
tree_metrics_wide[
order(-mean_validation_gain),
.(
type,
minsplit,
minbucket,
maxdepth,
mean_validation_gain,
std_err_validation_gain,
mean_validation_relative_gain,
mean_brier
)
],
digits = 4,
caption = "Cross-validated tree metrics"
)| type | minsplit | minbucket | maxdepth | mean_validation_gain | std_err_validation_gain | mean_validation_relative_gain | mean_brier |
|---|---|---|---|---|---|---|---|
| CIg | 180 | 60 | 2 | 0.0002 | 0.0019 | 0.0543 | 0.0720 |
| CIg | 180 | 60 | 3 | 0.0001 | 0.0015 | 0.0229 | 0.0745 |
| CIg | 120 | 60 | 2 | 0.0000 | 0.0022 | 0.0477 | 0.0720 |
| CIg | 120 | 60 | 3 | -0.0003 | 0.0018 | 0.0113 | 0.0744 |
| CI | 120 | 60 | 2 | -0.0359 | 0.0711 | -0.1205 | 0.0780 |
| CI | 180 | 60 | 2 | -0.0359 | 0.0711 | -0.1205 | 0.0780 |
| CI | 120 | 60 | 3 | -0.0517 | 0.0861 | -0.1565 | 0.0787 |
| CI | 180 | 60 | 3 | -0.0566 | 0.0812 | -0.1761 | 0.0780 |
Use ci_select_best() to keep the best setting within
each criterion type.
tree_best <- ci_select_best(
tree_tuning,
metric = "validation_gain"
)
knitr::kable(
tree_best[
,
.(type, minsplit, minbucket, minprob, maxdepth)
],
digits = 4,
caption = "Selected tree controls"
)| type | minsplit | minbucket | minprob | maxdepth |
|---|---|---|---|---|
| CI | 120 | 60 | 0.05 | 2 |
| CIg | 180 | 60 | 0.05 | 2 |
For reports, ci_fit_summary_table() combines selected
settings, validation metrics, training diagnostics, and root-objective
summaries.
tree_fit_summary <- ci_fit_summary_table(
tree_tuning,
selected = tree_best,
metrics = c(
"train_gain",
"validation_gain",
"train_relative_gain",
"relative_validation_gain"
),
include_percent = TRUE
)
knitr::kable(
tree_fit_summary[
,
.(
type,
mean_root_objective,
mean_train_gain,
mean_percent_train_gain,
mean_validation_gain,
mean_percent_validation_gain,
std_err_validation_gain
)
],
digits = 4,
caption = "Selected tree fit summary"
)| type | mean_root_objective | mean_train_gain | mean_percent_train_gain | mean_validation_gain | mean_percent_validation_gain | std_err_validation_gain |
|---|---|---|---|---|---|---|
| CI | 0.2824 | 0.1709 | 60.5244 | -0.0359 | -12.7167 | 0.0711 |
| CIg | 0.0219 | 0.0099 | 45.3569 | 0.0002 | 1.0728 | 0.0019 |
Saved validation predictions are available when
save_pred = TRUE.
head(
ci_collect_predictions(tree_tuning)[
,
.(grid_id, fold_id, type, row_id, outcome, .pred)
]
)
#> grid_id fold_id type row_id outcome .pred
#> <int> <int> <char> <int> <num> <num>
#> 1: 1 1 CI 2 0 0.01084559
#> 2: 1 1 CI 4 0 0.01084559
#> 3: 1 1 CI 13 0 0.26721087
#> 4: 1 1 CI 15 0 0.01084559
#> 5: 1 1 CI 19 0 0.26721087
#> 6: 1 1 CI 23 0 0.26721087The selected refit is stored on the tuning object.
ci_tree_terminal_summary(tree_tuning$best_fit)[
,
.(node, n, depth, ci, outcome_percent, rule)
]
#> node n depth ci outcome_percent
#> <int> <int> <int> <num> <num>
#> 1: 3 280 2 0.03773129 1.333262
#> 2: 4 312 2 0.18348088 7.736424
#> 3: 5 108 1 0.34569839 23.847287
#> rule
#> <char>
#> 1: reg in {Mombasa, Kilifi, Tana River, Lamu, Taita Taveta, Garissa, Wajir, Mandera, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Kitui, Machakos, Makueni, Nyeri, Kirinyaga, Murang'a, Kiambu, Turkana, West Pokot, Uasin Gishu, Elgeyo-Marakwet, Baringo, Laikipia, Nakuru, Narok, Kajiado, Kericho, Bomet, Kakamega, Vihiga, Bungoma, Kisumu, Migori, Kisii, Nyamira, Nairobi} & reg in {Tana River, Lamu, Taita Taveta, Garissa, Wajir, Mandera, Isiolo, Meru, Embu, Nyeri, Kirinyaga, Murang'a, Turkana, West Pokot, Elgeyo-Marakwet, Baringo, Laikipia, Narok, Bomet, Vihiga, Kisumu, Kisii, Nyamira, Nairobi}
#> 2: reg in {Mombasa, Kilifi, Tana River, Lamu, Taita Taveta, Garissa, Wajir, Mandera, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Kitui, Machakos, Makueni, Nyeri, Kirinyaga, Murang'a, Kiambu, Turkana, West Pokot, Uasin Gishu, Elgeyo-Marakwet, Baringo, Laikipia, Nakuru, Narok, Kajiado, Kericho, Bomet, Kakamega, Vihiga, Bungoma, Kisumu, Migori, Kisii, Nyamira, Nairobi} & reg in {Mombasa, Kilifi, Marsabit, Tharaka-Nithi, Kitui, Machakos, Makueni, Kiambu, Uasin Gishu, Nakuru, Kajiado, Kericho, Kakamega, Bungoma, Migori}
#> 3: reg in {Kwale, Nyandarua, Samburu, Trans Nzoia, Nandi, Busia, Siaya, Homa Bay}Cross-Validate Forests
Forest tuning uses the same metric and collector API. The forest grid
also includes ntree and mtry.
forest_grid <- ci_tree_control_grid(
minsplit = 120L,
minbucket = 60L,
minprob = 0.05,
maxdepth = 2L,
mtry = c(1L, 2L),
ntree = c(6L, 10L)
)
forest_grid
#> minsplit minbucket minprob maxdepth min_gain min_relative_gain mtry ntree
#> <int> <int> <num> <int> <num> <num> <int> <int>
#> 1: 120 60 0.05 2 0 0 1 6
#> 2: 120 60 0.05 2 0 0 2 6
#> 3: 120 60 0.05 2 0 0 1 10
#> 4: 120 60 0.05 2 0 0 2 10tune_ci_forest() scores concentration-index validation
gain from the forest’s internal trees. Each member tree supplies a
validation-fold partition, the tree-level validation gains are averaged,
and prediction metrics such as Brier score are computed from the
averaged forest predictions. When refit = TRUE, the
selected forest is still accompanied by a surrogate tree for
interpretation.
forest_tuning <- tune_ci_forest(
formula = ci_formula,
data = tuning_data,
rank_name = "wealth",
outcome_name = "deadu5_num",
weights = tuning_data$sample_weight,
type = criterion_types,
control_grid = forest_grid,
v = 3L,
strata = "deadu5_num",
seed = 20260522,
metrics = tuning_metrics,
prediction_name = "forest_risk",
parallel_over = "none",
control = control_ci_tune(save_pred = TRUE),
refit = TRUE
)
forest_metrics_wide <- ci_collect_metrics(
forest_tuning,
metric = tuning_metrics,
format = "wide"
)
knitr::kable(
forest_metrics_wide[
order(-mean_validation_gain),
.(
type,
ntree,
mtry,
minbucket,
mean_validation_gain,
std_err_validation_gain,
mean_validation_relative_gain,
mean_brier
)
],
digits = 4,
caption = "Cross-validated forest metrics"
)| type | ntree | mtry | minbucket | mean_validation_gain | std_err_validation_gain | mean_validation_relative_gain | mean_brier |
|---|---|---|---|---|---|---|---|
| CI | 10 | 1 | 60 | 0.0087 | 0.0090 | 0.0286 | 0.0692 |
| CI | 6 | 1 | 60 | 0.0070 | 0.0161 | 0.0197 | 0.0700 |
| CI | 10 | 2 | 60 | 0.0017 | 0.0272 | -0.0189 | 0.0715 |
| CIg | 10 | 2 | 60 | -0.0001 | 0.0007 | -0.0162 | 0.0701 |
| CIg | 6 | 2 | 60 | -0.0001 | 0.0011 | -0.0240 | 0.0730 |
| CIg | 10 | 1 | 60 | -0.0005 | 0.0018 | -0.0504 | 0.0701 |
| CIg | 6 | 1 | 60 | -0.0010 | 0.0008 | -0.0443 | 0.0700 |
| CI | 6 | 2 | 60 | -0.0023 | 0.0112 | -0.0038 | 0.0731 |
forest_best <- ci_select_best(
forest_tuning,
metric = "validation_gain"
)
knitr::kable(
forest_best[
,
.(type, ntree, mtry, minsplit, minbucket, maxdepth)
],
digits = 4,
caption = "Selected forest controls"
)| type | ntree | mtry | minsplit | minbucket | maxdepth |
|---|---|---|---|---|---|
| CI | 10 | 1 | 120 | 60 | 2 |
| CIg | 10 | 2 | 120 | 60 | 2 |
forest_fit_summary <- ci_fit_summary_table(
forest_tuning,
selected = forest_best,
metrics = c(
"train_gain",
"validation_gain",
"train_relative_gain",
"relative_validation_gain"
),
include_percent = TRUE
)
knitr::kable(
forest_fit_summary[
,
.(
type,
mean_root_objective,
mean_train_gain,
mean_percent_train_gain,
mean_validation_gain,
mean_percent_validation_gain,
std_err_validation_gain
)
],
digits = 4,
caption = "Selected forest fit summary"
)| type | mean_root_objective | mean_train_gain | mean_percent_train_gain | mean_validation_gain | mean_percent_validation_gain | std_err_validation_gain |
|---|---|---|---|---|---|---|
| CI | 0.3009 | 0.0189 | 6.2691 | 0.0087 | 2.8924 | 9e-03 |
| CIg | 0.0222 | 0.0022 | 9.9192 | -0.0001 | -0.2852 | 7e-04 |
With refit = TRUE, the final fitted forest and its
surrogate tree are stored on the tuning object.
ci_forest_summary(forest_tuning$best_fit)
#> ntree mtry type n mean_outcome mean_prediction outcome_ci
#> <int> <int> <char> <int> <num> <num> <num>
#> 1: 10 1 CI 700 0.07412078 0.07102726 0.2915001
#> prediction_ci mean_terminal_nodes mean_max_depth
#> <num> <num> <num>
#> 1: 0.05217861 2.3 1.3
ci_tree_terminal_summary(forest_tuning$best_surrogate)[
,
.(node, n, depth, outcome_percent, rule)
]
#> node n depth outcome_percent
#> <int> <int> <int> <num>
#> 1: 3 147 2 4.218972
#> 2: 4 376 2 7.448354
#> 3: 6 54 2 8.287333
#> 4: 7 123 2 9.382548
#> rule
#> <char>
#> 1: ed in {a education} & reg in {Tana River, Mandera, Machakos, Nyeri, Kirinyaga, Murang'a, Turkana, Narok, Bomet, Kisumu, Kisii, Nyamira, Nairobi}
#> 2: ed in {a education} & reg in {Mombasa, Kwale, Kilifi, Lamu, Taita Taveta, Garissa, Wajir, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Kitui, Makueni, Nyandarua, Kiambu, West Pokot, Samburu, Trans Nzoia, Uasin Gishu, Elgeyo-Marakwet, Nandi, Baringo, Laikipia, Nakuru, Kajiado, Kericho, Kakamega, Vihiga, Bungoma, Busia, Siaya, Homa Bay, Migori}
#> 3: ed in {b no education} & reg in {Isiolo, Meru, Kirinyaga, Murang'a, Kiambu, Elgeyo-Marakwet, Nakuru, Narok, Vihiga, Kisumu, Nairobi}
#> 4: ed in {b no education} & reg in {Mombasa, Kwale, Kilifi, Tana River, Lamu, Taita Taveta, Garissa, Wajir, Mandera, Marsabit, Tharaka-Nithi, Kitui, Machakos, Makueni, Nyandarua, Turkana, West Pokot, Samburu, Trans Nzoia, Uasin Gishu, Nandi, Baringo, Laikipia, Kajiado, Kericho, Kakamega, Bungoma, Busia, Siaya, Homa Bay, Migori, Kisii, Nyamira}Minimal Template
For most analyses, the tuning pattern is:
grid <- ci_tree_control_grid(
minsplit = c(100L, 200L),
minbucket = c(50L, 100L),
maxdepth = 2:4
)
tuned <- tune_ci_tree(
formula = cbind(rank, outcome) ~ x1 + x2 + x3,
data = analysis_data,
rank_name = "rank",
outcome_name = "outcome",
weights = analysis_data$weights,
type = c("CI", "CIg"),
control_grid = grid,
v = 5L,
strata = "outcome",
metrics = c("validation_gain", "relative_validation_gain"),
control = control_ci_tune(save_pred = TRUE),
refit = TRUE
)
selected <- ci_select_best(tuned, metric = "validation_gain")
ci_fit_summary_table(
tuned,
selected = selected,
metrics = c(
"train_gain",
"validation_gain",
"train_relative_gain",
"relative_validation_gain"
)
)Use tune_ci_forest() with a grid that includes
ntree and mtry when you want the same
cross-validation workflow for forests.