Introduction
This short tutorial shows the current cross-validation API for greedy concentration-index trees and concentration-index forests. The core workflow is the same for both model classes:
- Build a small control grid.
- Run cross-validation with
tune_ci_tree()ortune_ci_forest(). - Select settings with
ci_select_best(). - Collect metrics with
ci_collect_metrics(). - Build a report-ready table with
ci_fit_summary_table().
The examples use the simulated Kenya child-survival data included
with ineqTrees.
library(ineqTrees)
library(data.table)
#>
#> Attaching package: 'data.table'
#> The following object is masked from 'package:base':
#>
#> %notin%
library(knitr)
data(kenya, package = "ineqTrees")
setDT(kenya)
tuning_vars <- c(
"wealth",
"deadu5_num",
"rural",
"ed",
"reg",
"unskilled",
"sample_weight"
)
tuning_data <- kenya[
complete.cases(kenya[, ..tuning_vars]),
..tuning_vars
]
set.seed(20260521)
tuning_data <- tuning_data[
sample.int(.N, min(700L, .N))
]
ci_formula <- cbind(wealth, deadu5_num) ~ rural + ed + reg + unskilled
criterion_types <- c("CI", "CIg")
tuning_metrics <- c(
"validation_gain",
"relative_validation_gain",
"brier"
)validation_gain is the reduction in validation-fold
concentration-index impurity after observations are assigned to terminal
nodes. The relative metric scales that gain by the absolute validation
root impurity:
relative_validation_gain = validation_gain / abs(root_impurity)The raw gain is useful for selecting within a criterion. The relative gain and the percentage gain columns are useful for interpreting how much validation root impurity the model recovers.
Cross-Validate Trees
Start with a grid of greedy tree controls. These are split-search controls, not conditional-inference test controls.
tree_grid <- ci_tree_control_grid(
minsplit = c(120L, 180L),
minbucket = 60L,
minprob = 0.05,
maxdepth = 2:3
)
tree_grid
#> minsplit minbucket minprob maxdepth min_gain min_relative_gain
#> <int> <int> <num> <int> <num> <num>
#> 1: 120 60 0.05 2 0 0
#> 2: 180 60 0.05 2 0 0
#> 3: 120 60 0.05 3 0 0
#> 4: 180 60 0.05 3 0 0Run cross-validation with tune_ci_tree(). The first
metric is the selection metric used for
tree_tuning$best_fit when refit = TRUE.
tree_tuning <- tune_ci_tree(
formula = ci_formula,
data = tuning_data,
rank_name = "wealth",
outcome_name = "deadu5_num",
weights = tuning_data$sample_weight,
type = criterion_types,
control_grid = tree_grid,
v = 3L,
strata = "deadu5_num",
seed = 20260521,
metrics = tuning_metrics,
control = control_ci_tune(save_pred = TRUE),
refit = TRUE
)Use ci_collect_metrics(format = "wide") when you want
one row per setting with metric columns.
tree_metrics_wide <- ci_collect_metrics(
tree_tuning,
metric = tuning_metrics,
format = "wide"
)
knitr::kable(
tree_metrics_wide[
order(-mean_validation_gain),
.(
type,
minsplit,
minbucket,
maxdepth,
mean_validation_gain,
std_err_validation_gain,
mean_validation_relative_gain,
mean_brier
)
],
digits = 4,
caption = "Cross-validated tree metrics"
)| type | minsplit | minbucket | maxdepth | mean_validation_gain | std_err_validation_gain | mean_validation_relative_gain | mean_brier |
|---|---|---|---|---|---|---|---|
| CIg | 120 | 60 | 2 | -0.0008 | 0.0041 | -0.0327 | 0.0723 |
| CIg | 180 | 60 | 2 | -0.0008 | 0.0041 | -0.0327 | 0.0723 |
| CIg | 120 | 60 | 3 | -0.0013 | 0.0039 | -0.0675 | 0.0748 |
| CIg | 180 | 60 | 3 | -0.0014 | 0.0039 | -0.0675 | 0.0748 |
| CI | 120 | 60 | 2 | -0.0386 | 0.0173 | -0.1368 | 0.0753 |
| CI | 180 | 60 | 2 | -0.0386 | 0.0173 | -0.1368 | 0.0753 |
| CI | 180 | 60 | 3 | -0.0571 | 0.0256 | -0.2061 | 0.0755 |
| CI | 120 | 60 | 3 | -0.0847 | 0.0497 | -0.2952 | 0.0756 |
Use ci_select_best() to keep the best setting within
each criterion type.
tree_best <- ci_select_best(
tree_tuning,
metric = "validation_gain"
)
knitr::kable(
tree_best[
,
.(type, minsplit, minbucket, minprob, maxdepth)
],
digits = 4,
caption = "Selected tree controls"
)| type | minsplit | minbucket | minprob | maxdepth |
|---|---|---|---|---|
| CI | 120 | 60 | 0.05 | 2 |
| CIg | 120 | 60 | 0.05 | 2 |
For reports, ci_fit_summary_table() combines selected
settings, validation metrics, training diagnostics, and root-objective
summaries.
tree_fit_summary <- ci_fit_summary_table(
tree_tuning,
selected = tree_best,
metrics = c(
"train_gain",
"validation_gain",
"train_relative_gain",
"relative_validation_gain"
),
include_percent = TRUE
)
knitr::kable(
tree_fit_summary[
,
.(
type,
mean_root_objective,
mean_train_gain,
mean_percent_train_gain,
mean_validation_gain,
mean_percent_validation_gain,
std_err_validation_gain
)
],
digits = 4,
caption = "Selected tree fit summary"
)| type | mean_root_objective | mean_train_gain | mean_percent_train_gain | mean_validation_gain | mean_percent_validation_gain | std_err_validation_gain |
|---|---|---|---|---|---|---|
| CI | 0.2880 | 0.1841 | 63.9471 | -0.0386 | -13.3993 | 0.0173 |
| CIg | 0.0209 | 0.0079 | 37.9971 | -0.0008 | -3.8397 | 0.0041 |
Saved validation predictions are available when
save_pred = TRUE.
head(
ci_collect_predictions(tree_tuning)[
,
.(grid_id, fold_id, type, row_id, outcome, .pred)
]
)
#> grid_id fold_id type row_id outcome .pred
#> <int> <int> <char> <int> <num> <num>
#> 1: 1 1 CI 2 0 0.00985935
#> 2: 1 1 CI 4 0 0.12775741
#> 3: 1 1 CI 10 1 0.12775741
#> 4: 1 1 CI 13 0 0.12775741
#> 5: 1 1 CI 15 0 0.12775741
#> 6: 1 1 CI 19 0 0.12775741The selected refit is stored on the tuning object.
ci_tree_terminal_summary(tree_tuning$best_fit)[
,
.(node, n, depth, ci, outcome_percent, rule)
]
#> node n depth ci outcome_percent
#> <int> <int> <int> <num> <num>
#> 1: 3 280 2 0.03773129 1.333262
#> 2: 4 312 2 0.18348088 7.736424
#> 3: 5 108 1 0.34569839 23.847287
#> rule
#> <char>
#> 1: reg in {Mombasa, Kilifi, Tana River, Lamu, Taita Taveta, Garissa, Wajir, Mandera, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Kitui, Machakos, Makueni, Nyeri, Kirinyaga, Murang'a, Kiambu, Turkana, West Pokot, Uasin Gishu, Elgeyo-Marakwet, Baringo, Laikipia, Nakuru, Narok, Kajiado, Kericho, Bomet, Kakamega, Vihiga, Bungoma, Kisumu, Migori, Kisii, Nyamira, Nairobi} & reg in {Tana River, Lamu, Taita Taveta, Garissa, Wajir, Mandera, Isiolo, Meru, Embu, Nyeri, Kirinyaga, Murang'a, Turkana, West Pokot, Elgeyo-Marakwet, Baringo, Laikipia, Narok, Bomet, Vihiga, Kisumu, Kisii, Nyamira, Nairobi}
#> 2: reg in {Mombasa, Kilifi, Tana River, Lamu, Taita Taveta, Garissa, Wajir, Mandera, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Kitui, Machakos, Makueni, Nyeri, Kirinyaga, Murang'a, Kiambu, Turkana, West Pokot, Uasin Gishu, Elgeyo-Marakwet, Baringo, Laikipia, Nakuru, Narok, Kajiado, Kericho, Bomet, Kakamega, Vihiga, Bungoma, Kisumu, Migori, Kisii, Nyamira, Nairobi} & reg in {Mombasa, Kilifi, Marsabit, Tharaka-Nithi, Kitui, Machakos, Makueni, Kiambu, Uasin Gishu, Nakuru, Kajiado, Kericho, Kakamega, Bungoma, Migori}
#> 3: reg in {Kwale, Nyandarua, Samburu, Trans Nzoia, Nandi, Busia, Siaya, Homa Bay}Cross-Validate Forests
Forest tuning uses the same metric and collector API. The forest grid
also includes ntree and mtry.
forest_grid <- ci_tree_control_grid(
minsplit = 120L,
minbucket = 60L,
minprob = 0.05,
maxdepth = 2L,
mtry = c(1L, 2L),
ntree = c(6L, 10L)
)
forest_grid
#> minsplit minbucket minprob maxdepth min_gain min_relative_gain mtry ntree
#> <int> <int> <num> <int> <num> <num> <int> <int>
#> 1: 120 60 0.05 2 0 0 1 6
#> 2: 120 60 0.05 2 0 0 2 6
#> 3: 120 60 0.05 2 0 0 1 10
#> 4: 120 60 0.05 2 0 0 2 10tune_ci_forest() scores concentration-index validation
gain from the forest’s internal trees. Each member tree supplies a
validation-fold partition, the tree-level validation gains are averaged,
and prediction metrics such as Brier score are computed from the
averaged forest predictions. When refit = TRUE, the
selected forest is still accompanied by a surrogate tree for
interpretation.
forest_tuning <- tune_ci_forest(
formula = ci_formula,
data = tuning_data,
rank_name = "wealth",
outcome_name = "deadu5_num",
weights = tuning_data$sample_weight,
type = criterion_types,
control_grid = forest_grid,
v = 3L,
strata = "deadu5_num",
seed = 20260522,
metrics = tuning_metrics,
prediction_name = "forest_risk",
parallel_over = "none",
control = control_ci_tune(save_pred = TRUE),
refit = TRUE
)
forest_metrics_wide <- ci_collect_metrics(
forest_tuning,
metric = tuning_metrics,
format = "wide"
)
knitr::kable(
forest_metrics_wide[
order(-mean_validation_gain),
.(
type,
ntree,
mtry,
minbucket,
mean_validation_gain,
std_err_validation_gain,
mean_validation_relative_gain,
mean_brier
)
],
digits = 4,
caption = "Cross-validated forest metrics"
)| type | ntree | mtry | minbucket | mean_validation_gain | std_err_validation_gain | mean_validation_relative_gain | mean_brier |
|---|---|---|---|---|---|---|---|
| CI | 10 | 2 | 60 | 0.0159 | 0.0140 | 0.0501 | 0.0713 |
| CI | 10 | 1 | 60 | 0.0077 | 0.0057 | 0.0028 | 0.0692 |
| CIg | 10 | 1 | 60 | 0.0019 | 0.0016 | 0.0615 | 0.0690 |
| CIg | 10 | 2 | 60 | 0.0010 | 0.0016 | 0.0114 | 0.0701 |
| CIg | 6 | 2 | 60 | -0.0003 | 0.0000 | -0.0110 | 0.0717 |
| CIg | 6 | 1 | 60 | -0.0003 | 0.0009 | 0.0024 | 0.0698 |
| CI | 6 | 1 | 60 | -0.0036 | 0.0285 | -0.0765 | 0.0701 |
| CI | 6 | 2 | 60 | -0.0052 | 0.0293 | -0.1068 | 0.0716 |
forest_best <- ci_select_best(
forest_tuning,
metric = "validation_gain"
)
knitr::kable(
forest_best[
,
.(type, ntree, mtry, minsplit, minbucket, maxdepth)
],
digits = 4,
caption = "Selected forest controls"
)| type | ntree | mtry | minsplit | minbucket | maxdepth |
|---|---|---|---|---|---|
| CI | 10 | 2 | 120 | 60 | 2 |
| CIg | 10 | 1 | 120 | 60 | 2 |
forest_fit_summary <- ci_fit_summary_table(
forest_tuning,
selected = forest_best,
metrics = c(
"train_gain",
"validation_gain",
"train_relative_gain",
"relative_validation_gain"
),
include_percent = TRUE
)
knitr::kable(
forest_fit_summary[
,
.(
type,
mean_root_objective,
mean_train_gain,
mean_percent_train_gain,
mean_validation_gain,
mean_percent_validation_gain,
std_err_validation_gain
)
],
digits = 4,
caption = "Selected forest fit summary"
)| type | mean_root_objective | mean_train_gain | mean_percent_train_gain | mean_validation_gain | mean_percent_validation_gain | std_err_validation_gain |
|---|---|---|---|---|---|---|
| CI | 0.2998 | 0.0574 | 19.1426 | 0.0159 | 5.3161 | 0.0140 |
| CIg | 0.0214 | 0.0019 | 8.7779 | 0.0019 | 9.0248 | 0.0016 |
With refit = TRUE, the final fitted forest and its
surrogate tree are stored on the tuning object.
ci_forest_summary(forest_tuning$best_fit)
#> ntree mtry type n mean_outcome mean_prediction outcome_ci
#> <int> <int> <char> <int> <num> <num> <num>
#> 1: 10 2 CI 700 0.07412078 0.07009817 0.2915001
#> prediction_ci mean_terminal_nodes mean_max_depth
#> <num> <num> <num>
#> 1: 0.06744326 2.9 1.8
ci_tree_terminal_summary(forest_tuning$best_surrogate)[
,
.(node, n, depth, outcome_percent, rule)
]
#> node n depth outcome_percent
#> <int> <int> <int> <num>
#> 1: 2 234 1 3.066650
#> 2: 4 288 2 6.904813
#> 3: 5 178 2 12.880643
#> rule
#> <char>
#> 1: reg in {Tana River, Lamu, Taita Taveta, Garissa, Makueni, Nyeri, Kirinyaga, Murang'a, Turkana, West Pokot, Elgeyo-Marakwet, Baringo, Laikipia, Narok, Bomet, Kisumu, Kisii, Nyamira, Nairobi}
#> 2: reg in {Mombasa, Kwale, Kilifi, Wajir, Mandera, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Kitui, Machakos, Nyandarua, Kiambu, Samburu, Trans Nzoia, Uasin Gishu, Nandi, Nakuru, Kajiado, Kericho, Kakamega, Vihiga, Bungoma, Busia, Siaya, Homa Bay, Migori} & reg in {Mombasa, Wajir, Mandera, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Machakos, Kiambu, Uasin Gishu, Nandi, Kajiado, Kericho, Kakamega, Vihiga, Bungoma}
#> 3: reg in {Mombasa, Kwale, Kilifi, Wajir, Mandera, Marsabit, Isiolo, Meru, Tharaka-Nithi, Embu, Kitui, Machakos, Nyandarua, Kiambu, Samburu, Trans Nzoia, Uasin Gishu, Nandi, Nakuru, Kajiado, Kericho, Kakamega, Vihiga, Bungoma, Busia, Siaya, Homa Bay, Migori} & reg in {Kwale, Kilifi, Kitui, Nyandarua, Samburu, Trans Nzoia, Nakuru, Busia, Siaya, Homa Bay, Migori}Minimal Template
For most analyses, the tuning pattern is:
grid <- ci_tree_control_grid(
minsplit = c(100L, 200L),
minbucket = c(50L, 100L),
maxdepth = 2:4
)
tuned <- tune_ci_tree(
formula = cbind(rank, outcome) ~ x1 + x2 + x3,
data = analysis_data,
rank_name = "rank",
outcome_name = "outcome",
weights = analysis_data$weights,
type = c("CI", "CIg"),
control_grid = grid,
v = 5L,
strata = "outcome",
metrics = c("validation_gain", "relative_validation_gain"),
control = control_ci_tune(save_pred = TRUE),
refit = TRUE
)
selected <- ci_select_best(tuned, metric = "validation_gain")
ci_fit_summary_table(
tuned,
selected = selected,
metrics = c(
"train_gain",
"validation_gain",
"train_relative_gain",
"relative_validation_gain"
)
)Use tune_ci_forest() with a grid that includes
ntree and mtry when you want the same
cross-validation workflow for forests.