Medial Code Documentation
Loading...
Searching...
No Matches
test_auc.h
1
4#pragma once
5
6#include <xgboost/metric.h>
7
8#include "../helpers.h"
9
10namespace xgboost {
11namespace metric {
12
13inline void VerifyBinaryAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
14 auto ctx = MakeCUDACtx(GPUIDX);
15 std::unique_ptr<Metric> uni_ptr{Metric::Create("auc", &ctx)};
16 Metric* metric = uni_ptr.get();
17 ASSERT_STREQ(metric->Name(), "auc");
18
19 // Binary
20 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 1.0f, 1e-10);
21 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {1, 0}, {}, {}, data_split_mode), 0.0f, 1e-10);
22 EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10);
23 EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10);
24 EXPECT_NEAR(GetMetricEval(metric, {0, 0}, {1, 0}, {}, {}, data_split_mode), 0.5f, 1e-10);
25 EXPECT_NEAR(GetMetricEval(metric, {1, 1}, {1, 0}, {}, {}, data_split_mode), 0.5f, 1e-10);
26 EXPECT_NEAR(GetMetricEval(metric, {1, 0, 0}, {0, 0, 1}, {}, {}, data_split_mode), 0.25f, 1e-10);
27
28 // Invalid dataset
29 auto p_fmat = EmptyDMatrix();
30 MetaInfo& info = p_fmat->Info();
31 info.labels = linalg::Tensor<float, 2>{{0.0f, 0.0f}, {2}, -1};
32 float auc = metric->Evaluate({1, 1}, p_fmat);
33 ASSERT_TRUE(std::isnan(auc));
34 *info.labels.Data() = HostDeviceVector<float>{};
35 auc = metric->Evaluate(HostDeviceVector<float>{}, p_fmat);
36 ASSERT_TRUE(std::isnan(auc));
37
38 EXPECT_NEAR(GetMetricEval(metric, {0, 1, 0, 1}, {0, 1, 0, 1}, {}, {}, data_split_mode), 1.0f,
39 1e-10);
40
41 // AUC with instance weights
42 EXPECT_NEAR(GetMetricEval(metric, {0.9f, 0.1f, 0.4f, 0.3f}, {0, 0, 1, 1},
43 {1.0f, 3.0f, 2.0f, 4.0f}, {}, data_split_mode),
44 0.75f, 0.001f);
45
46 // regression test case
47 ASSERT_NEAR(GetMetricEval(metric, {0.79523796, 0.5201713, 0.79523796, 0.24273258, 0.53452194,
48 0.53452194, 0.24273258, 0.5201713, 0.79523796, 0.53452194,
49 0.24273258, 0.53452194, 0.79523796, 0.5201713, 0.24273258,
50 0.5201713, 0.5201713, 0.53452194, 0.5201713, 0.53452194},
51 {0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0}, {}, {},
52 data_split_mode),
53 0.5, 1e-10);
54}
55
56inline void VerifyMultiClassAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
57 auto ctx = MakeCUDACtx(GPUIDX);
58 std::unique_ptr<Metric> uni_ptr{Metric::Create("auc", &ctx)};
59 auto metric = uni_ptr.get();
60
61 // MultiClass
62 // 3x3
63 EXPECT_NEAR(GetMetricEval(metric,
64 {
65 1.0f, 0.0f, 0.0f, // p_0
66 0.0f, 1.0f, 0.0f, // p_1
67 0.0f, 0.0f, 1.0f // p_2
68 },
69 {0, 1, 2}, {}, {}, data_split_mode),
70 1.0f, 1e-10);
71
72 EXPECT_NEAR(GetMetricEval(metric,
73 {
74 1.0f, 0.0f, 0.0f, // p_0
75 0.0f, 1.0f, 0.0f, // p_1
76 0.0f, 0.0f, 1.0f // p_2
77 },
78 {0, 1, 2}, {1.0f, 1.0f, 1.0f}, {}, data_split_mode),
79 1.0f, 1e-10);
80
81 EXPECT_NEAR(GetMetricEval(metric,
82 {
83 1.0f, 0.0f, 0.0f, // p_0
84 0.0f, 1.0f, 0.0f, // p_1
85 0.0f, 0.0f, 1.0f // p_2
86 },
87 {2, 1, 0}, {}, {}, data_split_mode),
88 0.5f, 1e-10);
89
90 EXPECT_NEAR(GetMetricEval(metric,
91 {
92 1.0f, 0.0f, 0.0f, // p_0
93 0.0f, 1.0f, 0.0f, // p_1
94 0.0f, 0.0f, 1.0f // p_2
95 },
96 {2, 0, 1}, {}, {}, data_split_mode),
97 0.25f, 1e-10);
98
99 // invalid dataset
100 float auc = GetMetricEval(metric,
101 {
102 1.0f, 0.0f, 0.0f, // p_0
103 0.0f, 1.0f, 0.0f, // p_1
104 0.0f, 0.0f, 1.0f // p_2
105 },
106 {0, 1, 1}, {}, {}, data_split_mode); // no class 2.
107 EXPECT_TRUE(std::isnan(auc)) << auc;
108
109 HostDeviceVector<float> predts{
110 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f,
111 };
112 std::vector<float> labels{1.0f, 0.0f, 2.0f, 1.0f};
113 auc = GetMetricEval(metric, predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}, {}, data_split_mode);
114 ASSERT_GT(auc, 0.714);
115}
116
117inline void VerifyRankingAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
118 auto ctx = MakeCUDACtx(GPUIDX);
119 std::unique_ptr<Metric> metric{Metric::Create("auc", &ctx)};
120
121 // single group
122 EXPECT_NEAR(GetMetricEval(metric.get(), {0.7f, 0.2f, 0.3f, 0.6f}, {1.0f, 0.8f, 0.4f, 0.2f},
123 /*weights=*/{}, {0, 4}, data_split_mode),
124 0.5f, 1e-10);
125
126 // multi group
127 EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, {0, 1, 2, 0, 1, 2}, /*weights=*/{},
128 {0, 3, 6}, data_split_mode),
129 1.0f, 1e-10);
130
131 EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1, 2, 0, 1, 2}, {0, 1, 2, 0, 1, 2},
132 /*weights=*/{1.0f, 2.0f}, {0, 3, 6}, data_split_mode),
133 1.0f, 1e-10);
134
135 // AUC metric for grouped datasets - exception scenarios
136 ASSERT_TRUE(std::isnan(
137 GetMetricEval(metric.get(), {0, 1, 2}, {0, 0, 0}, {}, {0, 2, 3}, data_split_mode)));
138
139 // regression case
140 HostDeviceVector<float> predt{
141 0.33935383, 0.5149714, 0.32138085, 1.4547751, 1.2010975, 0.42651367, 0.23104341, 0.83610827,
142 0.8494239, 0.07136688, 0.5623144, 0.8086237, 1.5066161, -4.094787, 0.76887935, -2.4082742};
143 std::vector<bst_group_t> groups{0, 7, 16};
144 std::vector<float> labels{1., 0., 0., 1., 2., 1., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0.};
145
146 EXPECT_NEAR(GetMetricEval(metric.get(), std::move(predt), labels,
147 /*weights=*/{}, groups, data_split_mode),
148 0.769841f, 1e-6);
149}
150
151inline void VerifyPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
152 auto ctx = MakeCUDACtx(GPUIDX);
153
154 xgboost::Metric* metric = xgboost::Metric::Create("aucpr", &ctx);
155 ASSERT_STREQ(metric->Name(), "aucpr");
156 EXPECT_NEAR(GetMetricEval(metric, {0, 0, 1, 1}, {0, 0, 1, 1}, {}, {}, data_split_mode), 1, 1e-10);
157 EXPECT_NEAR(
158 GetMetricEval(metric, {0.1f, 0.9f, 0.1f, 0.9f}, {0, 0, 1, 1}, {}, {}, data_split_mode), 0.5f,
159 0.001f);
160 EXPECT_NEAR(GetMetricEval(metric, {0.4f, 0.2f, 0.9f, 0.1f, 0.2f, 0.4f, 0.1f, 0.1f, 0.2f, 0.1f},
161 {0, 0, 0, 0, 0, 1, 0, 0, 1, 1}, {}, {}, data_split_mode),
162 0.2908445f, 0.001f);
163 EXPECT_NEAR(
164 GetMetricEval(metric, {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, 0.09f, 0.10f, 0.97f,
165 0.76f, 0.69f, 0.15f, 0.20f, 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
166 {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, {}, {},
167 data_split_mode),
168 0.2769199f, 0.001f);
169 auto auc = GetMetricEval(metric, {0, 1}, {}, {}, {}, data_split_mode);
170 ASSERT_TRUE(std::isnan(auc));
171
172 // AUCPR with instance weights
173 EXPECT_NEAR(GetMetricEval(metric,
174 {0.29f, 0.52f, 0.11f, 0.21f, 0.219f, 0.93f, 0.493f, 0.17f, 0.47f, 0.13f,
175 0.43f, 0.59f, 0.87f, 0.007f},
176 {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0},
177 {1, 2, 7, 4, 5, 2.2f, 3.2f, 5, 6, 1, 2, 1.1f, 3.2f, 4.5f}, // weights
178 {}, data_split_mode),
179 0.694435f, 0.001f);
180
181 // Both groups contain only pos or neg samples.
182 auc = GetMetricEval(metric, {0, 0.1f, 0.3f, 0.5f, 0.7f}, {1, 1, 0, 0, 0}, {}, {0, 2, 5},
183 data_split_mode);
184 ASSERT_TRUE(std::isnan(auc));
185 delete metric;
186}
187
188inline void VerifyMultiClassPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
189 auto ctx = MakeCUDACtx(GPUIDX);
190
191 std::unique_ptr<Metric> metric{Metric::Create("aucpr", &ctx)};
192
193 float auc = 0;
194 std::vector<float> labels{1.0f, 0.0f, 2.0f};
195 HostDeviceVector<float> predts{
196 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f,
197 };
198 auc = GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode);
199 EXPECT_EQ(auc, 1.0f);
200
201 auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 1.0f, 1.0f}, {}, data_split_mode);
202 EXPECT_EQ(auc, 1.0f);
203
204 predts.HostVector() = {
205 0.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f,
206 };
207 labels = {1.0f, 0.0f, 2.0f, 1.0f};
208 auc = GetMetricEval(metric.get(), predts, labels, {1.0f, 2.0f, 3.0f, 4.0f}, {}, data_split_mode);
209 ASSERT_GT(auc, 0.699);
210}
211
212inline void VerifyRankingPRAUC(DataSplitMode data_split_mode = DataSplitMode::kRow) {
213 auto ctx = MakeCUDACtx(GPUIDX);
214
215 std::unique_ptr<Metric> metric{Metric::Create("aucpr", &ctx)};
216
217 std::vector<float> labels{1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f};
218 std::vector<uint32_t> groups{0, 2, 6};
219
220 float auc = 0;
221 auc = GetMetricEval(metric.get(), {1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f}, labels, {}, groups,
222 data_split_mode);
223 EXPECT_EQ(auc, 1.0f);
224
225 auc = GetMetricEval(metric.get(), {1.0f, 0.5f, 0.8f, 0.3f, 0.2f, 1.0f}, labels, {}, groups,
226 data_split_mode);
227 EXPECT_EQ(auc, 1.0f);
228
229 auc = GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f},
230 {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f}, {}, groups, data_split_mode);
231 ASSERT_TRUE(std::isnan(auc));
232
233 // Incorrect label
234 ASSERT_THROW(GetMetricEval(metric.get(), {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f},
235 {1.0f, 1.0f, 0.0f, 0.0f, 0.0f, 3.0f}, {}, groups, data_split_mode),
236 dmlc::Error);
237
238 // AUCPR with groups and no weights
239 EXPECT_NEAR(
240 GetMetricEval(metric.get(),
241 {0.87f, 0.31f, 0.40f, 0.42f, 0.25f, 0.66f, 0.95f, 0.09f, 0.10f, 0.97f,
242 0.76f, 0.69f, 0.15f, 0.20f, 0.30f, 0.14f, 0.07f, 0.58f, 0.61f, 0.08f},
243 {0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1}, {}, // weights
244 {0, 2, 5, 9, 14, 20}, // group info
245 data_split_mode),
246 0.556021f, 0.001f);
247}
248} // namespace metric
249} // namespace xgboost
xgboost::Metric
interface of evaluation metric used to evaluate model performance. This has nothing to do with traini...
Definition metric.h:29
xgboost::Metric::Create
static Metric * Create(const std::string &name, Context const *ctx)
create a metric according to name.
Definition metric.cc:46
xgboost::Metric::Name
virtual const char * Name() const =0
xgboost
namespace of xgboost
Definition base.h:90
xgboost::MakeCUDACtx
Context MakeCUDACtx(std::int32_t device)
Make a context that uses CUDA if device >= 0.
Definition helpers.h:410
dmlc::Error
exception class that will be thrown by default logger if DMLC_LOG_FATAL_THROW == 1
Definition logging.h:29
metric.h
Copyright 2014-2023 by XGBoost Contributors.
Generated on Mon Sep 15 2025 12:12:20 for Medial Code Documentation by doxygen 1.9.8