Medial Code Documentation
Loading...
Searching...
No Matches
test_elementwise_metric.h
1
4#pragma once
5#include <xgboost/json.h>
6#include <xgboost/metric.h>
7
8#include <map>
9#include <memory>
10
11#include "../../../src/common/linalg_op.h"
12#include "../helpers.h"
13
14namespace xgboost {
15namespace metric {
16
17inline void CheckDeterministicMetricElementWise(StringView name, int32_t device) {
18 auto ctx = MakeCUDACtx(device);
19 std::unique_ptr<Metric> metric{Metric::Create(name.c_str(), &ctx)};
20
21 HostDeviceVector<float> predts;
22 size_t n_samples = 2048;
23
24 auto p_fmat = EmptyDMatrix();
25 MetaInfo& info = p_fmat->Info();
26 info.labels.Reshape(n_samples, 1);
27 info.num_row_ = n_samples;
28 auto &h_labels = info.labels.Data()->HostVector();
29 auto &h_predts = predts.HostVector();
30
31 SimpleLCG lcg;
32 SimpleRealUniformDistribution<float> dist{0.0f, 1.0f};
33
34 h_labels.resize(n_samples);
35 h_predts.resize(n_samples);
36
37 for (size_t i = 0; i < n_samples; ++i) {
38 h_predts[i] = dist(&lcg);
39 h_labels[i] = dist(&lcg);
40 }
41
42 auto result = metric->Evaluate(predts, p_fmat);
43 for (size_t i = 0; i < 8; ++i) {
44 ASSERT_EQ(metric->Evaluate(predts, p_fmat), result);
45 }
46}
47
48inline void VerifyRMSE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
49 auto ctx = MakeCUDACtx(GPUIDX);
50 xgboost::Metric * metric = xgboost::Metric::Create("rmse", &ctx);
51 metric->Configure({});
52 ASSERT_STREQ(metric->Name(), "rmse");
53 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
54 EXPECT_NEAR(GetMetricEval(metric,
55 {0.1f, 0.9f, 0.1f, 0.9f},
56 { 0, 0, 1, 1}, {}, {}, data_split_mode),
57 0.6403f, 0.001f);
58 auto expected = 2.8284f;
59 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
60 expected = sqrt(8.0f * collective::GetWorldSize());
61 }
62 EXPECT_NEAR(GetMetricEval(metric,
63 {0.1f, 0.9f, 0.1f, 0.9f},
64 { 0, 0, 1, 1},
65 { -1, 1, 9, -9}, {}, data_split_mode),
66 expected, 0.001f);
67 EXPECT_NEAR(GetMetricEval(metric,
68 {0.1f, 0.9f, 0.1f, 0.9f},
69 { 0, 0, 1, 1},
70 { 1, 2, 9, 8}, {}, data_split_mode),
71 0.6708f, 0.001f);
72 delete metric;
73
74 CheckDeterministicMetricElementWise(StringView{"rmse"}, GPUIDX);
75}
76
77inline void VerifyRMSLE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
78 auto ctx = MakeCUDACtx(GPUIDX);
79 xgboost::Metric * metric = xgboost::Metric::Create("rmsle", &ctx);
80 metric->Configure({});
81 ASSERT_STREQ(metric->Name(), "rmsle");
82 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
83 EXPECT_NEAR(GetMetricEval(metric,
84 {0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
85 {1.0f, 1.0f, 1.0f, 1.0f, 1.0f}, {}, {}, data_split_mode),
86 0.4063f, 1e-4);
87 auto expected = 0.6212f;
88 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
89 expected = sqrt(0.3859f * collective::GetWorldSize());
90 }
91 EXPECT_NEAR(GetMetricEval(metric,
92 {0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
93 {1.0f, 1.0f, 1.0f, 1.0f, 1.0f},
94 { 0, -1, 1, -9, 9}, {}, data_split_mode),
95 expected, 1e-4);
96 EXPECT_NEAR(GetMetricEval(metric,
97 {0.1f, 0.2f, 0.4f, 0.8f, 1.6f},
98 {1.0f, 1.0f, 1.0f, 1.0f, 1.0f},
99 { 0, 1, 2, 9, 8}, {}, data_split_mode),
100 0.2415f, 1e-4);
101 delete metric;
102
103 CheckDeterministicMetricElementWise(StringView{"rmsle"}, GPUIDX);
104}
105
106inline void VerifyMAE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
107 auto ctx = MakeCUDACtx(GPUIDX);
108 xgboost::Metric * metric = xgboost::Metric::Create("mae", &ctx);
109 metric->Configure({});
110 ASSERT_STREQ(metric->Name(), "mae");
111 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
112 EXPECT_NEAR(GetMetricEval(metric,
113 {0.1f, 0.9f, 0.1f, 0.9f},
114 { 0, 0, 1, 1}, {}, {}, data_split_mode),
115 0.5f, 0.001f);
116 auto expected = 8.0f;
117 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
118 expected *= collective::GetWorldSize();
119 }
120 EXPECT_NEAR(GetMetricEval(metric,
121 {0.1f, 0.9f, 0.1f, 0.9f},
122 { 0, 0, 1, 1},
123 { -1, 1, 9, -9}, {}, data_split_mode),
124 expected, 0.001f);
125 EXPECT_NEAR(GetMetricEval(metric,
126 {0.1f, 0.9f, 0.1f, 0.9f},
127 { 0, 0, 1, 1},
128 { 1, 2, 9, 8}, {}, data_split_mode),
129 0.54f, 0.001f);
130 delete metric;
131
132 CheckDeterministicMetricElementWise(StringView{"mae"}, GPUIDX);
133}
134
135inline void VerifyMAPE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
136 auto ctx = MakeCUDACtx(GPUIDX);
137 xgboost::Metric * metric = xgboost::Metric::Create("mape", &ctx);
138 metric->Configure({});
139 ASSERT_STREQ(metric->Name(), "mape");
140 EXPECT_NEAR(GetMetricEval(metric, {150, 300}, {100, 200}, {}, {}, data_split_mode), 0.5f, 1e-10);
141 EXPECT_NEAR(GetMetricEval(metric,
142 {50, 400, 500, 4000},
143 {100, 200, 500, 1000}, {}, {}, data_split_mode),
144 1.125f, 0.001f);
145 auto expected = -26.5f;
146 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
147 expected *= collective::GetWorldSize();
148 }
149 EXPECT_NEAR(GetMetricEval(metric,
150 {50, 400, 500, 4000},
151 {100, 200, 500, 1000},
152 { -1, 1, 9, -9}, {}, data_split_mode),
153 expected, 0.001f);
154 EXPECT_NEAR(GetMetricEval(metric,
155 {50, 400, 500, 4000},
156 {100, 200, 500, 1000},
157 { 1, 2, 9, 8}, {}, data_split_mode),
158 1.3250f, 0.001f);
159 delete metric;
160
161 CheckDeterministicMetricElementWise(StringView{"mape"}, GPUIDX);
162}
163
164inline void VerifyMPHE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
165 auto ctx = MakeCUDACtx(GPUIDX);
166 std::unique_ptr<xgboost::Metric> metric{xgboost::Metric::Create("mphe", &ctx)};
167 metric->Configure({});
168 ASSERT_STREQ(metric->Name(), "mphe");
169 EXPECT_NEAR(GetMetricEval(metric.get(), {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
170 EXPECT_NEAR(GetMetricEval(metric.get(),
171 {0.1f, 0.9f, 0.1f, 0.9f},
172 { 0, 0, 1, 1}, {}, {}, data_split_mode),
173 0.1751f, 1e-4);
174 auto expected = 3.40375f;
175 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
176 expected *= collective::GetWorldSize();
177 }
178 EXPECT_NEAR(GetMetricEval(metric.get(),
179 {0.1f, 0.9f, 0.1f, 0.9f},
180 { 0, 0, 1, 1},
181 { -1, 1, 9, -9}, {}, data_split_mode),
182 expected, 1e-4);
183 EXPECT_NEAR(GetMetricEval(metric.get(),
184 {0.1f, 0.9f, 0.1f, 0.9f},
185 { 0, 0, 1, 1},
186 { 1, 2, 9, 8}, {}, data_split_mode),
187 0.1922f, 1e-4);
188
189 CheckDeterministicMetricElementWise(StringView{"mphe"}, GPUIDX);
190
191 metric->Configure({{"huber_slope", "0.1"}});
192 EXPECT_NEAR(GetMetricEval(metric.get(),
193 {0.1f, 0.9f, 0.1f, 0.9f},
194 { 0, 0, 1, 1},
195 { 1, 2, 9, 8}, {}, data_split_mode),
196 0.0461686f, 1e-4);
197}
198
199inline void VerifyLogLoss(DataSplitMode data_split_mode = DataSplitMode::kRow) {
200 auto ctx = MakeCUDACtx(GPUIDX);
201 xgboost::Metric * metric = xgboost::Metric::Create("logloss", &ctx);
202 metric->Configure({});
203 ASSERT_STREQ(metric->Name(), "logloss");
204 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
205 EXPECT_NEAR(GetMetricEval(metric,
206 {0.5f, 1e-17f, 1.0f+1e-17f, 0.9f},
207 { 0, 0, 1, 1}, {}, {}, data_split_mode),
208 0.1996f, 0.001f);
209 EXPECT_NEAR(GetMetricEval(metric,
210 {0.1f, 0.9f, 0.1f, 0.9f},
211 { 0, 0, 1, 1}, {}, {}, data_split_mode),
212 1.2039f, 0.001f);
213 auto expected = 21.9722f;
214 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
215 expected *= collective::GetWorldSize();
216 }
217 EXPECT_NEAR(GetMetricEval(metric,
218 {0.1f, 0.9f, 0.1f, 0.9f},
219 { 0, 0, 1, 1},
220 { -1, 1, 9, -9}, {}, data_split_mode),
221 expected, 0.001f);
222 EXPECT_NEAR(GetMetricEval(metric,
223 {0.1f, 0.9f, 0.1f, 0.9f},
224 { 0, 0, 1, 1},
225 { 1, 2, 9, 8}, {}, data_split_mode),
226 1.3138f, 0.001f);
227 delete metric;
228
229 CheckDeterministicMetricElementWise(StringView{"logloss"}, GPUIDX);
230}
231
232inline void VerifyError(DataSplitMode data_split_mode = DataSplitMode::kRow) {
233 auto ctx = MakeCUDACtx(GPUIDX);
234 xgboost::Metric * metric = xgboost::Metric::Create("error", &ctx);
235 metric->Configure({});
236 ASSERT_STREQ(metric->Name(), "error");
237 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
238 EXPECT_NEAR(GetMetricEval(metric,
239 {0.1f, 0.9f, 0.1f, 0.9f},
240 { 0, 0, 1, 1}, {}, {}, data_split_mode),
241 0.5f, 0.001f);
242 auto expected = 10.0f;
243 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
244 expected *= collective::GetWorldSize();
245 }
246 EXPECT_NEAR(GetMetricEval(metric,
247 {0.1f, 0.9f, 0.1f, 0.9f},
248 { 0, 0, 1, 1},
249 { -1, 1, 9, -9}, {}, data_split_mode),
250 expected, 0.001f);
251 EXPECT_NEAR(GetMetricEval(metric,
252 {0.1f, 0.9f, 0.1f, 0.9f},
253 { 0, 0, 1, 1},
254 { 1, 2, 9, 8}, {}, data_split_mode),
255 0.55f, 0.001f);
256
257 EXPECT_ANY_THROW(xgboost::Metric::Create("error@abc", &ctx));
258 delete metric;
259
260 metric = xgboost::Metric::Create("error@0.5f", &ctx);
261 metric->Configure({});
262 EXPECT_STREQ(metric->Name(), "error");
263
264 delete metric;
265
266 metric = xgboost::Metric::Create("error@0.1", &ctx);
267 metric->Configure({});
268 ASSERT_STREQ(metric->Name(), "error@0.1");
269 EXPECT_STREQ(metric->Name(), "error@0.1");
270 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0, 1e-10);
271 EXPECT_NEAR(GetMetricEval(metric,
272 {-0.1f, -0.9f, 0.1f, 0.9f},
273 { 0, 0, 1, 1}, {}, {}, data_split_mode),
274 0.25f, 0.001f);
275 expected = 9.0f;
276 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
277 expected *= collective::GetWorldSize();
278 }
279 EXPECT_NEAR(GetMetricEval(metric,
280 {-0.1f, -0.9f, 0.1f, 0.9f},
281 { 0, 0, 1, 1},
282 { -1, 1, 9, -9}, {}, data_split_mode),
283 expected, 0.001f);
284 EXPECT_NEAR(GetMetricEval(metric,
285 {-0.1f, -0.9f, 0.1f, 0.9f},
286 { 0, 0, 1, 1},
287 { 1, 2, 9, 8}, {}, data_split_mode),
288 0.45f, 0.001f);
289 delete metric;
290
291 CheckDeterministicMetricElementWise(StringView{"error@0.5"}, GPUIDX);
292}
293
294inline void VerifyPoissonNegLogLik(DataSplitMode data_split_mode = DataSplitMode::kRow) {
295 auto ctx = MakeCUDACtx(GPUIDX);
296 xgboost::Metric * metric = xgboost::Metric::Create("poisson-nloglik", &ctx);
297 metric->Configure({});
298 ASSERT_STREQ(metric->Name(), "poisson-nloglik");
299 EXPECT_NEAR(GetMetricEval(metric, {0, 1}, {0, 1}, {}, {}, data_split_mode), 0.5f, 1e-10);
300 EXPECT_NEAR(GetMetricEval(metric,
301 {0.5f, 1e-17f, 1.0f+1e-17f, 0.9f},
302 { 0, 0, 1, 1}, {}, {}, data_split_mode),
303 0.6263f, 0.001f);
304 EXPECT_NEAR(GetMetricEval(metric,
305 {0.1f, 0.9f, 0.1f, 0.9f},
306 { 0, 0, 1, 1}, {}, {}, data_split_mode),
307 1.1019f, 0.001f);
308 auto expected = 13.3750f;
309 if (collective::IsDistributed() && data_split_mode == DataSplitMode::kRow) {
310 expected *= collective::GetWorldSize();
311 }
312 EXPECT_NEAR(GetMetricEval(metric,
313 {0.1f, 0.9f, 0.1f, 0.9f},
314 { 0, 0, 1, 1},
315 { -1, 1, 9, -9}, {}, data_split_mode),
316 expected, 0.001f);
317 EXPECT_NEAR(GetMetricEval(metric,
318 {0.1f, 0.9f, 0.1f, 0.9f},
319 { 0, 0, 1, 1},
320 { 1, 2, 9, 8}, {}, data_split_mode),
321 1.5783f, 0.001f);
322 delete metric;
323
324 CheckDeterministicMetricElementWise(StringView{"poisson-nloglik"}, GPUIDX);
325}
326
327inline void VerifyMultiRMSE(DataSplitMode data_split_mode = DataSplitMode::kRow) {
328 size_t n_samples = 32, n_targets = 8;
329 linalg::Tensor<float, 2> y{{n_samples, n_targets}, GPUIDX};
330 auto &h_y = y.Data()->HostVector();
331 std::iota(h_y.begin(), h_y.end(), 0);
332
333 HostDeviceVector<float> predt(n_samples * n_targets, 0);
334
335 auto ctx = MakeCUDACtx(GPUIDX);
336 std::unique_ptr<Metric> metric{Metric::Create("rmse", &ctx)};
337 metric->Configure({});
338
339 auto loss = GetMultiMetricEval(metric.get(), predt, y, {}, {}, data_split_mode);
340 std::vector<float> weights(n_samples, 1);
341 auto loss_w = GetMultiMetricEval(metric.get(), predt, y, weights, {}, data_split_mode);
342
343 std::transform(h_y.cbegin(), h_y.cend(), h_y.begin(), [](auto &v) { return v * v; });
344 auto ret = std::sqrt(std::accumulate(h_y.cbegin(), h_y.cend(), 1.0, std::plus<>{}) / h_y.size());
345 ASSERT_FLOAT_EQ(ret, loss);
346 ASSERT_FLOAT_EQ(ret, loss_w);
347}
348
349inline void VerifyQuantile(DataSplitMode data_split_mode = DataSplitMode::kRow) {
350 auto ctx = MakeCUDACtx(GPUIDX);
351 std::unique_ptr<Metric> metric{Metric::Create("quantile", &ctx)};
352
353 HostDeviceVector<float> predts{0.1f, 0.9f, 0.1f, 0.9f};
354 std::vector<float> labels{0.5f, 0.5f, 0.9f, 0.1f};
355 std::vector<float> weights{0.2f, 0.4f, 0.6f, 0.8f};
356
357 metric->Configure(Args{{"quantile_alpha", "[0.0]"}});
358 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.400f,
359 0.001f);
360 metric->Configure(Args{{"quantile_alpha", "[0.2]"}});
361 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.376f,
362 0.001f);
363 metric->Configure(Args{{"quantile_alpha", "[0.4]"}});
364 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.352f,
365 0.001f);
366 metric->Configure(Args{{"quantile_alpha", "[0.8]"}});
367 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.304f,
368 0.001f);
369 metric->Configure(Args{{"quantile_alpha", "[1.0]"}});
370 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, weights, {}, data_split_mode), 0.28f,
371 0.001f);
372
373 metric->Configure(Args{{"quantile_alpha", "[0.0]"}});
374 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
375 metric->Configure(Args{{"quantile_alpha", "[0.2]"}});
376 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
377 metric->Configure(Args{{"quantile_alpha", "[0.4]"}});
378 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
379 metric->Configure(Args{{"quantile_alpha", "[0.8]"}});
380 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
381 metric->Configure(Args{{"quantile_alpha", "[1.0]"}});
382 EXPECT_NEAR(GetMetricEval(metric.get(), predts, labels, {}, {}, data_split_mode), 0.3f, 0.001f);
383}
384} // namespace metric
385} // namespace xgboost
xgboost::Metric
interface of evaluation metric used to evaluate model performance. This has nothing to do with traini...
Definition metric.h:29
xgboost::Metric::Create
static Metric * Create(const std::string &name, Context const *ctx)
create a metric according to name.
Definition metric.cc:46
xgboost::Metric::Configure
virtual void Configure(const std::vector< std::pair< std::string, std::string > > &)
Configure the Metric with the specified parameters.
Definition metric.h:38
xgboost::Metric::Name
virtual const char * Name() const =0
xgboost::collective::GetWorldSize
int GetWorldSize()
Get total number of processes.
Definition communicator-inl.h:83
xgboost::collective::IsDistributed
bool IsDistributed()
Get if the communicator is distributed.
Definition communicator-inl.h:90
xgboost
namespace of xgboost
Definition base.h:90
xgboost::MakeCUDACtx
Context MakeCUDACtx(std::int32_t device)
Make a context that uses CUDA if device >= 0.
Definition helpers.h:410
metric.h
Copyright 2014-2023 by XGBoost Contributors.
Generated on Mon Sep 15 2025 12:12:20 for Medial Code Documentation by doxygen 1.9.8