Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix multiclass auc with empty dataset. #6947

Merged
merged 1 commit into from
May 12, 2021
Merged

Fix multiclass auc with empty dataset. #6947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 12 additions & 7 deletions src/metric/auc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,7 @@ std::tuple<float, float, float> BinaryAUC(std::vector<float> const &predts,
* - Kleiman, Ross and Page, David. $AUC_{\mu}$: A Performance Metric for Multi-Class
* Machine Learning Models
*/
float MultiClassOVR(std::vector<float> const& predts, MetaInfo const& info) {
auto n_classes = predts.size() / info.labels_.Size();
float MultiClassOVR(std::vector<float> const& predts, MetaInfo const& info, size_t n_classes) {
CHECK_NE(n_classes, 0);
auto const& labels = info.labels_.ConstHostVector();

Expand Down Expand Up @@ -230,6 +229,10 @@ class EvalAUC : public Metric {
info.labels_.SetDevice(tparam_->gpu_id);
info.weights_.SetDevice(tparam_->gpu_id);
}
// We use the global size to handle empty dataset.
std::array<size_t, 2> meta{info.labels_.Size(), preds.Size()};
rabit::Allreduce<rabit::op::Max>(meta.data(), meta.size());

if (!info.group_ptr_.empty()) {
/**
* learning to rank
Expand Down Expand Up @@ -261,16 +264,17 @@ class EvalAUC : public Metric {
CHECK_LE(auc, 1) << "Total AUC across groups: " << auc * valid_groups
<< ", valid groups: " << valid_groups;
}
} else if (info.labels_.Size() != preds.Size() &&
preds.Size() % info.labels_.Size() == 0) {
} else if (meta[0] != meta[1] && meta[1] % meta[0] == 0) {
/**
* multi class
*/
size_t n_classes = meta[1] / meta[0];
CHECK_NE(n_classes, 0);
if (tparam_->gpu_id == GenericParameter::kCpuId) {
auc = MultiClassOVR(preds.ConstHostVector(), info);
auc = MultiClassOVR(preds.ConstHostVector(), info, n_classes);
} else {
auc = GPUMultiClassAUCOVR(preds.ConstDeviceSpan(), info, tparam_->gpu_id,
&this->d_cache_);
&this->d_cache_, n_classes);
}
} else {
/**
Expand Down Expand Up @@ -323,7 +327,8 @@ GPUBinaryAUC(common::Span<float const> predts, MetaInfo const &info,
}

float GPUMultiClassAUCOVR(common::Span<float const> predts, MetaInfo const &info,
int32_t device, std::shared_ptr<DeviceAUCCache>* cache) {
int32_t device, std::shared_ptr<DeviceAUCCache>* cache,
size_t n_classes) {
common::AssertGPUSupport();
return 0;
}
Expand Down
86 changes: 55 additions & 31 deletions src/metric/auc.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,12 @@ struct DeviceAUCCache {
neg_pos.resize(sorted_idx.size());
if (is_multi) {
predts_t.resize(sorted_idx.size());
reducer.reset(new dh::AllReducer);
reducer->Init(rabit::GetRank());
}
}
if (is_multi && !reducer) {
reducer.reset(new dh::AllReducer);
reducer->Init(device);
}
}
};

Expand Down Expand Up @@ -197,12 +199,48 @@ XGBOOST_DEVICE size_t LastOf(size_t group, common::Span<Idx> indptr) {
return indptr[group + 1] - 1;
}


float ScaleClasses(common::Span<float> results, common::Span<float> local_area,
common::Span<float> fp, common::Span<float> tp,
common::Span<float> auc, std::shared_ptr<DeviceAUCCache> cache,
size_t n_classes) {
dh::XGBDeviceAllocator<char> alloc;
if (rabit::IsDistributed()) {
CHECK_EQ(dh::CudaGetPointerDevice(results.data()), dh::CurrentDevice());
cache->reducer->AllReduceSum(results.data(), results.data(), results.size());
}
auto reduce_in = dh::MakeTransformIterator<thrust::pair<float, float>>(
thrust::make_counting_iterator(0), [=] __device__(size_t i) {
if (local_area[i] > 0) {
return thrust::make_pair(auc[i] / local_area[i] * tp[i], tp[i]);
}
return thrust::make_pair(std::numeric_limits<float>::quiet_NaN(), 0.0f);
});

float tp_sum;
float auc_sum;
thrust::tie(auc_sum, tp_sum) = thrust::reduce(
thrust::cuda::par(alloc), reduce_in, reduce_in + n_classes,
thrust::make_pair(0.0f, 0.0f),
[=] __device__(auto const &l, auto const &r) {
return thrust::make_pair(l.first + r.first, l.second + r.second);
});
if (tp_sum != 0 && !std::isnan(auc_sum)) {
auc_sum /= tp_sum;
} else {
return std::numeric_limits<float>::quiet_NaN();
}
return auc_sum;
}

/**
* MultiClass implementation is similar to binary classification, except we need to split
* up each class in all kernels.
*/
float GPUMultiClassAUCOVR(common::Span<float const> predts, MetaInfo const &info,
int32_t device, std::shared_ptr<DeviceAUCCache>* p_cache) {
int32_t device, std::shared_ptr<DeviceAUCCache>* p_cache,
size_t n_classes) {
dh::safe_cuda(cudaSetDevice(device));
auto& cache = *p_cache;
if (!cache) {
cache.reset(new DeviceAUCCache);
Expand All @@ -213,8 +251,19 @@ float GPUMultiClassAUCOVR(common::Span<float const> predts, MetaInfo const &info
auto weights = info.weights_.ConstDeviceSpan();

size_t n_samples = labels.size();
size_t n_classes = predts.size() / labels.size();
CHECK_NE(n_classes, 0);

if (n_samples == 0) {
dh::TemporaryArray<float> resutls(n_classes * 4, 0.0f);
auto d_results = dh::ToSpan(resutls);
dh::LaunchN(device, n_classes * 4, [=]__device__(size_t i) {
d_results[i] = 0.0f;
});
auto local_area = d_results.subspan(0, n_classes);
auto fp = d_results.subspan(n_classes, n_classes);
auto tp = d_results.subspan(2 * n_classes, n_classes);
auto auc = d_results.subspan(3 * n_classes, n_classes);
return ScaleClasses(d_results, local_area, fp, tp, auc, cache, n_classes);
}

/**
* Create sorted index for each class
Expand Down Expand Up @@ -377,32 +426,7 @@ float GPUMultiClassAUCOVR(common::Span<float const> predts, MetaInfo const &info
tp[c] = last.second;
local_area[c] = last.first * last.second;
});
if (rabit::IsDistributed()) {
cache->reducer->AllReduceSum(resutls.data().get(), resutls.data().get(),
resutls.size());
}
auto reduce_in = dh::MakeTransformIterator<thrust::pair<float, float>>(
thrust::make_counting_iterator(0), [=] __device__(size_t i) {
if (local_area[i] > 0) {
return thrust::make_pair(auc[i] / local_area[i] * tp[i], tp[i]);
}
return thrust::make_pair(std::numeric_limits<float>::quiet_NaN(), 0.0f);
});

float tp_sum;
float auc_sum;
thrust::tie(auc_sum, tp_sum) = thrust::reduce(
thrust::cuda::par(alloc), reduce_in, reduce_in + n_classes,
thrust::make_pair(0.0f, 0.0f),
[=] __device__(auto const &l, auto const &r) {
return thrust::make_pair(l.first + r.first, l.second + r.second);
});
if (tp_sum != 0 && !std::isnan(auc_sum)) {
auc_sum /= tp_sum;
} else {
return std::numeric_limits<float>::quiet_NaN();
}
return auc_sum;
return ScaleClasses(d_results, local_area, fp, tp, auc, cache, n_classes);
}

namespace {
Expand Down
3 changes: 2 additions & 1 deletion src/metric/auc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ GPUBinaryAUC(common::Span<float const> predts, MetaInfo const &info,
int32_t device, std::shared_ptr<DeviceAUCCache> *p_cache);

float GPUMultiClassAUCOVR(common::Span<float const> predts, MetaInfo const &info,
int32_t device, std::shared_ptr<DeviceAUCCache>* cache);
int32_t device, std::shared_ptr<DeviceAUCCache>* cache,
size_t n_classes);

std::pair<float, uint32_t>
GPURankingAUC(common::Span<float const> predts, MetaInfo const &info,
Expand Down
2 changes: 1 addition & 1 deletion tests/python-gpu/test_gpu_with_dask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ def test_dask_classifier(
X = dask_cudf.from_dask_dataframe(dd.from_dask_array(X_))
y = dask_cudf.from_dask_dataframe(dd.from_dask_array(y_))
w = dask_cudf.from_dask_dataframe(dd.from_dask_array(w_))
run_dask_classifier(X, y, w, model, client, 10)
run_dask_classifier(X, y, w, model, "gpu_hist", client, 10)

@pytest.mark.skipif(**tm.no_dask())
@pytest.mark.skipif(**tm.no_dask_cuda())
Expand Down
21 changes: 13 additions & 8 deletions tests/python/test_with_dask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,18 +317,19 @@ def run_dask_classifier(
y: xgb.dask._DaskCollection,
w: xgb.dask._DaskCollection,
model: str,
tree_method: Optional[str],
client: "Client",
n_classes,
) -> None:
metric = "merror" if n_classes > 2 else "logloss"

if model == "boosting":
classifier = xgb.dask.DaskXGBClassifier(
verbosity=1, n_estimators=2, eval_metric=metric
verbosity=1, n_estimators=2, eval_metric=metric, tree_method=tree_method
)
else:
classifier = xgb.dask.DaskXGBRFClassifier(
verbosity=1, n_estimators=2, eval_metric=metric
verbosity=1, n_estimators=2, eval_metric=metric, tree_method=tree_method
)

assert classifier._estimator_type == "classifier"
Expand Down Expand Up @@ -397,12 +398,12 @@ def run_dask_classifier(
def test_dask_classifier(model: str, client: "Client") -> None:
X, y, w = generate_array(with_weights=True)
y = (y * 10).astype(np.int32)
run_dask_classifier(X, y, w, model, client, 10)
run_dask_classifier(X, y, w, model, None, client, 10)

y_bin = y.copy()
y_bin[y > 5] = 1.0
y_bin[y <= 5] = 0.0
run_dask_classifier(X, y_bin, w, model, client, 2)
run_dask_classifier(X, y_bin, w, model, None, client, 2)


@pytest.mark.skipif(**tm.no_sklearn())
Expand Down Expand Up @@ -568,22 +569,26 @@ def run_empty_dmatrix_auc(client: "Client", tree_method: str, n_workers: int) ->
# multiclass
X_, y_ = make_classification(
n_samples=n_samples,
n_classes=10,
n_classes=n_workers,
n_informative=n_features,
n_redundant=0,
n_repeated=0
)
for i in range(y_.shape[0]):
y_[i] = i % n_workers
X = dd.from_array(X_, chunksize=10)
y = dd.from_array(y_, chunksize=10)

n_samples = n_workers - 1
valid_X_, valid_y_ = make_classification(
n_samples=n_samples,
n_classes=10,
n_classes=n_workers,
n_informative=n_features,
n_redundant=0,
n_repeated=0
)
for i in range(valid_y_.shape[0]):
valid_y_[i] = i % n_workers
valid_X = dd.from_array(valid_X_, chunksize=n_samples)
valid_y = dd.from_array(valid_y_, chunksize=n_samples)

Expand All @@ -594,9 +599,9 @@ def run_empty_dmatrix_auc(client: "Client", tree_method: str, n_workers: int) ->


def test_empty_dmatrix_auc() -> None:
with LocalCluster(n_workers=2) as cluster:
with LocalCluster(n_workers=8) as cluster:
with Client(cluster) as client:
run_empty_dmatrix_auc(client, "hist", 2)
run_empty_dmatrix_auc(client, "hist", 8)


def run_auc(client: "Client", tree_method: str) -> None:
Expand Down