Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GPU Custom objective support #2690

Closed
wants to merge 21 commits into from
Closed

GPU Custom objective support #2690

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
aeabeb7
added custom tests
Dec 23, 2023
f323893
User metrics can be computed even when task_type=GPU
Dec 22, 2023
becd6a1
Fixed PR issues
Mar 4, 2024
0c9bbbc
removed custom tests from repo
Mar 4, 2024
5af47cd
added class for custom gpu metrics
Mar 22, 2024
c37a326
tmp commit
Apr 6, 2024
19b054a
working
Apr 16, 2024
e181d04
added custom eval metric on gpu support
Apr 16, 2024
d5ebeb7
Fixed PR issues
Apr 30, 2024
70b1f7c
Fixed PR issued #2
May 5, 2024
be6477d
fixed build errors
May 19, 2024
421cd51
Fixed more build errors
May 19, 2024
170c433
More resolved PR issues
May 25, 2024
dcf894b
Fixed style issues
May 29, 2024
041dab9
Minor fixes
May 29, 2024
f57d727
Added unit test
Jun 3, 2024
dbc32a3
Now it builds
Jun 2, 2024
4825c4c
It works, kind of
Jun 2, 2024
61a0bfd
Fixed whitespace issues (hopefully)
Jun 4, 2024
503bae8
Fixed PR issues
Jun 6, 2024
94c85ea
Added dynamic boosting
Jun 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions catboost/cuda/methods/doc_parallel_boosting.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ namespace NCatboostCuda {
const NCatboostOptions::TBoostingOptions& Config;
const NCatboostOptions::TModelBasedEvalOptions& ModelBasedEvalConfig;
const NCatboostOptions::TLossDescription& TargetOptions;
const TMaybe<TCustomObjectiveDescriptor>& ObjectiveDescriptor;

NPar::ILocalExecutor* LocalExecutor;

Expand Down Expand Up @@ -289,9 +290,11 @@ namespace NCatboostCuda {
}

THolder<TObjective> CreateTarget(const TDocParallelDataSet& dataSet) const {
return MakeHolder<TObjective>(dataSet,
Random,
TargetOptions);
return MakeHolder<TObjective>(
dataSet,
Random,
TargetOptions,
ObjectiveDescriptor);
}

//TODO(noxoomo): remove overhead of multiple target for permutation datasets
Expand Down Expand Up @@ -441,6 +444,7 @@ namespace NCatboostCuda {
public:
TBoosting(TBinarizedFeaturesManager& binarizedFeaturesManager,
const NCatboostOptions::TCatBoostOptions& catBoostOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor,
EGpuCatFeaturesStorage,
TGpuAwareRandom& random,
NPar::ILocalExecutor* localExecutor)
Expand All @@ -451,6 +455,7 @@ namespace NCatboostCuda {
, Config(catBoostOptions.BoostingOptions)
, ModelBasedEvalConfig(catBoostOptions.ModelBasedEvalOptions)
, TargetOptions(catBoostOptions.LossFunctionDescription)
, ObjectiveDescriptor(objectiveDescriptor)
, LocalExecutor(localExecutor)
{
}
Expand Down
6 changes: 5 additions & 1 deletion catboost/cuda/methods/dynamic_boosting.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ namespace NCatboostCuda {
const NCatboostOptions::TCatBoostOptions& CatBoostOptions;
const NCatboostOptions::TBoostingOptions& Config;
const NCatboostOptions::TLossDescription& TargetOptions;
const TMaybe<TCustomObjectiveDescriptor>& ObjectiveDescriptor;

NPar::ILocalExecutor* LocalExecutor;

Expand Down Expand Up @@ -169,7 +170,8 @@ namespace NCatboostCuda {
return MakeHolder<TObjective>(dataSet,
Random,
slice,
TargetOptions);
TargetOptions,
ObjectiveDescriptor);
}

inline ui32 MinEstimationSize(ui32 docCount) const {
Expand Down Expand Up @@ -502,6 +504,7 @@ namespace NCatboostCuda {
public:
TDynamicBoosting(TBinarizedFeaturesManager& binarizedFeaturesManager,
const NCatboostOptions::TCatBoostOptions& catBoostOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor,
EGpuCatFeaturesStorage catFeaturesStorage,
TGpuAwareRandom& random,
NPar::ILocalExecutor* localExecutor)
Expand All @@ -511,6 +514,7 @@ namespace NCatboostCuda {
, CatBoostOptions(catBoostOptions)
, Config(catBoostOptions.BoostingOptions)
, TargetOptions(catBoostOptions.LossFunctionDescription)
, ObjectiveDescriptor(objectiveDescriptor)
, LocalExecutor(localExecutor)
{
}
Expand Down
8 changes: 6 additions & 2 deletions catboost/cuda/targets/combination_targets_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@ namespace NCatboostCuda {
const TDataSet& dataSet,
TGpuAwareRandom& random,
TSlice slice,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor = Nothing())
: TParent(dataSet, random, slice)
{
CB_ENSURE(!objectiveDescriptor.Defined());
CreateLosses(targetOptions);

const auto createQuerywiseTarget = [&] (const auto& weightAndLoss) {
Expand All @@ -63,9 +65,11 @@ namespace NCatboostCuda {
TCombinationTargetsImpl(
const TDataSet& dataSet,
TGpuAwareRandom& random,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor = Nothing())
: TParent(dataSet, random)
{
CB_ENSURE(!objectiveDescriptor.Defined());
CreateLosses(targetOptions);

const auto createQuerywiseTarget = [&] (const auto& weightAndLoss) {
Expand Down
8 changes: 5 additions & 3 deletions catboost/cuda/targets/gpu_metrics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -712,13 +712,15 @@ namespace NCatboostCuda {
const bool haveEvalMetricFromUser = metricOptions->EvalMetric.IsSet();
const NCatboostOptions::TLossDescription& evalMetricDescription =
haveEvalMetricFromUser ? metricOptions->EvalMetric.Get() : objectiveMetricDescription;
CB_ENSURE(!IsUserDefined(objectiveMetricDescription.GetLossFunction()), "Error: GPU doesn't support user-defined loss");


TVector<THolder<IGpuMetric>> createdObjectiveMetrics = CreateGpuMetricFromDescription(
TVector<THolder<IGpuMetric>> createdObjectiveMetrics;
if (!IsUserDefined(objectiveMetricDescription.GetLossFunction())){
createdObjectiveMetrics = CreateGpuMetricFromDescription(
objectiveMetricDescription.GetLossFunction(),
objectiveMetricDescription,
cpuApproxDim);
}

if (hasWeights) {
for (auto&& metric : createdObjectiveMetrics) {
const auto& useWeights = metric->GetUseWeights();
Expand Down
20 changes: 20 additions & 0 deletions catboost/cuda/targets/gpu_metrics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,3 +167,23 @@ namespace NCatboostCuda {
const TMaybe<TCustomMetricDescriptor>& evalMetricDescriptor
);
}


class TCustomGpuMetricDescriptor {

using TEvalFuncPtr = TMetricHolder (*)(
TConstArrayRef<TConstArrayRef<double>>& approx,
TConstArrayRef<float> target,
TConstArrayRef<float> weight,
int begin,
int end,
void* customData);

using TGetFinalErrorFuncPtr = double (*)(const TMetricHolder& error, void* customData);

void* CustomData = nullptr;
TMaybe<TEvalFuncPtr> EvalFunc;

TIsMaxOptimalFuncPtr IsMaxOptimalFunc = nullptr;
TGetFinalErrorFuncPtr GetFinalErrorFunc = nullptr;
};
14 changes: 14 additions & 0 deletions catboost/cuda/targets/kernel.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <catboost/cuda/targets/kernel/yeti_rank_pointwise.cuh>
#include <catboost/cuda/targets/kernel/pfound_f.cuh>
#include <catboost/cuda/gpu_data/kernel/query_helper.cuh>
#include <catboost/cuda/targets/user_defined.h>

#include <util/generic/cast.h>

Expand Down Expand Up @@ -849,6 +850,19 @@ inline void ApproximatePointwise(const TCudaBuffer<const float, TMapping>& targe
LaunchKernels<TKernel>(target.NonEmptyDevices(), stream, target, weights, point, alpha, lossFunction, score, weightedDer, weightedDer2);
}

template <class TMapping>
inline void ApproximateUserDefined(const TCudaBuffer<const float, TMapping>& target,
const TCudaBuffer<const float, TMapping>& weights,
const TCudaBuffer<const float, TMapping>& point,
const TCustomObjectiveDescriptor& objectiveDescriptor,
TCudaBuffer<float, TMapping>* value,
TCudaBuffer<float, TMapping>* weightedDer,
TCudaBuffer<float, TMapping>* weightedDer2,
ui32 stream = 0) {
using TKernel = NKernelHost::TUserDefinedObjectiveKernel;
LaunchKernels<TKernel>(target.NonEmptyDevices(), stream, objectiveDescriptor, target, weights, point, value, weightedDer, weightedDer2);
}

template <class TMapping>
inline void ApproximateCrossEntropy(const TCudaBuffer<const float, TMapping>& target,
const TCudaBuffer<const float, TMapping>& weights,
Expand Down
5 changes: 4 additions & 1 deletion catboost/cuda/targets/multiclass_targets.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include <catboost/libs/logging/logging.h>
#include <catboost/private/libs/options/enums.h>
#include <catboost/private/libs/options/loss_description.h>
#include <catboost/private/libs/algo_helpers/custom_objective_descriptor.h>

#include <util/generic/algorithm.h>
#include <util/generic/mapfindptr.h>
Expand All @@ -32,9 +33,11 @@ namespace NCatboostCuda {
template <class TDataSet>
TMultiClassificationTargets(const TDataSet& dataSet,
TGpuAwareRandom& random,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor)
: TParent(dataSet,
random) {
CB_ENSURE(!objectiveDescriptor.Defined());
Init(targetOptions, dataSet.GetDataProvider());
}

Expand Down
5 changes: 4 additions & 1 deletion catboost/cuda/targets/pair_logit_pairwise.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <catboost/private/libs/options/bootstrap_options.h>
#include <catboost/private/libs/options/enums.h>
#include <catboost/private/libs/options/loss_description.h>
#include <catboost/private/libs/algo_helpers/custom_objective_descriptor.h>

#include <util/generic/map.h>
#include <util/generic/string.h>
Expand All @@ -32,9 +33,11 @@ namespace NCatboostCuda {
template <class TDataSet>
TPairLogitPairwise(const TDataSet& dataSet,
TGpuAwareRandom& random,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor)
: TParent(dataSet,
random) {
CB_ENSURE(!objectiveDescriptor.Defined());
Init(targetOptions);
}

Expand Down
5 changes: 4 additions & 1 deletion catboost/cuda/targets/pfound_f.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <catboost/private/libs/options/bootstrap_options.h>
#include <catboost/private/libs/options/enums.h>
#include <catboost/private/libs/options/loss_description.h>
#include <catboost/private/libs/algo_helpers/custom_objective_descriptor.h>

#include <util/generic/ptr.h>
#include <util/generic/utility.h>
Expand All @@ -34,9 +35,11 @@ namespace NCatboostCuda {
template <class TDataSet>
TPFoundF(const TDataSet& dataSet,
TGpuAwareRandom& random,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor)
: TParent(dataSet,
random) {
CB_ENSURE(!objectiveDescriptor.Defined());
Init(targetOptions);
}

Expand Down
35 changes: 31 additions & 4 deletions catboost/cuda/targets/pointwise_target_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <catboost/cuda/cuda_util/algorithm.h>
#include <catboost/cuda/gpu_data/feature_parallel_dataset.h>
#include <catboost/cuda/gpu_data/doc_parallel_dataset.h>
#include <catboost/private/libs/algo_helpers/custom_objective_descriptor.h>

namespace NCatboostCuda {
template <class TDocLayout>
Expand All @@ -23,19 +24,24 @@ namespace NCatboostCuda {
TPointwiseTargetsImpl(const TDataSet& dataSet,
TGpuAwareRandom& random,
TSlice slice,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor = Nothing())
: TParent(dataSet,
random,
slice) {
slice)
, ObjectiveDescriptor(objectiveDescriptor) {
Init(targetOptions);
}

template <class TDataSet>
TPointwiseTargetsImpl(const TDataSet& dataSet,
TGpuAwareRandom& random,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor = Nothing())
: TParent(dataSet,
random) {
random)
, ObjectiveDescriptor(objectiveDescriptor)
{
Init(targetOptions);
}

Expand All @@ -47,6 +53,7 @@ namespace NCatboostCuda {
, Alpha(target.GetAlpha())
, Border(target.GetBorder())
, MetricName(target.ScoreMetricName())
, ObjectiveDescriptor(target.GetObjectiveDescriptor())
{
}

Expand All @@ -56,6 +63,7 @@ namespace NCatboostCuda {
, Alpha(target.GetAlpha())
, Border(target.GetBorder())
, MetricName(target.ScoreMetricName())
, ObjectiveDescriptor(target.GetObjectiveDescriptor())
{
}

Expand All @@ -68,6 +76,7 @@ namespace NCatboostCuda {
, Alpha(basedOn.GetAlpha())
, Border(basedOn.GetBorder())
, MetricName(basedOn.ScoreMetricName())
, ObjectiveDescriptor(basedOn.GetObjectiveDescriptor())
{
}

Expand All @@ -77,6 +86,7 @@ namespace NCatboostCuda {
, Alpha(other.GetAlpha())
, Border(other.GetBorder())
, MetricName(other.ScoreMetricName())
, ObjectiveDescriptor(other.GetObjectiveDescriptor())
{
}

Expand Down Expand Up @@ -240,10 +250,15 @@ namespace NCatboostCuda {
return 1;
}

TMaybe<TCustomObjectiveDescriptor> GetObjectiveDescriptor() const {
return ObjectiveDescriptor;
}

private:
void Init(const NCatboostOptions::TLossDescription& targetOptions) {
Type = targetOptions.GetLossFunction();
switch (targetOptions.GetLossFunction()) {
case ELossFunction::PythonUserDefinedPerObject:
case ELossFunction::Poisson:
case ELossFunction::MAPE:
case ELossFunction::RMSE:
Expand Down Expand Up @@ -304,6 +319,17 @@ namespace NCatboostCuda {
CB_ENSURE(der2Row == 0);

switch (Type) {
case ELossFunction::PythonUserDefinedPerObject: {
ApproximateUserDefined(target,
weights,
point,
*ObjectiveDescriptor,
value,
der,
der2,
stream);
break;
}
case ELossFunction::CrossEntropy:
case ELossFunction::Logloss: {
ApproximateCrossEntropy(target,
Expand Down Expand Up @@ -338,6 +364,7 @@ namespace NCatboostCuda {
double Border = 0;
double VariancePower = 1.5;
TString MetricName;
TMaybe<TCustomObjectiveDescriptor> ObjectiveDescriptor = {};
};

}
5 changes: 4 additions & 1 deletion catboost/cuda/targets/query_cross_entropy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <catboost/private/libs/options/bootstrap_options.h>
#include <catboost/private/libs/options/enums.h>
#include <catboost/private/libs/options/loss_description.h>
#include <catboost/private/libs/algo_helpers/custom_objective_descriptor.h>

#include <util/generic/map.h>
#include <util/generic/ptr.h>
Expand All @@ -35,9 +36,11 @@ namespace NCatboostCuda {
template <class TDataSet>
TQueryCrossEntropy(const TDataSet& dataSet,
TGpuAwareRandom& random,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor)
: TParent(dataSet,
random) {
CB_ENSURE(!objectiveDescriptor.Defined());
Init(targetOptions);
}

Expand Down
11 changes: 8 additions & 3 deletions catboost/cuda/targets/querywise_targets_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,24 @@ namespace NCatboostCuda {
TQuerywiseTargetsImpl(const TDataSet& dataSet,
TGpuAwareRandom& random,
TSlice slice,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor = Nothing())
: TParent(dataSet,
random,
slice) {
CB_ENSURE(!objectiveDescriptor.Defined(), "Querywise target is not supported for custom loss");
Init(targetOptions);
}

template <class TDataSet>
TQuerywiseTargetsImpl(const TDataSet& dataSet,
TGpuAwareRandom& random,
const NCatboostOptions::TLossDescription& targetOptions)
const NCatboostOptions::TLossDescription& targetOptions,
const TMaybe<TCustomObjectiveDescriptor>& objectiveDescriptor = Nothing())
: TParent(dataSet,
random) {
random)
{
CB_ENSURE(!objectiveDescriptor.Defined(), "Querywise target is not supported for custom loss");
Init(targetOptions);
}

Expand Down
Loading