Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ConvTransposeGrad CUDA Kernel #17201

Merged
merged 6 commits into from
Aug 24, 2023
Merged

ConvTransposeGrad CUDA Kernel #17201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e158626
ConvTransposeGrad CUDA Kernel
baijumeswani Aug 17, 2023
7cd4bbc
Reorder includes
baijumeswani Aug 17, 2023
ba00b10
Address pipeline error
baijumeswani Aug 18, 2023
fba872e
Merge branch 'main' of https://github.com/microsoft/onnxruntime into …
baijumeswani Aug 18, 2023
9028a8c
Avoid hipifying conv_transpose_grad
baijumeswani Aug 18, 2023
61c9c07
Disable ortmodule conv transpose grad tests for rocm pipeline
baijumeswani Aug 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmake/onnxruntime_rocm_hipify.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,10 @@ set(training_ops_excluded_files
"reduction/reduction_ops.cc" # no double type support
"cuda_training_kernels.cc"
"cuda_training_kernels.h"
"nn/conv_shared.cc"
"nn/conv_shared.h"
"nn/conv_transpose_grad.cc"
"nn/conv_transpose_grad.h"
)

function(auto_set_source_files_hip_language)
Expand Down
17 changes: 17 additions & 0 deletions orttraining/orttraining/core/graph/gradient_builder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2070,5 +2070,22 @@
{GO(0), O(0)}, {GI(0)}, SrcNodeAttributes())};
}

IMPLEMENT_GRADIENT_BUILDER(GetConvTransposeGradient) {
std::vector<ArgDef> outputs;
for (int i = 0; i < GetSrcNodeInputSize(); i++) {
if (IsGradientRequiredForSrcNodeInput(i)) {
outputs.push_back(GI(i));
} else {
outputs.push_back(ArgDef("", nullptr));
}
}

return std::vector<NodeDef>{

Check warning on line 2083 in orttraining/orttraining/core/graph/gradient_builder.cc

View workflow job for this annotation

GitHub Actions / cpplint

[cpplint] orttraining/orttraining/core/graph/gradient_builder.cc#L2083 

 Add #include <vector> for vector<>  [build/include_what_you_use] [4]
Raw output 
orttraining/orttraining/core/graph/gradient_builder.cc:2083:  Add #include <vector> for vector<>  [build/include_what_you_use] [4]
NodeDef(OpDef{"ConvTransposeGrad", kMSDomain, 1},
{GO(0), I(0), I(1)},
outputs,
SrcNodeAttributes())};
}

} // namespace training
} // namespace onnxruntime
1 change: 1 addition & 0 deletions orttraining/orttraining/core/graph/gradient_builder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ DECLARE_GRADIENT_BUILDER(GetLSTMGradient)
DECLARE_GRADIENT_BUILDER(GetGRUGradient)
DECLARE_GRADIENT_BUILDER(GetReciprocalGradient)
DECLARE_GRADIENT_BUILDER(GetLeakyReluGradient)
DECLARE_GRADIENT_BUILDER(GetConvTransposeGradient)

DECLARE_GRADIENT_BUILDER(GetExternalGradient)

Expand Down
1 change: 1 addition & 0 deletions orttraining/orttraining/core/graph/gradient_builder_registry.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ void GradientBuilderRegistry::RegisterGradientBuilders() {
REGISTER_GRADIENT_BUILDER("GRUTraining", GetGRUGradient);
REGISTER_GRADIENT_BUILDER("Reciprocal", GetReciprocalGradient);
REGISTER_GRADIENT_BUILDER("LeakyRelu", GetLeakyReluGradient);
REGISTER_GRADIENT_BUILDER("ConvTranspose", GetConvTransposeGradient);

REGISTER_GRADIENT_BUILDER("ExternalGradient", GetExternalGradient);
};
Expand Down
15 changes: 15 additions & 0 deletions orttraining/orttraining/core/graph/training_op_defs.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4908,6 +4908,21 @@ Return true if all elements are true and false otherwise.
}
}
});

ONNX_CONTRIB_OPERATOR_SCHEMA(ConvTransposeGrad)
.SetDomain(kMSDomain)
.SinceVersion(1)
.Input(0, "dY", "Gradient of output Y", "T")
.Input(1, "X", "Input tensor", "T")
.Input(2, "W", "Weight tensor", "T")
.Output(0, "dX", "Gradient of X", "T", OpSchema::Optional)
.Output(1, "dW", "Gradient of W", "T", OpSchema::Optional)
.Output(2, "dB", "Gradient of B", "T", OpSchema::Optional)
.AllowUncheckedAttributes()
.TypeConstraint(
"T",
{"tensor(float16)", "tensor(float)", "tensor(double)"},
"Constrain input and output types to float tensors.");
}

} // namespace training
Expand Down
198 changes: 198 additions & 0 deletions orttraining/orttraining/test/gradient/gradient_ops_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3039,6 +3039,204 @@ TEST(GradientCheckerTest, LeakyReluGrad) {
UnaryOpGradientTest("LeakyRelu", kOnnxDomain, 16, nullptr, &transformer);
}

#ifdef USE_CUDA
void ConvTransposeGradientCheckerTest(std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers) {
float max_error;
GradientChecker<float, float, float> gradient_checker;
OpDef op_def{"ConvTranspose"};

float error_tolerance = 1e-1f;

// 1D convolution
{
TensorShape x_shape({2, 2, 5});
TensorShape w_shape({2, 2, 3});
TensorShape b_shape({2});
TensorShape y_shape({2, 2, 5});
ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(
op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3}), MakeAttribute("pads", std::vector<int64_t>{1, 1})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 1D strided convolution
{
TensorShape x_shape({2, 1, 7});
TensorShape w_shape({1, 1, 3});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 13});
ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(
op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3}), MakeAttribute("pads", std::vector<int64_t>{1, 1}),
MakeAttribute("strides", std::vector<int64_t>{2})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 1D pointwise convolution (with padding)
{
TensorShape x_shape({2, 1, 5});
TensorShape w_shape({1, 1, 1});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 3});
ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(
op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{1}), MakeAttribute("pads", std::vector<int64_t>{1, 1})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 1D pointwise convolution (no padding)
{
TensorShape x_shape({2, 1, 5});
TensorShape w_shape({1, 1, 1});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 5});
ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(
op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{1}), MakeAttribute("pads", std::vector<int64_t>{0, 0})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 2D convolution
{
TensorShape x_shape({1, 1, 3, 3});
TensorShape w_shape({1, 1, 3, 3});
TensorShape b_shape({1});
TensorShape y_shape({1, 1, 3, 3});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3, 3}),
MakeAttribute("pads", std::vector<int64_t>{1, 1, 1, 1})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 2D convolution
{
TensorShape x_shape({2, 1, 5, 5});
TensorShape w_shape({1, 1, 3, 3});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 5, 5});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3, 3}),
MakeAttribute("pads", std::vector<int64_t>{1, 1, 1, 1})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 2D pointwise convolution (with padding)
{
TensorShape x_shape({1, 1, 3, 3});
TensorShape w_shape({1, 1, 1, 1});
TensorShape b_shape({1});
TensorShape y_shape({1, 1, 1, 1});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{1, 1}),
MakeAttribute("pads", std::vector<int64_t>{1, 1, 1, 1})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 2D pointwise convolution (no padding)
{
TensorShape x_shape({1, 1, 3, 3});
TensorShape w_shape({1, 1, 1, 1});
TensorShape b_shape({1});
TensorShape y_shape({1, 1, 3, 3});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{1, 1}),
MakeAttribute("pads", std::vector<int64_t>{0, 0, 0, 0})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 2D strided convolution
{
TensorShape x_shape({2, 1, 7, 5});
TensorShape w_shape({1, 1, 3, 3});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 13, 9});
ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(
op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3, 3}),
MakeAttribute("pads", std::vector<int64_t>{1, 1, 1, 1}), MakeAttribute("strides", std::vector<int64_t>{2, 2})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 2D dilated convolution (no padding)
{
TensorShape x_shape({2, 1, 5, 5});
TensorShape w_shape({1, 1, 3, 3});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 9, 9});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3, 3}),
MakeAttribute("pads", std::vector<int64_t>{0, 0, 0, 0}),
MakeAttribute("dilations", std::vector<int64_t>{2, 2})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 2D dilated convolution (with padding)
{
TensorShape x_shape({2, 1, 7, 5});
TensorShape w_shape({1, 1, 3, 3});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 9, 7});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3, 3}),
MakeAttribute("pads", std::vector<int64_t>{1, 1, 1, 1}),
MakeAttribute("dilations", std::vector<int64_t>{2, 2})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 3D convolution
{
TensorShape x_shape({2, 1, 5, 5, 5});
TensorShape w_shape({1, 1, 3, 3, 3});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 5, 5, 5});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3, 3, 3}),
MakeAttribute("pads", std::vector<int64_t>{1, 1, 1, 1, 1, 1})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}

// 3D strided convolution
{
TensorShape x_shape({2, 1, 7, 5, 5});
TensorShape w_shape({1, 1, 3, 3, 3});
TensorShape b_shape({1});
TensorShape y_shape({2, 1, 13, 9, 9});
ASSERT_STATUS_OK(
gradient_checker.ComputeGradientError(op_def, {x_shape, w_shape, b_shape}, {y_shape}, &max_error,
{MakeAttribute("kernel_shape", std::vector<int64_t>{3, 3, 3}),
MakeAttribute("pads", std::vector<int64_t>{1, 1, 1, 1, 1, 1}),
MakeAttribute("strides", std::vector<int64_t>{2, 2, 2})},
false, false, execution_providers));
EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
}
}

TEST(GradientCheckerTest, ConvTransposeGrad) {
std::vector<std::unique_ptr<IExecutionProvider>> execution_providers;
execution_providers.push_back(DefaultCudaExecutionProvider());
ConvTransposeGradientCheckerTest(&execution_providers);
}
#endif // USE_CUDA

} // namespace test
} // namespace onnxruntime

Expand Down
Loading
Loading