Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[TensorRT EP] Update TRT10.0 deprecated api #20989

Merged
merged 14 commits into from
Jul 2, 2024
Merged

[TensorRT EP] Update TRT10.0 deprecated api #20989

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
67254e4
update
yf711 Jun 8, 2024
3d5477e
fix
yf711 Jun 10, 2024
b19283d
fix
yf711 Jun 10, 2024
275b02f
lint
yf711 Jun 10, 2024
abd9796
fix
yf711 Jun 11, 2024
b56482d
Merge main into yifanl/trt_api
yf711 Jun 11, 2024
facd228
remove 4996 in nv_includes.h & test_main.cc
yf711 Jun 12, 2024
663c8b7
Merge branch 'main' into yifanl/trt_api
yf711 Jun 13, 2024
4e99543
Revert "remove 4996 in nv_includes.h & test_main.cc"
yf711 Jun 17, 2024
3755eb8
fix
yf711 Jun 17, 2024
dbcc3c7
move this change to follow-up PR, more work needed to fix unit test f…
yf711 Jun 20, 2024
f2c2915
comment applied
yf711 Jun 28, 2024
db65a7d
test subset
yf711 Jun 29, 2024
774d83b
revert
yf711 Jun 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,11 +160,20 @@
nvinfer1::TacticSource source{};
t = toUpper(t);
if (t == "CUBLAS") {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] Tactic kCUBLAS is deprecated in TensorRT 10.0";
#if NV_TENSORRT_MAJOR < 10
source = nvinfer1::TacticSource::kCUBLAS;
#endif
} else if (t == "CUBLASLT" || t == "CUBLAS_LT") {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] Tactic kCUBLAS_LT is deprecated in TensorRT 9.0";
#if NV_TENSORRT_MAJOR < 9
source = nvinfer1::TacticSource::kCUBLAS_LT;
#endif
} else if (t == "CUDNN") {
LOGS_DEFAULT(WARNING) << "[TensorRT EP] Tactic kCUDNN is deprecated in TensorRT 10.0";
#if NV_TENSORRT_MAJOR < 10
source = nvinfer1::TacticSource::kCUDNN;
#endif
} else if (t == "EDGE_MASK_CONVOLUTIONS") {
source = nvinfer1::TacticSource::kEDGE_MASK_CONVOLUTIONS;
} else if (t == "JIT_CONVOLUTIONS") {
Expand Down Expand Up @@ -289,10 +298,30 @@
return g_host->CudaCall_true(retCode, exprString, libName, successCode, msg, file, line);
}

#if NV_TENSORRT_MAJOR > 8
yf711 marked this conversation as resolved.
Show resolved Hide resolved
void* OutputAllocator::reallocateOutputAsync(char const* /*tensorName*/, void* /*currentMemory*/, uint64_t size,
uint64_t /*alignment*/, cudaStream_t /*stream*/) noexcept {
// Some memory allocators return nullptr when allocating zero bytes, but TensorRT requires a non-null ptr
// even for empty tensors, so allocate a dummy byte.
size = std::max(size, static_cast<uint64_t>(1));
if (size > allocated_size) {
cudaFree(outputPtr);
outputPtr = nullptr;
allocated_size = 0;
if (cudaMalloc(&outputPtr, size) == cudaSuccess) {
allocated_size = size;
}
}
// if cudaMalloc fails, returns nullptr.
return outputPtr;
}
#else
// Only override this method when TensorRT <= 8.6
void* OutputAllocator::reallocateOutput(char const* /*tensorName*/, void* /*currentMemory*/, uint64_t size,
uint64_t /*alignment*/) noexcept {
// Some memory allocators return nullptr when allocating zero bytes, but TensorRT requires a non-null ptr
// even for empty tensors, so allocate a dummy byte.
LOGS_DEFAULT(WARNING) << "[TensorRT EP] reallocateOutput() is deprecated after TensorRT 8.6";
yf711 marked this conversation as resolved.
Show resolved Hide resolved
size = std::max(size, static_cast<uint64_t>(1));
if (size > allocated_size) {
cudaFree(outputPtr);
Expand All @@ -305,6 +334,7 @@
// if cudaMalloc fails, returns nullptr.
return outputPtr;
}
#endif

void OutputAllocator::notifyShape(char const* /*tensorName*/, nvinfer1::Dims const& dims) noexcept {
output_shapes.clear();
Expand Down Expand Up @@ -3142,7 +3172,11 @@
if (mem_size > max_ctx_mem_size_) {
max_ctx_mem_size_ = mem_size;
}
#if NV_TENSORRT_MAJOR < 10
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContextWithoutDeviceMemory());
#else
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext(nvinfer1::ExecutionContextAllocationStrategy::kUSER_MANAGED));

Check warning on line 3178 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

View workflow job for this annotation

GitHub Actions / Lint C++ 

[cpplint] reported by reviewdog 🐶
 Lines should be <= 120 characters long  [whitespace/line_length] [2]

Raw Output:
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:3178:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
#endif
} else {
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
}
Expand Down Expand Up @@ -3588,8 +3622,13 @@

if (context_update) {
if (trt_state->context_memory_sharing_enable) {
#if NV_TENSORRT_MAJOR < 10
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContextWithoutDeviceMemory());
#else
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContext(nvinfer1::ExecutionContextAllocationStrategy::kUSER_MANAGED));

Check warning on line 3630 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

View workflow job for this annotation

GitHub Actions / Lint C++ 

[cpplint] reported by reviewdog 🐶
 Lines should be <= 120 characters long  [whitespace/line_length] [2]

Raw Output:
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:3630:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
#endif
} else {
*(trt_state->context) = std::unique_ptr<nvinfer1::IExecutionContext>(
trt_state->engine->get()->createExecutionContext());
Expand Down Expand Up @@ -3805,7 +3844,11 @@
if (mem_size > max_ctx_mem_size_) {
max_ctx_mem_size_ = mem_size;
}
#if NV_TENSORRT_MAJOR < 10
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContextWithoutDeviceMemory());
#else
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext(nvinfer1::ExecutionContextAllocationStrategy::kUSER_MANAGED));

Check warning on line 3850 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc

View workflow job for this annotation

GitHub Actions / Lint C++ 

[cpplint] reported by reviewdog 🐶
 Lines should be <= 120 characters long  [whitespace/line_length] [2]

Raw Output:
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:3850:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
#endif
} else {
trt_context = std::unique_ptr<nvinfer1::IExecutionContext>(trt_engine->createExecutionContext());
}
Expand Down
5 changes: 4 additions & 1 deletion onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,11 @@
//
class OutputAllocator : public nvinfer1::IOutputAllocator {
public:
#if NV_TENSORRT_MAJOR >= 10
void* reallocateOutputAsync(char const* tensorName, void* currentMemory, uint64_t size, uint64_t alignment, cudaStream_t stream) noexcept override;

Check warning on line 120 in onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h

View workflow job for this annotation

GitHub Actions / Lint C++ 

[cpplint] reported by reviewdog 🐶
 Lines should be <= 120 characters long  [whitespace/line_length] [2]

Raw Output:
onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h:120:  Lines should be <= 120 characters long  [whitespace/line_length] [2]
#else
void* reallocateOutput(char const* tensorName, void* currentMemory, uint64_t size, uint64_t alignment) noexcept override;

#endif
void notifyShape(char const* tensorName, nvinfer1::Dims const& dims) noexcept override;

void* getBuffer() {
Expand Down
10 changes: 10 additions & 0 deletions onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_custom_ops.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,22 @@ common::Status CreateTensorRTCustomOpDomainList(std::vector<OrtCustomOpDomain*>&
initLibNvInferPlugins(&trt_logger, "");

int num_plugin_creator = 0;
#if NV_TENSORRT_MAJOR >= 10
auto plugin_creators = getPluginRegistry()->getAllCreators(&num_plugin_creator);
#else
auto plugin_creators = getPluginRegistry()->getPluginCreatorList(&num_plugin_creator);
#endif
std::unordered_set<std::string> registered_plugin_names;

for (int i = 0; i < num_plugin_creator; i++) {
#if NV_TENSORRT_MAJOR >= 10
auto plugin_creator_interface = plugin_creators[i];
auto plugin_creator = static_cast<nvinfer1::IPluginCreator*>(plugin_creator_interface);
yf711 marked this conversation as resolved.
Show resolved Hide resolved
std::string plugin_name(plugin_creator->getPluginName());
#else
auto plugin_creator = plugin_creators[i];
std::string plugin_name(plugin_creator->getPluginName());
#endif
LOGS_DEFAULT(VERBOSE) << "[TensorRT EP] " << plugin_name << ", version : " << plugin_creator->getPluginVersion();

// plugin has different versions and we only register once
Expand Down
Loading