feat: Python Deployment of Triton Inference Server

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -105,8 +105,18 @@ add_executable(
   common.h
   shared_memory_manager.h
   triton_signal.h
+  server_interface.h
 )
 
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+add_library(server-interface-library server_interface.h)
+set_target_properties(server-interface-library 
+                      PROPERTIES 
+                      POSITION_INDEPENDENT_CODE ON
+                      LINKER_LANGUAGE CXX)
+target_compile_features(server-interface-library PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD})
+target_compile_definitions(server-interface-library PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY})
+
 # On windows a *.lib file can be generated for a exe. When creating
 # tritonserver.exe if we try to create tritonserver.lib it will fail
 # because there is already a trtionserver.lib for tritonserver.dll,
@@ -352,7 +362,7 @@ if(${TRITON_ENABLE_HTTP}
     http-endpoint-library EXCLUDE_FROM_ALL
     ${HTTP_ENDPOINT_SRCS} ${HTTP_ENDPOINT_HDRS}
   )
-
+  
   target_compile_features(http-endpoint-library PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD})
   if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
     target_compile_options(
@@ -367,7 +377,7 @@ if(${TRITON_ENABLE_HTTP}
         -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Werror
     )
   endif()
-
+ 
   set_target_properties(
     http-endpoint-library
     PROPERTIES
@@ -777,3 +787,47 @@ endif() # NOT WIN32
 if ( NOT WIN32)
   add_subdirectory(test test)
 endif() # NOT WIN32
+
+find_package(CUDAToolkit REQUIRED)
+
+# TODO: Remove excess stuff
+add_library(
+  python-binding-dependency 
+  EXCLUDE_FROM_ALL
+  shared_memory_manager.h
+  shared_memory_manager.cc
+  data_compressor.h
+  common.h
+  common.cc
+  restricted_features.h
+  tracer.h
+  classification.cc
+)
+
+target_link_libraries(
+  python-binding-dependency
+  PUBLIC
+  http-endpoint-library 
+  grpc-endpoint-library
+  CUDA::cudart
+  triton-core-serverapi           # from repo-core
+  triton-core-serverstub          # from repo-core
+  triton-common-json              # from repo-common
+)
+
+
+set_target_properties(python-binding-dependency PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+target_compile_features(python-binding-dependency PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD})
+
+target_compile_definitions(
+    python-binding-dependency
+    PRIVATE TRITON_ENABLE_GPU=1
+    PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY}
+  )
+
+# Add the -fPIC flag
+# set_target_properties(server-src-lib PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+
+add_subdirectory(python python)

src/grpc/grpc_server.cc

@@ -2435,6 +2435,57 @@ Server::Create(
   return nullptr;  // success
 }
 
+bool
+Server::Create_Wrapper(
+      std::shared_ptr<TRITONSERVER_Server>& server, 
+      UnorderedMapType& data, 
+      std::unique_ptr<triton::server::grpc::Server>* service,
+      const RestrictedFeatures& restricted_features) {
+
+    std::string address = get_value<std::string>(data, "address");
+    int port = get_value<int>(data, "port");
+    bool reuse_port = get_value<int>(data, "reuse_port");
+
+    SocketOptions socket_selection{address, port, reuse_port};
+
+    bool use_ssl = get_value<int>(data, "use_ssl");
+    std::string server_cert = get_value<std::string>(data, "server_cert");
+    std::string server_key = get_value<std::string>(data, "server_key");
+    std::string root_cert = get_value<std::string>(data, "root_cert");
+    bool use_mutual_auth = get_value<int>(data, "use_mutual_auth");
+
+    SslOptions ssl_selection{use_ssl, server_cert, server_key, root_cert, use_mutual_auth};
+
+    int keepalive_time_ms = get_value<int>(data, "keepalive_time_ms");
+    int keepalive_timeout_ms = get_value<int>(data, "keepalive_timeout_ms");
+    bool keepalive_permit_without_calls = get_value<int>(data, "keepalive_permit_without_calls");
+    int http2_max_pings_without_data = get_value<int>(data, "http2_max_pings_without_data");
+    int http2_min_recv_ping_interval_without_data_ms = get_value<int>(data, "http2_min_recv_ping_interval_without_data_ms");
+    int http2_max_ping_strikes = get_value<int>(data, "http2_max_ping_strikes");
+    int max_connection_age_ms = get_value<int>(data, "max_connection_age_ms");
+    int max_connection_age_grace_ms = get_value<int>(data, "max_connection_age_grace_ms");
+
+    KeepAliveOptions keep_alive_selection{keepalive_time_ms, keepalive_timeout_ms,
+                     keepalive_permit_without_calls, http2_max_pings_without_data,
+                     http2_min_recv_ping_interval_without_data_ms, http2_max_ping_strikes,
+                     max_connection_age_ms, max_connection_age_grace_ms};
+
+    grpc_compression_level infer_compression_level = static_cast<grpc_compression_level>(get_value<int>(data, "infer_compression_level"));
+    int infer_allocation_pool_size = get_value<int>(data, "infer_allocation_pool_size");
+    std::string forward_header_pattern = get_value<std::string>(data, "forward_header_pattern");
+
+
+    Options grpc_options{socket_selection, ssl_selection, keep_alive_selection,
+                        infer_compression_level, infer_allocation_pool_size, 
+                        restricted_features, forward_header_pattern};
+
+    TRITONSERVER_Error* err = Create(server, nullptr, nullptr, 
+                                      grpc_options, service);
+    if(err == nullptr) {return true;}
+
+    return false;
+}   
+
 TRITONSERVER_Error*
 Server::Start()
 {

src/grpc/grpc_server.h

@@ -39,6 +39,7 @@
 #include "infer_handler.h"
 #include "stream_infer_handler.h"
 #include "triton/core/tritonserver.h"
+// #include "../server_interface.h"
 
 namespace triton { namespace server { namespace grpc {
 
@@ -92,13 +93,32 @@ struct Options {
   std::string forward_header_pattern_;
 };
 
+using VariantType = std::variant<int, bool, std::string>;
+using UnorderedMapType = std::unordered_map<std::string, VariantType>;
+
+template <typename T>
+T get_value(const UnorderedMapType& options, const std::string& key) {
+  auto curr = options.find(key);
+  bool is_present = (curr != options.end());
+  bool correct_type = std::holds_alternative<T>(curr->second);
+
+  return std::get<T>(curr->second); 
+}
+
 class Server {
  public:
   static TRITONSERVER_Error* Create(
       const std::shared_ptr<TRITONSERVER_Server>& tritonserver,
       triton::server::TraceManager* trace_manager,
       const std::shared_ptr<SharedMemoryManager>& shm_manager,
       const Options& server_options, std::unique_ptr<Server>* server);
+
+  static bool Create_Wrapper(
+      std::shared_ptr<TRITONSERVER_Server>& server, 
+      UnorderedMapType& data, 
+      std::unique_ptr<triton::server::grpc::Server>* service,
+      const RestrictedFeatures& restricted_features); 
+
 
   ~Server();
 

src/http_server.cc

@@ -46,6 +46,11 @@
 #define TRITONJSON_STATUSSUCCESS nullptr
 #include "triton/common/triton_json.h"
 
+#include <unistd.h> // For sleep 
+
+// #define LOG_VERBOSE_IS_ON ... something like that
+// TODO: hard code log verbose mode to see output in Handle()
+
 extern "C" {
 #include <b64/cdecode.h>
 }
@@ -228,6 +233,9 @@ HTTPServer::Start()
     event_add(break_ev_, NULL);
     worker_ = std::thread(event_base_loop, evbase_, 0);
 
+    const std::string addr = address_ + ":" + std::to_string(port_);
+    LOG_INFO << "Started HTTPService at " << addr;
+
     return nullptr;
   }
 
@@ -1181,6 +1189,7 @@ HTTPAPIServer::HTTPAPIServer(
 
 HTTPAPIServer::~HTTPAPIServer()
 {
+  LOG_VERBOSE(2) << "~HTTPAPIServer()";
   if (server_metadata_err_ != nullptr) {
     TRITONSERVER_ErrorDelete(server_metadata_err_);
   }
@@ -4688,12 +4697,37 @@ HTTPAPIServer::Create(
       server, trace_manager, shm_manager, port, reuse_port, address,
       header_forward_pattern, thread_cnt, restricted_features));
 
-  const std::string addr = address + ":" + std::to_string(port);
-  LOG_INFO << "Started HTTPService at " << addr;
 
   return nullptr;
 }
 
+
+bool
+HTTPAPIServer::Create_Wrapper(
+      std::shared_ptr<TRITONSERVER_Server>& server, 
+      UnorderedMapType& data, 
+      std::unique_ptr<HTTPServer>* service,
+      const RestrictedFeatures& restricted_features) 
+{ 
+    int port = get_value<int>(data, "port");
+    bool reuse_port = get_value<int>(data, "reuse_port");
+    std::string address = get_value<std::string>(data, "address"); 
+    std::string header_forward_pattern = get_value<std::string>(data, "header_forward_pattern");
+    int thread_count = get_value<int>(data, "thread_count");
+
+    TRITONSERVER_Error* err = Create(server, // shared_ptr<TRITONSERVER_Server>
+                                     nullptr, nullptr, // TraceManager, SharedMemoryManager 
+                                     port, reuse_port, address, // port, reuse_port, address
+                                     header_forward_pattern,  thread_count, // header_forward_pattern, thread_count 
+                                     restricted_features, // RestrictedFeatures
+                                     service); // HTTPServer instance
+
+    if(err == nullptr) return true;
+
+    return false;
+}
+
+
 bool
 HTTPAPIServer::RespondIfRestricted(
     evhtp_request_t* req, const Restriction& restriction)
@@ -4712,4 +4746,5 @@ HTTPAPIServer::RespondIfRestricted(
   return false;
 }
 
+
 }}  // namespace triton::server

src/http_server.h

@@ -35,7 +35,6 @@
 #include <queue>
 #include <string>
 #include <thread>
-#include <unordered_map>
 
 #include "common.h"
 #include "data_compressor.h"
@@ -183,6 +182,18 @@ using HttpTextMapCarrier = void*;
 #endif
 
 
+using VariantType = std::variant<int, bool, std::string>;
+using UnorderedMapType = std::unordered_map<std::string, VariantType>;
+
+template <typename T>
+T get_value(const UnorderedMapType& options, const std::string& key) {
+  auto curr = options.find(key);
+  bool is_present = (curr != options.end());
+  bool correct_type = std::holds_alternative<T>(curr->second);
+
+  return std::get<T>(curr->second); 
+}
+
 // HTTP API server that implements KFServing community standard inference
 // protocols and extensions used by Triton.
 class HTTPAPIServer : public HTTPServer {
@@ -196,6 +207,13 @@ class HTTPAPIServer : public HTTPServer {
       const RestrictedFeatures& restricted_apis,
       std::unique_ptr<HTTPServer>* http_server);
 
+
+static bool Create_Wrapper(
+      std::shared_ptr<TRITONSERVER_Server>& server, 
+      UnorderedMapType& data, 
+      std::unique_ptr<HTTPServer>* service,
+      const RestrictedFeatures& restricted_features); 
+
   virtual ~HTTPAPIServer();
 
   //
@@ -627,4 +645,4 @@ class HTTPAPIServer : public HTTPServer {
       evhtp_request_t* req, const Restriction& restriction);
 };
 
-}}  // namespace triton::server
+}}  // namespace triton::server

src/python/CMakeLists.txt

@@ -0,0 +1,78 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+cmake_minimum_required(VERSION 3.18)
+
+add_subdirectory(tritonfrontend)
+
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION ${TRITON_VERSION})
+# configure_file(../LICENSE LICENSE.txt COPYONLY)
+configure_file(setup.py setup.py @ONLY)
+file(COPY test/  DESTINATION ./test/.)
+
+set(WHEEL_DEPENDS
+      ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION
+      # ${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt
+      ${CMAKE_CURRENT_BINARY_DIR}/setup.py
+      ${CMAKE_CURRENT_BINARY_DIR}/tritonfrontend
+      py-bindings
+)
+
+set(wheel_stamp_file "stamp.whl")
+
+add_custom_command(
+  OUTPUT "${wheel_stamp_file}"
+  COMMAND python3
+  ARGS
+    "${CMAKE_CURRENT_SOURCE_DIR}/build_wheel.py"
+    --dest-dir "${CMAKE_CURRENT_BINARY_DIR}/generic"
+    --binding-path $<TARGET_FILE:py-bindings>
+  DEPENDS ${WHEEL_DEPENDS}
+)
+
+add_custom_target(
+  frontend-server-wheel ALL
+  DEPENDS
+    "${wheel_stamp_file}"
+)
+
+
+# Wheel
+set(WHEEL_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/generic/wheel/dist/")
+install(
+  DIRECTORY
+  ${WHEEL_OUT_DIR}
+  DESTINATION "${CMAKE_INSTALL_PREFIX}/python"
+)
+
+
+# Tests
+set(TEST_DIR "${CMAKE_CURRENT_BINARY_DIR}/test")
+install(
+  DIRECTORY
+  ${TEST_DIR}
+  DESTINATION "${CMAKE_INSTALL_PREFIX}/python"
+)