Add test for PB DLPack tensor (triton-inference-server#3064)

* Clean up of older Python models and added DLPack identity model

* Add dlpack add_sub models for Python backend

* Add DLPack tests in L0_infer

* Add flush=True to the env test

* Increase the server timeout

* Add unittest for DLPack tensors

* Fix dlpack_add_sub models

* Add unittest for DLPack tesnsors

* Add ensemble io test

qa/L0_backend_python/io/io_test.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+
+sys.path.append("../../common")
+
+import test_util as tu
+import tritonclient.http as httpclient
+from tritonclient.utils import *
+import numpy as np
+import unittest
+
+
+class IOTest(tu.TestResultCollector):
+    def test_ensemble_io(self):
+        model_name = "ensemble_io"
+        with httpclient.InferenceServerClient("localhost:8000") as client:
+            input0 = np.random.random([1000]).astype(np.float32)
+            for model_1_in_gpu in [True, False]:
+                for model_2_in_gpu in [True, False]:
+                    for model_3_in_gpu in [True, False]:
+                        gpu_output = np.asarray(
+                            [model_1_in_gpu, model_2_in_gpu, model_3_in_gpu],
+                            dtype=bool)
+                        inputs = [
+                            httpclient.InferInput(
+                                "INPUT0", input0.shape,
+                                np_to_triton_dtype(input0.dtype)),
+                            httpclient.InferInput(
+                                "GPU_OUTPUT", gpu_output.shape,
+                                np_to_triton_dtype(gpu_output.dtype))
+                        ]
+                        inputs[0].set_data_from_numpy(input0)
+                        inputs[1].set_data_from_numpy(gpu_output)
+                        result = client.infer(model_name, inputs)
+                        output0 = result.as_numpy('OUTPUT0')
+                        self.assertTrue(output0 is not None)
+                        self.assertTrue(np.all(output0 == input0))
+
+
+if __name__ == '__main__':
+    unittest.main()

qa/L0_backend_python/io/test.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+UNITTEST_PY=./io_test.py
+CLIENT_LOG="./client.log"
+EXPECTED_NUM_TESTS="1"
+TEST_RESULT_FILE='test_results.txt'
+source ../common.sh
+source ../../common/util.sh
+
+SERVER=/opt/tritonserver/bin/tritonserver
+SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1"
+SERVER_LOG="./inference_server.log"
+REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
+DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"}
+
+RET=0
+rm -fr *.log ./models
+
+pip3 uninstall -y torch
+pip3 install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+
+for i in {1..3}; do
+    model_name=dlpack_io_identity_$i
+    mkdir -p models/$model_name/1/
+    cp ../../python_models/dlpack_io_identity/model.py ./models/$model_name/1/
+    cp ../../python_models/dlpack_io_identity/config.pbtxt ./models/$model_name/
+    (cd models/$model_name && \
+              sed -i "s/^name:.*/name: \"$model_name\"/" config.pbtxt)
+done
+
+mkdir -p models/ensemble_io/1/
+cp ../../python_models/ensemble_io/config.pbtxt ./models/ensemble_io
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    RET=1
+fi
+
+set +e
+python3 $UNITTEST_PY > $CLIENT_LOG
+if [ $? -ne 0 ]; then
+    echo -e "\n***\n*** io_test.py FAILED. \n***"
+    cat $CLIENT_LOG
+    RET=1
+else
+    check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
+fi
+
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+exit $RET

qa/L0_backend_python/restart/restart_test.py

@@ -56,11 +56,12 @@ def test_restart(self):
         # will return an exception.
         with self.assertRaises(InferenceServerException):
             self._infer_helper(model_name, shape, dtype)
-        
+
         # The second request should work properly since the stub process should
         # have come alive.
         self._infer_helper(model_name, shape, dtype)
 
 
 if __name__ == '__main__':
     unittest.main()
+

qa/L0_backend_python/test.sh

@@ -311,6 +311,16 @@ if [ $? -ne 0 ]; then
   RET=1
 fi
 
+(cd unittest && bash -ex test.sh)
+if [ $? -ne 0 ]; then
+  RET=1
+fi
+
+(cd io && bash -ex test.sh)
+if [ $? -ne 0 ]; then
+  RET=1
+fi
+
 if [ $RET -eq 0 ]; then
   echo -e "\n***\n*** Test Passed\n***"
 else
@@ -319,4 +329,3 @@ else
 fi
 
 exit $RET
-

qa/L0_backend_python/unittest/python_unittest.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+
+sys.path.append("../../common")
+
+import test_util as tu
+import unittest
+import tritonclient.http as httpclient
+from tritonclient.utils import *
+import numpy as np
+
+
+class PythonUnittest(tu.TestResultCollector):
+    def test_dlpack_tensor(self):
+        model_name = "dlpack_test"
+        with httpclient.InferenceServerClient("localhost:8000") as client:
+            # Input data is not used.
+            input_data = np.array([1], dtype=np.float32)
+            inputs = [
+                httpclient.InferInput("INPUT0", input_data.shape,
+                                      np_to_triton_dtype(input_data.dtype))
+            ]
+            inputs[0].set_data_from_numpy(input_data)
+            result = client.infer(model_name, inputs)
+            output0 = result.as_numpy('OUTPUT0')
+
+            # The model returns 1 if the tests were sucessfully passed.
+            # Otherwise, it will return 0.
+            self.assertTrue(output0 == [1])
+
+
+if __name__ == '__main__':
+    unittest.main()

qa/L0_backend_python/unittest/test.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+SERVER=/opt/tritonserver/bin/tritonserver
+SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1"
+CLIENT_PY=./python_unittest.py
+CLIENT_LOG="./client.log"
+EXPECTED_NUM_TESTS="1"
+TEST_RESULT_FILE='test_results.txt'
+SERVER_LOG="./inference_server.log"
+REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
+DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"}
+
+RET=0
+rm -fr *.log ./models
+
+source ../../common/util.sh
+
+# Uninstall the non CUDA version of PyTorch
+pip3 uninstall -y torch
+pip3 install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
+pip3 install tensorflow
+
+rm -fr *.log ./models
+
+mkdir -p models/dlpack_test/1/
+cp ../../python_models/dlpack_test/model.py models/dlpack_test/1/
+cp ../../python_models/dlpack_test/config.pbtxt models/dlpack_test
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    RET=1
+fi
+
+set +e
+python3 $CLIENT_PY > $CLIENT_LOG 2>&1 
+
+if [ $? -ne 0 ]; then
+    echo -e "\n***\n*** python_unittest.py FAILED. \n***"
+    RET=1
+else
+    check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Result Verification Failed\n***"
+        RET=1
+    fi
+fi
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+if [ $RET -eq 1 ]; then
+    cat $CLIENT_LOG
+    cat $SERVER_LOG
+    echo -e "\n***\n*** Unittest test FAILED. \n***"
+else
+    echo -e "\n***\n*** Unittest test PASSED. \n***"
+fi
+
+exit $RET

qa/L0_infer/infer_test.py

@@ -33,7 +33,7 @@
 import test_util as tu
 import os
 
-from tritonclientutils import *
+from tritonclient.utils import *
 
 TEST_SYSTEM_SHARED_MEMORY = bool(
     int(os.environ.get('TEST_SYSTEM_SHARED_MEMORY', 0)))
@@ -46,7 +46,7 @@
 assert USE_GRPC or USE_HTTP, "USE_GRPC or USE_HTTP must be non-zero"
 
 BACKENDS = os.environ.get('BACKENDS',
-                          "graphdef savedmodel onnx libtorch plan python")
+                          "graphdef savedmodel onnx libtorch plan python python_dlpack")
 ENSEMBLES = bool(int(os.environ.get('ENSEMBLES', 1)))
 
 np_dtype_string = np.dtype(object)
@@ -204,9 +204,21 @@ def _infer_exact_helper(tester,
                                         output0_raw=output0_raw,
                                         output1_raw=output1_raw,
                                         swap=swap)
+        for prefix in ensemble_prefix:
+            if prefix != "":
+                continue
 
-        if prefix == "":
-            if 'python' in BACKENDS:
+            if 'python_dlpack' in BACKENDS:
+                _infer_exact_helper(self,
+                                    prefix + 'python_dlpack', (input_size,),
+                                    8,
+                                    input_dtype,
+                                    output0_dtype,
+                                    output1_dtype,
+                                    output0_raw=output0_raw,
+                                    output1_raw=output1_raw,
+                                    swap=swap)
+            elif 'python' in BACKENDS:
                 _infer_exact_helper(self,
                                     prefix + 'python', (input_size,),
                                     8,