Include test duration in emails (triton-inference-server#3180)

* Include test duration in emails

* Update test duration

* Update test duration

* Update comments

* Update test duration format

Co-authored-by: Kris Hung <krish@krish-dt.nvidia.com>

qa/L0_client_memory_growth/test.sh

@@ -96,7 +96,9 @@ for PROTOCOL in http grpc; do
             EXTRA_ARGS="-r ${REPETITION_PY} -i ${PROTOCOL}"
         fi
 
+        SECONDS=0
         $LEAKCHECK $LEAKCHECK_ARGS $MEMORY_GROWTH_TEST $EXTRA_ARGS >> ${CLIENT_LOG} 2>&1
+        TEST_DURATION=$SECONDS
         if [ $? -ne 0 ]; then
             cat ${CLIENT_LOG}
             RET=1
@@ -116,7 +118,11 @@ for PROTOCOL in http grpc; do
                 RET=1
             fi
 
-            # Log the graph for memory growth and the change between Average and Max memory usage
+            # Log test duration, the graph for memory growth and the change between Average and Max memory usage
+            hrs=$(printf "%02d" $((TEST_DURATION / 3600)))
+            mins=$(printf "%02d" $(((TEST_DURATION / 60) % 60)))
+            secs=$(printf "%02d" $((TEST_DURATION % 60)))
+            echo -e "Test Duration: $hrs:$mins:$secs (HH:MM:SS)" >> ${GRAPH_LOG}
             cat ${CLIENT_LOG}.massif
             ms_print ${MASSIF_LOG} | head -n35 >> ${GRAPH_LOG}
             cat ${GRAPH_LOG}

qa/L0_memory_growth/test.sh

@@ -143,7 +143,9 @@ for MODEL in $(ls models); do
 
     # Run the perf analyzer 'REPETITION' times
     for ((i=1; i<=$REPETITION; i++)); do
+        SECONDS=0
         $PERF_ANALYZER -v -m $MODEL -i grpc --concurrency-range $CONCURRENCY -b $CLIENT_BS >> $CLIENT_LOG 2>&1
+        TEST_DURATION=$SECONDS
         if [ $? -ne 0 ]; then
             cat $CLIENT_LOG
             echo -e "\n***\n*** perf_analyzer for $MODEL failed on iteration $i\n***"
@@ -159,6 +161,11 @@ for MODEL in $(ls models); do
 
     set +e
 
+    # Log test duration and the graph for memory growth
+    hrs=$(printf "%02d" $((TEST_DURATION / 3600)))
+    mins=$(printf "%02d" $(((TEST_DURATION / 60) % 60)))
+    secs=$(printf "%02d" $((TEST_DURATION % 60)))
+    echo -e "Test Duration: $hrs:$mins:$secs (HH:MM:SS)" >> ${GRAPH_LOG}
     ms_print ${MASSIF_LOG} | head -n35 >> ${GRAPH_LOG}
     cat ${GRAPH_LOG}
     # Check the massif output
@@ -207,7 +214,9 @@ done
 # set +e
 
 # # Run the busy_op test
+# SECONDS=0
 # python $BUSY_OP_TEST -v -m graphdef_busyop -d $DELAY_CYCLES -n $NUM_REQUESTS > $CLIENT_LOG 2>&1
+# TEST_DURATION=$SECONDS
 # if [ $? -ne 0 ]; then
 #     cat $CLIENT_LOG
 #     echo -e "\n***\n*** Test graphdef_busyop Failed\n***"
@@ -221,6 +230,11 @@ done
 
 # set +e
 
+# # Log test duration and the graph for memory growth
+# hrs=$(printf "%02d" $((TEST_DURATION / 3600)))
+# mins=$(printf "%02d" $(((TEST_DURATION / 60) % 60)))
+# secs=$(printf "%02d" $((TEST_DURATION % 60)))
+# echo -e "Test Duration: $hrs:$mins:$secs (HH:MM:SS)" >> ${GRAPH_LOG}
 # ms_print ${MASSIF_LOG} | head -n35 >> ${GRAPH_LOG}
 # cat ${GRAPH_LOG}
 # # Check the massif output