🎉 Begin CPP integration tutorial

.gitignore

@@ -146,9 +146,6 @@ venv/
 # data
 ckpts/
 test*.py
-*.txt
-
-
-
-
 
+# cpp_integration tutorial build
+tutorials/cpp_integration_build

README.md

@@ -154,8 +154,10 @@ python3 train.py --config=example/hole_detector/cfgs/hole.yaml
 ```
 
 The trained models and weights will be located at `yolox/ckpts`.
-- `mAP-0.XXXX` files represent the full model in Keras format. Those models can be used with `simple_detector.py`.
-- `mAP-0.XXXX.h5` files contain only the weight.
+- `mAP-0.XXXX` files represent the **full model** in Keras format. Those models can be used with `simple_detector.py`.
+- `mAP-0.XXXX.h5` files contain **only** the **weights**.
+
+**Note:** The trained model can be converted into a **frozen graph** or a **ONNX** model thanks to [`model_converter.py`](./tools/model/model_converter.py). For further detail, please use `python3 model_converter.py --help`.
 
 ---
 
@@ -262,3 +264,19 @@ Standard Method Package includes Flip left and right,  Crop and Zoom(jitter=0.3)
 | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ----- | ----- |
 | ✔    | ✔    |      | ✔    |      | CIoU | 40.6 | 72.2  | 40.3  |
 | ✔    | ✔    |      | ✔    | ✔    | CIoU |      |       |       |
+
+# How to use a python-based inference on a C++ environnement
+
+[`./tutorials/cpp_integration`](./tutorials/cpp_integration/tutorial_cpp_integration.cpp) is a mini python/c++ project which used a tf/keras trained model in a C++ environnement. Here, we use data from the [hole detector example](./example/hole_detector).
+
+Build and run:
+```shell
+cd tutorials/
+mkdir -p cpp_integration_build/ && cd cpp_integration_build/
+cmake -G Ninja ../cpp_integration/
+ninja && ./tutorial_cpp_integration --model ../../example/hole_detector/models/tiny_yolox --img ../../example/hole_detector/data/Images/frame0000.jpg
+```
+
+![](./misc/infer_from_cpp.png)
+
+The first number represents the class id and the second number ([%]) represents the detection score.

docker_images/cpu/Dockerfile

@@ -9,21 +9,86 @@ RUN apt update && apt install --no-install-recommends -y python3-pip &&\
     apt-mirror-updater -a -q &&\
     rm -rf /var/lib/apt/lists/*
 
-# OpenCV
+
+# Install Build, OpenCV dependencies
 RUN apt-mirror-updater -a -q &&\
-    apt install --no-install-recommends -y python3-opencv &&\
-    pip3 install -U pip &&\
-    pip3 install opencv-python &&\
-    pip3 install pyyaml &&\
+    apt install --no-install-recommends -y build-essential cmake ninja-build git pkg-config &&\
+    apt install --no-install-recommends -y libgtk2.0-dev libx11-dev libjpeg-dev libpng-dev &&\
+    rm -rf /var/lib/apt/lists/*
+
+# Install OpenCV
+RUN mkdir -p /tmp/opencv &&\
+    cd /tmp/opencv &&\
+    git clone https://github.com/opencv/opencv.git &&\
+    mkdir opencv/build &&\
+    cd opencv/build &&\
+    cmake -G Ninja .. \
+    -DBUILD_PERF_TESTS=Off \
+    -DBUILD_TESTS=Off \
+    -DBUILD_EXAMPLES=Off \
+    -DBUILD_opencv_apps=Off \
+    -DBUILD_opencv_calib3d=Off \
+    -DBUILD_opencv_core=On \
+    -DBUILD_opencv_dnn=Off \
+    -DBUILD_opencv_features2d=Off \
+    -DBUILD_opencv_flann=Off \
+    -DBUILD_opencv_highgui=On \
+    -DBUILD_opencv_imgcodecs=On \
+    -DBUILD_opencv_imgproc=On \
+    -DBUILD_opencv_java_bindings_generator=Off \
+    -DBUILD_opencv_js=Off \
+    -DBUILD_opencv_ml=Off \
+    -DBUILD_opencv_objdetect=Off \
+    -DBUILD_opencv_photo=Off \
+    -DBUILD_opencv_python2=Off \
+    -DBUILD_opencv_python3=On \
+    -DBUILD_opencv_python_bindings_generator=Off \
+    -DBUILD_opencv_python_tests=Off \
+    -DBUILD_opencv_shape=Off \
+    -DBUILD_opencv_stitching=Off \
+    -DBUILD_opencv_superres=Off \
+    -DBUILD_opencv_ts=Off \
+    -DBUILD_opencv_video=On \
+    -DBUILD_opencv_videoio=On \
+    -DBUILD_opencv_videostab=Off \
+    -DBUILD_opencv_world=Off \
+    -DWITH_JASPER=Off \
+    -DWITH_ITT=Off \
+    -DWITH_JPEG=On \
+    -DWITH_WEBP=Off \
+    -DWITH_PNG=On \
+    -DWITH_TIFF=Off \
+    -DWITH_OPENEXR=Off \
+    -DWITH_OPENCL=Off\
+    -DWITH_IPP=Off \
+    -DWITH_PROTOBUF=Off \
+    -DWITH_GTK_2_X=On \
+    -DCMAKE_BUILD_TYPE=Release &&\
+    ninja &&\
+    ninja install &&\
+    cd ~ &&\
+    rm -rf /tmp/opencv
+
+# Boost
+RUN apt-mirror-updater -a -q &&\
+    apt install --no-install-recommends -y libboost-python-dev libboost-numpy-dev &&\
     rm -rf /var/lib/apt/lists/*
 
+# OpenCV (Python)
+RUN pip3 install -U pip &&\
+    pip3 install opencv-python
+
+# Yaml
+RUN pip3 install pyyaml
+
 # LabelImg
 RUN apt-mirror-updater -a -q &&\
     apt install --no-install-recommends -y pyqt5-dev-tools &&\
     pip3 install -U pip &&\
     pip3 install labelImg
 
 # Model conversion
-RUN pip3 install tf2onnx
+RUN pip3 install -U pip &&\
+    pip3 install tf2onnx
 
 RUN ldconfig

docker_images/gpu/Dockerfile

@@ -21,21 +21,85 @@ RUN apt update && apt install --no-install-recommends -y python3-pip &&\
 RUN pip3 install -U pip &&\
     pip3 install -U tensorflow==2.7.0
 
-# OpenCV
+# Install Build, OpenCV dependencies
 RUN apt-mirror-updater -a -q &&\
-    apt install --no-install-recommends -y python3-opencv &&\
-    pip3 install -U pip &&\
-    pip3 install opencv-python &&\
-    pip3 install pyyaml &&\
+    apt install --no-install-recommends -y build-essential cmake ninja-build git pkg-config &&\
+    apt install --no-install-recommends -y libgtk2.0-dev libx11-dev libjpeg-dev libpng-dev &&\
+    rm -rf /var/lib/apt/lists/*
+
+# Install OpenCV
+RUN mkdir -p /tmp/opencv &&\
+    cd /tmp/opencv &&\
+    git clone https://github.com/opencv/opencv.git &&\
+    mkdir opencv/build &&\
+    cd opencv/build &&\
+    cmake -G Ninja .. \
+    -DBUILD_PERF_TESTS=Off \
+    -DBUILD_TESTS=Off \
+    -DBUILD_EXAMPLES=Off \
+    -DBUILD_opencv_apps=Off \
+    -DBUILD_opencv_calib3d=Off \
+    -DBUILD_opencv_core=On \
+    -DBUILD_opencv_dnn=Off \
+    -DBUILD_opencv_features2d=Off \
+    -DBUILD_opencv_flann=Off \
+    -DBUILD_opencv_highgui=On \
+    -DBUILD_opencv_imgcodecs=On \
+    -DBUILD_opencv_imgproc=On \
+    -DBUILD_opencv_java_bindings_generator=Off \
+    -DBUILD_opencv_js=Off \
+    -DBUILD_opencv_ml=Off \
+    -DBUILD_opencv_objdetect=Off \
+    -DBUILD_opencv_photo=Off \
+    -DBUILD_opencv_python2=Off \
+    -DBUILD_opencv_python3=On \
+    -DBUILD_opencv_python_bindings_generator=Off \
+    -DBUILD_opencv_python_tests=Off \
+    -DBUILD_opencv_shape=Off \
+    -DBUILD_opencv_stitching=Off \
+    -DBUILD_opencv_superres=Off \
+    -DBUILD_opencv_ts=Off \
+    -DBUILD_opencv_video=On \
+    -DBUILD_opencv_videoio=On \
+    -DBUILD_opencv_videostab=Off \
+    -DBUILD_opencv_world=Off \
+    -DWITH_JASPER=Off \
+    -DWITH_ITT=Off \
+    -DWITH_JPEG=On \
+    -DWITH_WEBP=Off \
+    -DWITH_PNG=On \
+    -DWITH_TIFF=Off \
+    -DWITH_OPENEXR=Off \
+    -DWITH_OPENCL=Off\
+    -DWITH_IPP=Off \
+    -DWITH_PROTOBUF=Off \
+    -DWITH_GTK_2_X=On \
+    -DCMAKE_BUILD_TYPE=Release &&\
+    ninja &&\
+    ninja install &&\
+    cd ~ &&\
+    rm -rf /tmp/opencv
+
+# Boost
+RUN apt-mirror-updater -a -q &&\
+    apt install --no-install-recommends -y libboost-python-dev libboost-numpy-dev &&\
     rm -rf /var/lib/apt/lists/*
 
+# OpenCV (Python)
+RUN pip3 install -U pip &&\
+    pip3 install opencv-python
+
+# Yaml
+RUN pip3 install pyyaml
+
 # LabelImg
 RUN apt-mirror-updater -a -q &&\
     apt install --no-install-recommends -y pyqt5-dev-tools &&\
     pip3 install -U pip &&\
     pip3 install labelImg
 
 # Model conversion
-RUN pip3 install tf2onnx
+RUN pip3 install -U pip &&\
+    pip3 install tf2onnx
 
 RUN ldconfig

example/hole_detector/data/train.txt

@@ -0,0 +1,8 @@
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0007.jpg 179,114,193,129,0 349,244,362,255,0 449,146,463,159,0 477,341,489,351,0 577,418,589,426,0 578,267,591,278,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0008.jpg 70,86,80,96,0 122,243,131,252,0 339,155,349,164,0 429,231,439,240,0 495,173,505,182,0 438,46,451,57,0 570,247,580,255,0 506,296,515,303,0 570,349,579,358,0 557,415,566,424,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0001.jpg 50,174,64,187,0 349,244,362,255,0 393,378,405,388,0 510,89,524,104,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0005.jpg 230,64,241,75,0 340,156,350,165,0 404,85,415,96,0 431,231,440,240,0 432,320,441,328,0 578,416,585,422,0 573,350,581,357,0 607,219,617,228,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0003.jpg 220,56,231,67,0 332,149,342,159,0 345,256,354,265,0 525,134,536,144,0 602,214,613,223,0 502,291,512,299,0 515,369,523,376,0 601,466,610,475,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0004.jpg 201,95,214,108,0 350,212,362,223,0 605,194,618,205,0 559,377,570,385,0 566,460,577,469,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0002.jpg 94,140,106,152,0 345,202,356,212,0 384,326,394,333,0 523,465,534,475,0 586,186,598,198,0 476,67,489,80,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0000.jpg 143,104,153,114,0 339,155,349,165,0 440,47,450,57,0 530,141,541,152,0 492,390,502,400,0 

example/hole_detector/data/valid.txt

@@ -0,0 +1,2 @@
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0009.jpg 234,56,244,67,0 339,155,349,164,0 151,159,160,168,0 341,398,351,407,0 
+/root/deep-learning-ws/yolox/example/hole_detector/data/Images/frame0006.jpg 5,171,14,180,0 229,63,239,73,0 339,155,349,164,0 403,84,414,95,0 430,231,439,240,0 495,173,505,182,0 506,295,516,303,0 500,371,508,378,0 628,397,637,404,0 606,219,615,228,0 

tutorials/cpp_integration/.clang-format

@@ -0,0 +1,12 @@
+Language: Cpp
+BasedOnStyle: LLVM
+BreakBeforeBraces: Allman
+ColumnLimit: 120
+ConstructorInitializerIndentWidth: 2
+Standard: Auto
+AlignConsecutiveAssignments: true
+AlwaysBreakAfterReturnType: TopLevel
+BreakConstructorInitializers: BeforeComma
+Cpp11BracedListStyle: false
+SpacesInAngles: true
+SpacesInParentheses: true

tutorials/cpp_integration/CMakeLists.txt

@@ -0,0 +1,77 @@
+cmake_minimum_required(VERSION 3.16.3)
+project(cpp_integration)
+
+# Python
+find_package( PythonLibs   3 )
+find_package( Python3 REQUIRED COMPONENTS Interpreter Development )
+set( PYTHON_VERSION ${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR} )
+
+# Boost
+find_package(Boost COMPONENTS
+  python${PYTHON_VERSION}
+  numpy${PYTHON_VERSION}
+)
+
+if( NOT Boost_FOUND )
+  ## Fallback to pythonMajor
+  find_package( Boost REQUIRED COMPONENTS
+    python${PYTHON_VERSION_MAJOR}
+    numpy${PYTHON_VERSION_MAJOR}
+  )
+endif()
+
+# OpenCV
+find_package(OpenCV REQUIRED )
+
+# CXX_STANDARD
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+###################
+## Build library ##
+###################
+set(PROJECT_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/include)
+set(PROJECT_LIBRARIES ${PROJECT_NAME})
+
+# Data
+set(data_lib
+  include/data/Hole.hpp
+  src/data/Hole.cpp
+)
+
+# Detector
+set(detector_lib
+  include/detector/DnnHoleLocalizer.hpp
+  src/detector/DnnHoleLocalizer.cpp
+)
+
+# Gather all libs
+set(srcs_lib
+  ${data_lib}
+  ${detector_lib}
+)
+
+add_library(${PROJECT_LIBRARIES} ${srcs_lib})
+target_link_libraries(${PROJECT_LIBRARIES} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} ${OpenCV_LIBS})
+include_directories(${Boost_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS})
+
+########################################
+## Install python_deeplearning module ##
+########################################
+add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy_directory
+        ${CMAKE_CURRENT_SOURCE_DIR}/deeplearning/python_deeplearning /tmp/python/python_deeplearning)
+
+###########################
+## Build tutorials ##
+###########################
+set(srcs_tutorials
+  tutorial_cpp_integration.cpp
+)
+
+foreach(cpp ${srcs_tutorials})
+  get_filename_component(target ${cpp} NAME_WE)
+  include_directories(${Boost_INCLUDE_DIRS} ${PROJECT_INCLUDE_DIRS})
+  add_executable("${target}" ${cpp})
+  target_link_libraries("${target}" ${PROJECT_LIBRARIES})
+endforeach()

tutorials/cpp_integration/deeplearning/python_deeplearning/__init__.py

@@ -0,0 +1,12 @@
+import tensorflow as tf
+import os
+
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
+
+gpus = tf.config.list_physical_devices('GPU')
+if gpus:
+    try:
+        for gpu in gpus:
+            tf.config.experimental.set_memory_growth(gpu, True)
+    except RuntimeError as e:
+        print(e)

tutorials/cpp_integration/deeplearning/python_deeplearning/predict.py

@@ -0,0 +1,25 @@
+import os
+import numpy as np
+
+from tensorflow.keras.models import load_model
+from .utils.utils import postprocess_bboxes
+
+
+class Inference:
+    def __init__(self, model_path):
+        assert os.path.isdir(model_path), \
+            "Model {} does not exist.".format(model_path)
+
+        # Init tf model
+        self.model = load_model(model_path)
+
+    def predict(self, image, origin_h, origin_w, net_size, score_thresh):
+
+        images = np.expand_dims(image, axis=0)
+        bboxes, scores, classes, valid_detections = self.model.predict(images)
+
+        bboxes = bboxes[0][:valid_detections[0]]
+        scores = scores[0][:valid_detections[0]]
+        classes = classes[0][:valid_detections[0]]
+
+        return postprocess_bboxes((net_size, net_size), (origin_w, origin_h), bboxes, scores, classes, score_thresh)

tutorials/cpp_integration/deeplearning/python_deeplearning/utils/bbox.py

@@ -0,0 +1,11 @@
+
+
+class BoundingBox:
+    def __init__(self, xmin, ymin, xmax, ymax, score, cls):
+        self.xmin = xmin
+        self.ymin = ymin
+        self.xmax = xmax
+        self.ymax = ymax
+
+        self.score = score
+        self.cls = int(cls)