switch to a new cv frontend, with imshow enabled

ErcBunny · Mar 8, 2023 · 5b749da · 5b749da
1 parent da8c2f1
commit 5b749da
Show file tree

Hide file tree

Showing 8 changed files with 144 additions and 14 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -14,7 +14,70 @@
         "stdio.h": "c",
         "string": "cpp",
         "tensor": "c",
-        "*.tcc": "c"
+        "*.tcc": "c",
+        "ostream": "cpp",
+        "iostream": "cpp",
+        "array": "cpp",
+        "atomic": "cpp",
+        "hash_map": "cpp",
+        "hash_set": "cpp",
+        "bit": "cpp",
+        "bitset": "cpp",
+        "cctype": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "cmath": "cpp",
+        "complex": "cpp",
+        "condition_variable": "cpp",
+        "cstdarg": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "deque": "cpp",
+        "list": "cpp",
+        "map": "cpp",
+        "set": "cpp",
+        "unordered_map": "cpp",
+        "unordered_set": "cpp",
+        "vector": "cpp",
+        "exception": "cpp",
+        "algorithm": "cpp",
+        "functional": "cpp",
+        "iterator": "cpp",
+        "memory": "cpp",
+        "memory_resource": "cpp",
+        "numeric": "cpp",
+        "optional": "cpp",
+        "random": "cpp",
+        "ratio": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "tuple": "cpp",
+        "type_traits": "cpp",
+        "utility": "cpp",
+        "fstream": "cpp",
+        "initializer_list": "cpp",
+        "iomanip": "cpp",
+        "iosfwd": "cpp",
+        "istream": "cpp",
+        "limits": "cpp",
+        "mutex": "cpp",
+        "new": "cpp",
+        "sstream": "cpp",
+        "stdexcept": "cpp",
+        "streambuf": "cpp",
+        "thread": "cpp",
+        "cinttypes": "cpp",
+        "typeindex": "cpp",
+        "typeinfo": "cpp",
+        "valarray": "cpp",
+        "*.traits": "cpp",
+        "fft": "cpp"
     },
     "C_Cpp.dimInactiveRegions": false,
     "esbonio.sphinx.confDir": "",

diff --git a/conf/airframes/tudelft/bebop_course_maze_runner.xml b/conf/airframes/tudelft/bebop_course_maze_runner.xml
@@ -61,17 +61,13 @@
       <define name="VIDEO_CAPTURE_CAMERA" value="front_camera"/>
       <define name="VIDEO_CAPTURE_FPS" value="10"/>
     </module>
-    <module name="cv_detect_color_object">
-      <define name="COLOR_OBJECT_DETECTOR_CAMERA1" value="front_camera"/>
-      <define name="COLOR_OBJECT_DETECTOR_FPS1" value="0"/>
-      <define name="COLOR_OBJECT_DETECTOR_DRAW1" value="1"/>
-    </module>
     <module name="maze_runner">
       <define name="MAZE_RUNNER_VISUAL_DETECTION_ID" value="COLOR_OBJECT_DETECTION1_ID"/>
     </module>
     <module name="cv_maze_runner">
       <define name="CAMERA" value="front_camera"/>
       <define name="MAX_FPS" value="0"/>
+      <define name="COLOR_FILTER_DRAW" value="1"/>
     </module>
     <module name="video_rtp_stream">
       <define name="VIEWVIDEO_CAMERA" value="front_camera"/>

diff --git a/conf/modules/cv_maze_runner.xml b/conf/modules/cv_maze_runner.xml
@@ -39,6 +39,7 @@
   <makefile target="nps">
     <file name="cv_maze_runner.c"/>
     <file name="opencv_maze_runner.cpp"/>
+    <!-- <file name="opencv_image_functions.cpp"/> -->
 
     <flag name="CXXFLAGS" value="I$(PAPARAZZI_SRC)/sw/ext/opencv_bebop/install_pc/include"/>
     <flag name="LDFLAGS" value="L$(PAPARAZZI_HOME)/sw/ext/opencv_bebop/install_pc/lib"/>
@@ -60,5 +61,11 @@
     <flag name="LDFLAGS" value="lfreetype"/>
     <flag name="LDFLAGS" value="lharfbuzz"/>
     <flag name="LDFLAGS" value="lrt"/>
+    <flag name="LDFLAGS" value="lgobject-2.0"/>
+    <flag name="LDFLAGS" value="lcairo"/>
+    <flag name="LDFLAGS" value="lgdk_pixbuf-2.0"/>
+    <flag name="LDFLAGS" value="lgtk-3"/>
+    <flag name="LDFLAGS" value="lgdk-3"/>
+
   </makefile>
 </module>
diff --git a/conf/userconf/tudelft/course_conf.xml b/conf/userconf/tudelft/course_conf.xml
@@ -29,7 +29,7 @@
    telemetry="telemetry/default_rotorcraft.xml"
    flight_plan="flight_plans/tudelft/course_maze_runner_cyberzoo.xml"
    settings="settings/rotorcraft_basic.xml"
-   settings_modules="modules/ahrs_int_cmpl_quat.xml modules/bebop_cam.xml modules/cv_detect_color_object.xml modules/gps.xml modules/guidance_rotorcraft.xml modules/imu_common.xml modules/ins_extended.xml modules/nav_basic_rotorcraft.xml modules/maze_runner.xml modules/stabilization_indi_simple.xml modules/video_capture.xml modules/video_rtp_stream.xml modules/cv_maze_runner.xml"
+   settings_modules="modules/ahrs_int_cmpl_quat.xml modules/bebop_cam.xml modules/cv_maze_runner.xml modules/gps.xml modules/guidance_rotorcraft.xml modules/imu_common.xml modules/ins_extended.xml modules/maze_runner.xml modules/nav_basic_rotorcraft.xml modules/stabilization_indi_simple.xml modules/video_capture.xml modules/video_rtp_stream.xml"
    gui_color="white"
   />
 </conf>
diff --git a/playground/misc.ipynb b/playground/misc.ipynb
@@ -223,6 +223,25 @@
     "plt.imshow(abs_grad_x, cmap = 'gray')\n",
     "plt.show()"
    ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## USE imshow for debug\n",
+    "\n",
+    "1. compile opencv with -DWITH_GTK=ON\n",
+    "2. link required libraries in module conf:\n",
+    "\n",
+    "```\n",
+    "<flag name=\"LDFLAGS\" value=\"lgobject-2.0\"/>\n",
+    "<flag name=\"LDFLAGS\" value=\"lcairo\"/>\n",
+    "<flag name=\"LDFLAGS\" value=\"lgdk_pixbuf-2.0\"/>\n",
+    "<flag name=\"LDFLAGS\" value=\"lgtk-3\"/>\n",
+    "<flag name=\"LDFLAGS\" value=\"lgdk-3\"/>\n",
+    "```"
+   ]
   }
  ],
  "metadata": {

diff --git a/sw/airborne/modules/computer_vision/cv_maze_runner.c b/sw/airborne/modules/computer_vision/cv_maze_runner.c
@@ -7,16 +7,13 @@
 #define VERBOSE_PRINT(...)
 #endif
 
-
 static pthread_mutex_t mutex;
+static bool first_cb = true;
 
 struct image_t *video_cb(struct image_t *img, uint8_t camera_id)
 {
-  // camera_id is set by user
-  // using front or down cam is set in cv_maze_runner.xml and bebop_course_maze_runner.xml
-
-
 
+  opencv_frontend_run(img);
   return NULL;
 }
 

diff --git a/sw/airborne/modules/computer_vision/opencv_maze_runner.cpp b/sw/airborne/modules/computer_vision/opencv_maze_runner.cpp
@@ -4,6 +4,51 @@
 #include <opencv2/imgproc.hpp>
 #include <opencv2/core/utility.hpp>
 #include <opencv2/optflow.hpp>
+#include <opencv2/highgui.hpp>
+
 
 using namespace std;
 using namespace cv;
+using namespace optflow;
+
+Mat rotate(Mat src);
+Mat rotate(Mat src)
+{
+  Mat rotated;
+  transpose(src, rotated);
+  flip(rotated, rotated, 0);
+  return rotated;
+}
+
+Mat prevgray;
+Ptr<DenseOpticalFlow> algorithm = createOptFlow_DIS(DISOpticalFlow::PRESET_MEDIUM);
+
+void opencv_frontend_run(struct image_t *input)
+{
+  Mat src_yuv, src_bgr, src_gray;
+  src_yuv = Mat(input->h, input->w, CV_8UC2, (char *)input->buf);
+  cvtColor(src_yuv, src_gray, CV_YUV2GRAY_Y422);
+
+  Mat rgb, frame;
+  Mat flow, flow_uv[2];
+  Mat mag, ang;
+  Mat hsv_split[3], hsv;
+  if (!prevgray.empty())
+  {
+    algorithm->calc(prevgray, src_gray, flow);
+    split(flow, flow_uv);
+    multiply(flow_uv[1], -1, flow_uv[1]);
+    cartToPolar(flow_uv[0], flow_uv[1], mag, ang, true);
+    normalize(mag, mag, 0, 1, NORM_MINMAX);
+    hsv_split[0] = ang;
+    hsv_split[1] = Mat::ones(ang.size(), ang.type());
+    hsv_split[2] = mag;
+    merge(hsv_split, 3, hsv);
+    cvtColor(hsv, rgb, COLOR_HSV2BGR);
+    imshow("flow", rotate(rgb));
+    imshow("orig", rotate(src_gray));
+  }
+  std::swap(prevgray, src_gray);
+
+  waitKey(1);
+}
diff --git a/sw/airborne/modules/computer_vision/opencv_maze_runner.h b/sw/airborne/modules/computer_vision/opencv_maze_runner.h
@@ -5,8 +5,11 @@
 extern "C" {
 #endif
 
-// declare wrapper functions here
-// do not include any opencv elements here
+#include "modules/computer_vision/cv.h"
+#include "modules/computer_vision/lib/vision/image.h"
+
+void opencv_frontend_run(struct image_t *input);
+
 
 #ifdef __cplusplus
 }