Image recognition using camera basler and Yolo

[ManoelVictor9]

Describe what you want to implement and what the issue & the steps to reproduce it are:

I am implementing a hardware trigger capture system with 4 images that are combined using basler camera, and I am aiming to develop image recognition using YOLOv8 for each of the 4 captured images, however, when I run the program, it prints that recognition has occurred of the images, but the bounding boxes do not appear in the images sampled in real time.
Could anyone help me with this error?

Is your camera operational in Basler pylon viewer on your platform

Yes

Hardware setup & camera model(s) used

Camera: camera basler aca1300-30gm

Runtime information:

Code used:

from pypylon import pylon
import cv2
import numpy as np
from ultralytics import YOLO


camera = pylon.InstantCamera(pylon.TlFactory.GetInstance().CreateFirstDevice())


model = YOLO('best2.pt') 

# changing this to True changes the Trigger to Line1 instead of the SpaceKey
EXTERNAL_TRIGGER = True

try:
    # setup camera aoi
    camera.Open()
    camera.Width.Value = camera.Width.Max
    camera.Height.Value = camera.Height.Max

    # set camera to frame trigger mode on Line1
    camera.TriggerSelector.Value = "FrameStart"
    if EXTERNAL_TRIGGER:
        camera.TriggerSource.Value = "Line1"
    else:
        camera.TriggerSource.Value = "Software"
    camera.TriggerMode.Value = "On"

except Exception as e:
    print(f"Error when configuring the camera: {e}")

# constant values
num_images = 4
screen_width, screen_height = 1080  , 720

# runtime values
resized_images = []
current_image_index = 0
combined_image = np.zeros((screen_height, screen_width, 3), dtype=np.uint8)

# configure image converter
converter = pylon.ImageFormatConverter()
converter.OutputPixelFormat = pylon.PixelType_BGR8packed
converter.OutputBitAlignment = pylon.OutputBitAlignment_MsbAligned

camera.StartGrabbing(pylon.GrabStrategy_LatestImageOnly)

while camera.IsGrabbing():
    while current_image_index < num_images:

        cv2.imshow('Combined Images', combined_image)
        key = cv2.waitKey(1)
        if key == 27:  # Esc key to exit
            camera.StopGrabbing()
            break
        elif not EXTERNAL_TRIGGER and key == ord(" "):
            camera.ExecuteSoftwareTrigger()

        # you cant check your key entry and wait for the next image in one thread at the same time,
        # so you can use this wait-object to check for new images and skip the 5 sec Timeout during RecieveResult
        if not camera.GetGrabResultWaitObject().Wait(10):
            continue

        try:
            # use the context handler, so you dont have to call "grabResult.Release" at the end
            with camera.RetrieveResult(5000, pylon.TimeoutHandling_ThrowException) as grabResult:
                assert grabResult.GrabSucceeded()
                # Accessing image data
                image = converter.Convert(grabResult)
                img = image.GetArray()
        except pylon.TimeoutException as timeout_error:
            raise AssertionError("Timeout error, this should not happen, "
                                 "because we waited for the image in the wait object before!") from timeout_error

        except AssertionError as assertion_error:
            raise AssertionError("Unsuccessful grab, this should not happen at all!") from assertion_error

         # Detect objects using YOLO
        detections = model.predict(img)

        # Draw bounding boxes on resized image
        try:
            for detection in detections:
                x1, y1, x2, y2, conf, cls = detection
                x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
                cv2.rectangle(resized_images[current_image_index], (x1, y1), (x2, y2), (0, 255, 0), 2)
                cv2.putText(resized_images[current_image_index], cls, (x1, y1), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
        except ValueError:
            pass  


        # every image is grabbed on purpose, just add every incoming image
        if len(resized_images) < num_images:
            resized_images.append(img.copy())
        else:
            resized_images[current_image_index] = img.copy()
            current_image_index = (current_image_index + 1) % num_images

        for i in range(len(resized_images)):
            h, w, _ = resized_images[i].shape
            h_ratio = screen_height // 2
            w_ratio = screen_width // 2
            row = i // 2
            col = i % 2

            resized_images[i] = cv2.resize(resized_images[i], (w_ratio, h_ratio))
            combined_image[row * h_ratio: (row + 1) * h_ratio, col * w_ratio: (col + 1) * w_ratio, :] = resized_images[
                i]

cv2.waitKey(0)
cv2.destroyAllWindows()
camera.Close()

[SMA2016a]

Just wanted to clarify that the issue described isn't actually related to pypylon.

may be you can try to add one more waitkey(1) for the end of while loop.

while current_image_index < num_images:

cv2.waitKey(1)