Oriented Object Detection

[ghost]

Search before asking

• I have searched the HUB issues and discussions and found no similar questions.

Question

What do you thing about this: https://blog.roboflow.com/yolov5-for-oriented-object-detection/? Is it possible to do this on ios? Some models contain additional properties such as:

//
// detection_v5.swift
//
// This file was automatically generated and should not be edited.
//

import CoreML


/// Model Prediction Input Type
@available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *)
class detection_v5Input : MLFeatureProvider {

    /// input as color (kCVPixelFormatType_32BGRA) image buffer, 256 pixels wide by 128 pixels high
    var input: CVPixelBuffer

    var featureNames: Set<String> {
        get {
            return ["input"]
        }
    }
    
    func featureValue(for featureName: String) -> MLFeatureValue? {
        if (featureName == "input") {
            return MLFeatureValue(pixelBuffer: input)
        }
        return nil
    }
    
    init(input: CVPixelBuffer) {
        self.input = input
    }

    convenience init(inputWith input: CGImage) throws {
        self.init(input: try MLFeatureValue(cgImage: input, pixelsWide: 256, pixelsHigh: 128, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!)
    }

    convenience init(inputAt input: URL) throws {
        self.init(input: try MLFeatureValue(imageAt: input, pixelsWide: 256, pixelsHigh: 128, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!)
    }

    func setInput(with input: CGImage) throws  {
        self.input = try MLFeatureValue(cgImage: input, pixelsWide: 256, pixelsHigh: 128, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!
    }

    func setInput(with input: URL) throws  {
        self.input = try MLFeatureValue(imageAt: input, pixelsWide: 256, pixelsHigh: 128, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!
    }

}


/// Model Prediction Output Type
@available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *)
class detection_v5Output : MLFeatureProvider {

    /// Source provided by CoreML
    private let provider : MLFeatureProvider

    /// probs as multidimensional array of floats
    var probs: MLMultiArray {
        return self.provider.featureValue(for: "probs")!.multiArrayValue!
    }

    /// probs as multidimensional array of floats
    @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
    var probsShapedArray: MLShapedArray<Float> {
        return MLShapedArray<Float>(self.probs)
    }

    /// coords as multidimensional array of floats
    var coords: MLMultiArray {
        return self.provider.featureValue(for: "coords")!.multiArrayValue!
    }

    /// coords as multidimensional array of floats
    @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
    var coordsShapedArray: MLShapedArray<Float> {
        return MLShapedArray<Float>(self.coords)
    }

    /// size_width as multidimensional array of floats
    var size_width: MLMultiArray {
        return self.provider.featureValue(for: "size_width")!.multiArrayValue!
    }

    /// size_width as multidimensional array of floats
    @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
    var size_widthShapedArray: MLShapedArray<Float> {
        return MLShapedArray<Float>(self.size_width)
    }

    /// size_height as multidimensional array of floats
    var size_height: MLMultiArray {
        return self.provider.featureValue(for: "size_height")!.multiArrayValue!
    }

    /// size_height as multidimensional array of floats
    @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
    var size_heightShapedArray: MLShapedArray<Float> {
        return MLShapedArray<Float>(self.size_height)
    }

    /// side_squeezed as multidimensional array of floats
    var side_squeezed: MLMultiArray {
        return self.provider.featureValue(for: "side_squeezed")!.multiArrayValue!
    }

    /// side_squeezed as multidimensional array of floats
    @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
    var side_squeezedShapedArray: MLShapedArray<Float> {
        return MLShapedArray<Float>(self.side_squeezed)
    }

    var featureNames: Set<String> {
        return self.provider.featureNames
    }
    
    func featureValue(for featureName: String) -> MLFeatureValue? {
        return self.provider.featureValue(for: featureName)
    }

    init(probs: MLMultiArray, coords: MLMultiArray, size_width: MLMultiArray, size_height: MLMultiArray, side_squeezed: MLMultiArray) {
        self.provider = try! MLDictionaryFeatureProvider(dictionary: ["probs" : MLFeatureValue(multiArray: probs), "coords" : MLFeatureValue(multiArray: coords), "size_width" : MLFeatureValue(multiArray: size_width), "size_height" : MLFeatureValue(multiArray: size_height), "side_squeezed" : MLFeatureValue(multiArray: side_squeezed)])
    }

    init(features: MLFeatureProvider) {
        self.provider = features
    }
}


/// Class for model loading and prediction
@available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *)
class detection_v5 {
    let model: MLModel

    /// URL of model assuming it was installed in the same bundle as this class
    class var urlOfModelInThisBundle : URL {
        let bundle = Bundle(for: self)
        return bundle.url(forResource: "detection_v5", withExtension:"mlmodelc")!
    }

    /**
        Construct detection_v5 instance with an existing MLModel object.

        Usually the application does not use this initializer unless it makes a subclass of detection_v5.
        Such application may want to use `MLModel(contentsOfURL:configuration:)` and `detection_v5.urlOfModelInThisBundle` to create a MLModel object to pass-in.

        - parameters:
          - model: MLModel object
    */
    init(model: MLModel) {
        self.model = model
    }

    /**
        Construct detection_v5 instance by automatically loading the model from the app's bundle.
    */
    @available(*, deprecated, message: "Use init(configuration:) instead and handle errors appropriately.")
    convenience init() {
        try! self.init(contentsOf: type(of:self).urlOfModelInThisBundle)
    }

    /**
        Construct a model with configuration

        - parameters:
           - configuration: the desired model configuration

        - throws: an NSError object that describes the problem
    */
    convenience init(configuration: MLModelConfiguration) throws {
        try self.init(contentsOf: type(of:self).urlOfModelInThisBundle, configuration: configuration)
    }

    /**
        Construct detection_v5 instance with explicit path to mlmodelc file
        - parameters:
           - modelURL: the file url of the model

        - throws: an NSError object that describes the problem
    */
    convenience init(contentsOf modelURL: URL) throws {
        try self.init(model: MLModel(contentsOf: modelURL))
    }

    /**
        Construct a model with URL of the .mlmodelc directory and configuration

        - parameters:
           - modelURL: the file url of the model
           - configuration: the desired model configuration

        - throws: an NSError object that describes the problem
    */
    convenience init(contentsOf modelURL: URL, configuration: MLModelConfiguration) throws {
        try self.init(model: MLModel(contentsOf: modelURL, configuration: configuration))
    }

    /**
        Construct detection_v5 instance asynchronously with optional configuration.

        Model loading may take time when the model content is not immediately available (e.g. encrypted model). Use this factory method especially when the caller is on the main thread.

        - parameters:
          - configuration: the desired model configuration
          - handler: the completion handler to be called when the model loading completes successfully or unsuccessfully
    */
    @available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)
    class func load(configuration: MLModelConfiguration = MLModelConfiguration(), completionHandler handler: @escaping (Swift.Result<detection_v5, Error>) -> Void) {
        return self.load(contentsOf: self.urlOfModelInThisBundle, configuration: configuration, completionHandler: handler)
    }

    /**
        Construct detection_v5 instance asynchronously with optional configuration.

        Model loading may take time when the model content is not immediately available (e.g. encrypted model). Use this factory method especially when the caller is on the main thread.

        - parameters:
          - configuration: the desired model configuration
    */
    @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
    class func load(configuration: MLModelConfiguration = MLModelConfiguration()) async throws -> detection_v5 {
        return try await self.load(contentsOf: self.urlOfModelInThisBundle, configuration: configuration)
    }

    /**
        Construct detection_v5 instance asynchronously with URL of the .mlmodelc directory with optional configuration.

        Model loading may take time when the model content is not immediately available (e.g. encrypted model). Use this factory method especially when the caller is on the main thread.

        - parameters:
          - modelURL: the URL to the model
          - configuration: the desired model configuration
          - handler: the completion handler to be called when the model loading completes successfully or unsuccessfully
    */
    @available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)
    class func load(contentsOf modelURL: URL, configuration: MLModelConfiguration = MLModelConfiguration(), completionHandler handler: @escaping (Swift.Result<detection_v5, Error>) -> Void) {
        MLModel.load(contentsOf: modelURL, configuration: configuration) { result in
            switch result {
            case .failure(let error):
                handler(.failure(error))
            case .success(let model):
                handler(.success(detection_v5(model: model)))
            }
        }
    }

    /**
        Construct detection_v5 instance asynchronously with URL of the .mlmodelc directory with optional configuration.

        Model loading may take time when the model content is not immediately available (e.g. encrypted model). Use this factory method especially when the caller is on the main thread.

        - parameters:
          - modelURL: the URL to the model
          - configuration: the desired model configuration
    */
    @available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
    class func load(contentsOf modelURL: URL, configuration: MLModelConfiguration = MLModelConfiguration()) async throws -> detection_v5 {
        let model = try await MLModel.load(contentsOf: modelURL, configuration: configuration)
        return detection_v5(model: model)
    }

    /**
        Make a prediction using the structured interface

        - parameters:
           - input: the input to the prediction as detection_v5Input

        - throws: an NSError object that describes the problem

        - returns: the result of the prediction as detection_v5Output
    */
    func prediction(input: detection_v5Input) throws -> detection_v5Output {
        return try self.prediction(input: input, options: MLPredictionOptions())
    }

    /**
        Make a prediction using the structured interface

        - parameters:
           - input: the input to the prediction as detection_v5Input
           - options: prediction options 

        - throws: an NSError object that describes the problem

        - returns: the result of the prediction as detection_v5Output
    */
    func prediction(input: detection_v5Input, options: MLPredictionOptions) throws -> detection_v5Output {
        let outFeatures = try model.prediction(from: input, options:options)
        return detection_v5Output(features: outFeatures)
    }

    /**
        Make a prediction using the convenience interface

        - parameters:
            - input as color (kCVPixelFormatType_32BGRA) image buffer, 256 pixels wide by 128 pixels high

        - throws: an NSError object that describes the problem

        - returns: the result of the prediction as detection_v5Output
    */
    func prediction(input: CVPixelBuffer) throws -> detection_v5Output {
        let input_ = detection_v5Input(input: input)
        return try self.prediction(input: input_)
    }

    /**
        Make a batch prediction using the structured interface

        - parameters:
           - inputs: the inputs to the prediction as [detection_v5Input]
           - options: prediction options 

        - throws: an NSError object that describes the problem

        - returns: the result of the prediction as [detection_v5Output]
    */
    func predictions(inputs: [detection_v5Input], options: MLPredictionOptions = MLPredictionOptions()) throws -> [detection_v5Output] {
        let batchIn = MLArrayBatchProvider(array: inputs)
        let batchOut = try model.predictions(from: batchIn, options: options)
        var results : [detection_v5Output] = []
        results.reserveCapacity(inputs.count)
        for i in 0..<batchOut.count {
            let outProvider = batchOut.features(at: i)
            let result =  detection_v5Output(features: outProvider)
            results.append(result)
        }
        return results
    }
}

 

This is needed to rotate the rectangle.

Additional

No response

[glenn-jocher]

@heyapps yes, this is based on https://github.com/hukaixuan19970627/yolov5_obb, which seems like a good addition to YOLOv5 :)

iOS integration is another story. The models will probably export to CoreML correctly but will naturally need some custom handling of the outputs and may or may not be able utilize CoreML's NMS module.

[ghost]

@glenn-jocher is it possible to implement something like this with yolov5?

https://stackoverflow.com/questions/74672499/transform-boundingbox-according-to-angle-and-rotation

[glenn-jocher]

@heyapps see https://github.com/hukaixuan19970627/yolov5_obb