[MXNET-116] Inference - ObjectDetector and SSDObjectDetectorExample (a…

…pache#10229)

* Inference - ObjectDetector and SSDObjectDetectorExample

scala-package/examples/scripts/inferexample/objectdetector/get_ssd_data.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+set -e
+
+MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd)
+
+data_path=$MXNET_ROOT/examples/scripts/inferexample/models/resnet50_ssd
+
+image_path=$MXNET_ROOT/examples/scripts/inferexample/images
+
+if [ ! -d "$data_path" ]; then
+  mkdir -p "$data_path"
+fi
+
+if [ ! -d "$image_path" ]; then
+  mkdir -p "$image_path"
+fi
+
+if [ ! -f "$data_path" ]; then
+    wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-symbol.json -P $data_path
+    wget https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-0000.params -P $data_path
+    wget https://raw.githubusercontent.com/awslabs/mxnet-model-server/master/examples/ssd/synset.txt -P $data_path
+    cd $image_path
+    wget https://cloud.githubusercontent.com/assets/3307514/20012566/cbb53c76-a27d-11e6-9aaa-91939c9a1cd5.jpg -O 000001.jpg
+    wget https://cloud.githubusercontent.com/assets/3307514/20012567/cbb60336-a27d-11e6-93ff-cbc3f09f5c9e.jpg -O dog.jpg
+    wget https://cloud.githubusercontent.com/assets/3307514/20012563/cbb41382-a27d-11e6-92a9-18dab4fd1ad3.jpg -O person.jpg
+fi
+

scala-package/examples/scripts/inferexample/objectdetector/run_ssd_example.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+MXNET_ROOT=$(cd "$(dirname $0)/../../../../../"; pwd)
+CLASS_PATH=$MXNET_ROOT/scala-package/assembly/osx-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/*:$MXNET_ROOT/scala-package/infer/target/*
+
+# model dir and prefix
+MODEL_DIR=$1
+# input image
+INPUT_IMG=$2
+# which input image dir
+INPUT_DIR=$3
+
+java -Xmx8G -cp $CLASS_PATH \
+	ml.dmlc.mxnetexamples.inferexample.objectdetector.SSDClassifierExample \
+	--model-path-prefix $MODEL_DIR \
+	--input-image $INPUT_IMG \
+	--input-dir $INPUT_DIR

scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/objectdetector/README.md

@@ -0,0 +1,116 @@
+# Single Shot Multi Object Detection using Scala Inference API
+
+In this example, you will learn how to use Scala Inference API to run Inference on pre-trained Single Shot Multi Object Detection (SSD) MXNet model.
+
+The model is trained on the [Pascal VOC 2012 dataset](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html). The network is a SSD model built on Resnet50 as base network to extract image features. The model is trained to detect the following entities (classes): ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']. For more details about the model, you can refer to the [MXNet SSD example](https://github.com/apache/incubator-mxnet/tree/master/example/ssd).
+
+
+## Contents
+
+1. [Prerequisites](#prerequisites)
+2. [Download artifacts](#download-artifacts)
+3. [Setup datapath and parameters](#setup-datapath-and-parameters)
+4. [Run the image inference example](#run-the-image-inference-example)
+5. [Infer APIs](#infer-api-details)
+6. [Next steps](#next-steps)
+
+
+## Prerequisites
+
+1. MXNet
+2. MXNet Scala Package
+3. [IntelliJ IDE (or alternative IDE) project setup](http://mxnet.incubator.apache.org/tutorials/scala/mxnet_scala_on_intellij.html) with the MXNet Scala Package
+4. wget
+
+
+## Setup Guide
+
+### Download Artifacts
+#### Step 1
+You can download the files using the script `get_ssd_data.sh`. It will download and place the model files in a `model` folder and the test image files in a `image` folder in the current directory.
+From the `scala-package/examples/scripts/inferexample/imageclassifier/` folder run:
+
+```bash
+./get_resnet_data.sh
+```
+
+**Note**: You may need to run `chmod +x get_resnet_data.sh` before running this script.
+
+Alternatively use the following links to download the Symbol and Params files via your browser:
+- [resnet50_ssd_model-symbol.json](https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-symbol.json)
+- [resnet50_ssd_model-0000.params](https://s3.amazonaws.com/model-server/models/resnet50_ssd/resnet50_ssd_model-0000.params)
+- [synset.txt](https://github.com/awslabs/mxnet-model-server/blob/master/examples/ssd/synset.txt)
+
+In the pre-trained model, the `input_name` is `data` and shape is `(1, 3, 512, 512)`.
+This shape translates to: a batch of `1` image, the image has color and uses `3` channels (RGB), and the image has the dimensions of `512` pixels in height by `512` pixels in width.
+
+`image/jpeg` is the expected input type, since this example's image pre-processor only supports the handling of binary JPEG images.
+
+The output shape is `(1, 6132, 6)`. As with the input, the `1` is the number of images. `6132` is the number of prediction results, and `6` is for the size of each prediction. Each prediction contains the following components:
+- `Class`
+- `Accuracy`
+- `Xmin`
+- `Ymin`
+- `Xmax`
+- `Ymax`
+
+
+### Setup Datapath and Parameters
+#### Step 2
+The code `Line 31: val baseDir = System.getProperty("user.dir")` in the example will automatically searches the work directory you have defined. Please put the files in your [work directory](https://stackoverflow.com/questions/16239130/java-user-dir-property-what-exactly-does-it-mean). <!-- how do you define the work directory? -->
+
+Alternatively, if you would like to use your own path, please change line 31 into your own path
+```scala
+val baseDir = <Your Own Path>
+```
+
+The followings is the parameters defined for this example, you can find more information in the `class SSDClassifierExample`.
+
+| Argument                      | Comments                                 |
+| ----------------------------- | ---------------------------------------- |
+| `model-path-prefix`                   | Folder path with prefix to the model (including json, params, and any synset file). |
+| `input-image`                 | The image to run inference on. |
+| `input-dir`                   | The directory of images to run inference on. |
+
+
+## How to Run Inference
+After the previous steps, you should be able to run the code using the following script that will pass all of the required parameters to the Infer API.
+
+From the `scala-package/examples/scripts/inferexample/objectdetector/` folder run:
+
+```bash
+./run_ssd_example.sh ../model/resnet50_ssd_model ../image/dog.jpg ../image
+```
+
+**Notes**:
+* These are relative paths to this script.
+* You may need to run `chmod +x run_ssd_example.sh` before running this script.
+
+The example should give expected output as shown below:
+```
+Class: car
+Probabilties: 0.99847263
+(Coord:,312.21335,72.0291,456.01443,150.66176)
+Class: bicycle
+Probabilties: 0.90473825
+(Coord:,155.95807,149.96362,383.8369,418.94513)
+Class: dog
+Probabilties: 0.8226818
+(Coord:,83.82353,179.13998,206.63783,476.7875)
+```
+the outputs come from the the input image, with top3 predictions picked.
+
+
+## Infer API Details
+This example uses ObjectDetector class provided by MXNet's scala package Infer APIs. It provides methods to load the images, create NDArray out of Java BufferedImage and run prediction using Classifier and Predictor APIs.
+
+
+## References
+This documentation used the model and inference setup guide from the [MXNet Model Server SSD example](https://github.com/awslabs/mxnet-model-server/blob/master/examples/ssd/README.md).
+
+
+## Next Steps
+
+Check out the following related tutorials and examples for the Infer API:
+
+* [Image Classification with the MXNet Scala Infer API](../imageclassifier/README.md)

scala-package/examples/src/main/scala/ml/dmlc/mxnetexamples/inferexample/objectdetector/SSDClassifierExample.scala

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ml.dmlc.mxnetexamples.inferexample.objectdetector
+
+import ml.dmlc.mxnet.{DType, Shape, DataDesc}
+import ml.dmlc.mxnet.infer._
+import org.kohsuke.args4j.{CmdLineParser, Option}
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+import java.nio.file.{Files, Paths}
+
+class SSDClassifierExample {
+  @Option(name = "--model-path-prefix", usage = "the input model directory and prefix of the model")
+  private val modelPathPrefix: String = "/model/ssd_resnet50_512"
+  @Option(name = "--input-image", usage = "the input image")
+  private val inputImagePath: String = "/images/dog.jpg"
+  @Option(name = "--input-dir", usage = "the input batch of images directory")
+  private val inputImageDir: String = "/images/"
+}
+
+object SSDClassifierExample {
+
+  private val logger = LoggerFactory.getLogger(classOf[SSDClassifierExample])
+  private type SSDOut = (String, Array[Float])
+
+  def runObjectDetectionSingle(modelPathPrefix: String, inputImagePath: String):
+  IndexedSeq[IndexedSeq[(String, Array[Float])]] = {
+    val dType = DType.Float32
+    val inputShape = Shape(1, 3, 512, 512)
+    // ssd detections, numpy.array([[id, score, x1, y1, x2, y2]...])
+    val outputShape = Shape(1, 6132, 6)
+    val inputDescriptors = IndexedSeq(DataDesc("data", inputShape, dType, "NCHW"))
+    val img = ImageClassifier.loadImageFromFile(inputImagePath)
+    val objDetector = new ObjectDetector(modelPathPrefix, inputDescriptors)
+    val output = objDetector.imageObjectDetect(img, Some(3))
+
+    output
+  }
+
+  def runObjectDetectionBatch(modelPathPrefix: String, inputImageDir: String):
+  IndexedSeq[IndexedSeq[(String, Array[Float])]] = {
+    val dType = DType.Float32
+    val inputShape = Shape(1, 3, 512, 512)
+    // ssd detections, numpy.array([[id, score, x1, y1, x2, y2]...])
+    val outputShape = Shape(1, 6132, 6)
+    val inputDescriptors = IndexedSeq(DataDesc("data", inputShape, dType, "NCHW"))
+    val imgList = ImageClassifier.loadInputBatch(inputImageDir)
+    val objDetector = new ObjectDetector(modelPathPrefix, inputDescriptors)
+    val outputList = objDetector.imageBatchObjectDetect(imgList, Some(1))
+    outputList
+  }
+
+  def main(args: Array[String]): Unit = {
+    val inst = new SSDClassifierExample
+    val parser : CmdLineParser = new CmdLineParser(inst)
+    parser.parseArgument(args.toList.asJava)
+    val baseDir = System.getProperty("user.dir")
+    val mdprefixDir = baseDir + inst.modelPathPrefix
+    val imgPath = baseDir + inst.inputImagePath
+    val imgDir = baseDir + inst.inputImageDir
+    if (!checkExist(Array(mdprefixDir + "-symbol.json", imgDir, imgPath))) {
+      logger.error("Model or input image path does not exist")
+      sys.exit(1)
+    }
+
+    try {
+      val inputShape = Shape(1, 3, 512, 512)
+      val outputShape = Shape(1, 6132, 6)
+
+      val width = inputShape(2)
+      val height = inputShape(3)
+      var outputStr : String = "\n"
+
+      val output = runObjectDetectionSingle(mdprefixDir, imgPath)
+
+
+      for (ele <- output) {
+        for (i <- ele) {
+          outputStr += "Class: " + i._1 + "\n"
+          val arr = i._2
+          outputStr += "Probabilties: " + arr(0) + "\n"
+          val coord = Array[Float](
+            arr(1) * width, arr(2) * height,
+            arr(3) * width, arr(4) * height
+          )
+          outputStr += "Coord:" + coord.mkString(",") + "\n"
+        }
+      }
+      logger.info(outputStr)
+
+      val outputList = runObjectDetectionBatch(mdprefixDir, imgDir)
+
+      outputStr = "\n"
+      for (idx <- outputList.indices) {
+        outputStr += "*** Image " + (idx + 1) + "***" + "\n"
+        for (i <- outputList(idx)) {
+          outputStr += "Class: " + i._1 + "\n"
+          val arr = i._2
+          outputStr += "Probabilties: " + arr(0) + "\n"
+          val coord = Array[Float](
+            arr(1) * width, arr(2) * height,
+            arr(3) * width, arr(4) * height
+          )
+          outputStr += "Coord:" + coord.mkString(",") + "\n"
+        }
+      }
+      logger.info(outputStr)
+
+    } catch {
+      case ex: Exception => {
+        logger.error(ex.getMessage, ex)
+        parser.printUsage(System.err)
+        sys.exit(1)
+      }
+    }
+    sys.exit(0)
+  }
+
+
+  def checkExist(arr : Array[String]) : Boolean = {
+    var exist : Boolean = true
+    for (item <- arr) {
+      exist = Files.exists(Paths.get(item)) && exist
+      if (!exist) {
+        logger.error("Cannot find: " + item)
+      }
+    }
+    exist
+  }
+
+}