diff --git a/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/FasterRcnnInception.java b/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/FasterRcnnInception.java new file mode 100644 index 0000000..da72f85 --- /dev/null +++ b/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/FasterRcnnInception.java @@ -0,0 +1,326 @@ +/* + * Copyright 2021 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ======================================================================= + */ + +package org.tensorflow.model.examples.cnn.fastrcnn; +/* + +From the web page this is the output dictionary + +num_detections: a tf.int tensor with only one value, the number of detections [N]. +detection_boxes: a tf.float32 tensor of shape [N, 4] containing bounding box coordinates in the following order: [ymin, xmin, ymax, xmax]. +detection_classes: a tf.int tensor of shape [N] containing detection class index from the label file. +detection_scores: a tf.float32 tensor of shape [N] containing detection scores. +raw_detection_boxes: a tf.float32 tensor of shape [1, M, 4] containing decoded detection boxes without Non-Max suppression. M is the number of raw detections. +raw_detection_scores: a tf.float32 tensor of shape [1, M, 90] and contains class score logits for raw detection boxes. M is the number of raw detections. +detection_anchor_indices: a tf.float32 tensor of shape [N] and contains the anchor indices of the detections after NMS. +detection_multiclass_scores: a tf.float32 tensor of shape [1, N, 90] and contains class score distribution (including background) for detection boxes in the image including background class. + +However using +venv\Scripts\python.exe venv\Lib\site-packages\tensorflow\python\tools\saved_model_cli.py show --dir models\faster_rcnn_inception_resnet_v2_1024x1024 --all +2021-03-19 12:25:37.000143: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library cudart64_110.dll + +MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs: + +signature_def['__saved_model_init_op']: + The given SavedModel SignatureDef contains the following input(s): + The given SavedModel SignatureDef contains the following output(s): + outputs['__saved_model_init_op'] tensor_info: + dtype: DT_INVALID + shape: unknown_rank + name: NoOp + Method name is: + +signature_def['serving_default']: + The given SavedModel SignatureDef contains the following input(s): + inputs['input_tensor'] tensor_info: + dtype: DT_UINT8 + shape: (1, -1, -1, 3) + name: serving_default_input_tensor:0 + The given SavedModel SignatureDef contains the following output(s): + outputs['detection_anchor_indices'] tensor_info: + dtype: DT_FLOAT + shape: (1, 300) + name: StatefulPartitionedCall:0 + outputs['detection_boxes'] tensor_info: + dtype: DT_FLOAT + shape: (1, 300, 4) + name: StatefulPartitionedCall:1 + outputs['detection_classes'] tensor_info: + dtype: DT_FLOAT + shape: (1, 300) + name: StatefulPartitionedCall:2 + outputs['detection_multiclass_scores'] tensor_info: + dtype: DT_FLOAT + shape: (1, 300, 91) + name: StatefulPartitionedCall:3 + outputs['detection_scores'] tensor_info: + dtype: DT_FLOAT + shape: (1, 300) + name: StatefulPartitionedCall:4 + outputs['num_detections'] tensor_info: + dtype: DT_FLOAT + shape: (1) + name: StatefulPartitionedCall:5 + outputs['raw_detection_boxes'] tensor_info: + dtype: DT_FLOAT + shape: (1, 300, 4) + name: StatefulPartitionedCall:6 + outputs['raw_detection_scores'] tensor_info: + dtype: DT_FLOAT + shape: (1, 300, 91) + name: StatefulPartitionedCall:7 + Method name is: tensorflow/serving/predict + +Defined Functions: + Function Name: '__call__' + Option #1 + Callable with: + Argument #1 + input_tensor: TensorSpec(shape=(1, None, None, 3), dtype=tf.uint8, name='input_tensor') + +So it appears there's a discrepancy between the web page and running saved_model_cli as +num_detections: a tf.int tensor with only one value, the number of detections [N]. +but the actual tensor is DT_FLOAT according to saved_model_cli +also the web page states +detection_classes: a tf.int tensor of shape [N] containing detection class index from the label file. +but again the actual tensor is DT_FLOAT according to saved_model_cli. +*/ + + +import org.tensorflow.Graph; +import org.tensorflow.SavedModelBundle; +import org.tensorflow.Session; +import org.tensorflow.Tensor; +import org.tensorflow.ndarray.Shape; +import org.tensorflow.op.Ops; +import org.tensorflow.op.core.Constant; +import org.tensorflow.op.core.Reshape; +import org.tensorflow.op.image.DecodeJpeg; +import org.tensorflow.op.io.ReadFile; +import org.tensorflow.types.TFloat32; +import org.tensorflow.types.TString; +import org.tensorflow.types.TUint8; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + + +/** + * Loads an image using ReadFile and DecodeJpeg and then uses the saved model + * faster_rcnn/inception_resnet_v2_1024x1024/1 to detect objects with a detection score greater than 0.3 + */ + +public class FasterRcnnInception { + + private final static String[] cocoLabels = new String[]{ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "street sign", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "hat", + "backpack", + "umbrella", + "shoe", + "eye glasses", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "plate", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted plant", + "bed", + "mirror", + "dining table", + "window", + "desk", + "toilet", + "door", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "blender", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", + "hair brush" + }; + + public static void main(String[] params) { + + if (params.length != 2) { + throw new IllegalArgumentException("Exactly 2 parameters required !"); + } + //my test image + String outputImagePath = params[1]; + String imagePath = params[0]; + // get path to model folder + String modelPath = "models/faster_rcnn_inception_resnet_v2_1024x1024"; + // load saved model + SavedModelBundle model = SavedModelBundle.load(modelPath, "serve"); + //create a map of the COCO 2017 labels + TreeMap cocoTreeMap = new TreeMap<>(); + float cocoCount = 0; + for (String cocoLabel : cocoLabels) { + cocoTreeMap.put(cocoCount, cocoLabel); + cocoCount++; + } + try (Graph g = new Graph(); Session s = new Session(g)) { + Ops tf = Ops.create(g); + Constant fileName = tf.constant(imagePath); + ReadFile readFile = tf.io.readFile(fileName); + Session.Runner runner = s.runner(); + s.run(tf.init()); + DecodeJpeg.Options options = DecodeJpeg.channels(3L); + DecodeJpeg decodeImage = tf.image.decodeJpeg(readFile.contents(), options); + //fetch image from file + try (TUint8 outputImage = (TUint8) runner.fetch(decodeImage).run().get(0)) { + Shape imageShape = outputImage.shape(); + //dimensions of test image + long[] shapeArray = imageShape.asArray(); + //reshape the tensor to 4D for input to model + Reshape reshape = tf.reshape(tf.constant(outputImage), + tf.array(1, + outputImage.shape().asArray()[0], + outputImage.shape().asArray()[1], + outputImage.shape().asArray()[2] + ) + ); + //fresh runner for reshape + runner = s.runner(); + s.run(tf.init()); + try (TUint8 reshapeTensor = (TUint8) runner.fetch(reshape).run().get(0)) { + Map feedDict = new HashMap<>(); + //The given SavedModel SignatureDef input + feedDict.put("input_tensor", reshapeTensor); + //The given SavedModel MetaGraphDef key + Map outputTensorMap = model.function("serving_default").call(feedDict); + //detection_classes, detectionBoxes etc. are model output names + try (TFloat32 detectionClasses = (TFloat32) outputTensorMap.get("detection_classes"); + TFloat32 detectionBoxes = (TFloat32) outputTensorMap.get("detection_boxes"); + TFloat32 rawDetectionBoxes = (TFloat32) outputTensorMap.get("raw_detection_boxes"); + TFloat32 numDetections = (TFloat32) outputTensorMap.get("num_detections"); + TFloat32 detectionScores = (TFloat32) outputTensorMap.get("detection_scores"); + TFloat32 rawDetectionScores = (TFloat32) outputTensorMap.get("raw_detection_scores"); + TFloat32 detectionAnchorIndices = (TFloat32) outputTensorMap.get("detection_anchor_indices"); + TFloat32 detectionMulticlassScores = (TFloat32) outputTensorMap.get("detection_multiclass_scores")) { + int numDetects = (int) numDetections.getFloat(0); + if (numDetects > 0) { + try { + BufferedImage bufferedImage = ImageIO.read(new File(imagePath)); + Graphics2D graphics2D = bufferedImage.createGraphics(); + //TODO tf.image.combinedNonMaxSuppression + for (int n = 0; n < numDetects; n++) { + //put probability and position in outputMap + float detectionScore = detectionScores.getFloat(0, n); + //only include those classes with detection score greater than 0.3f + if (detectionScore > 0.3f) { + float classVal = detectionClasses.getFloat(0, n); + //TODO tf.image.drawBoundingBoxes + int x1 = (int) (shapeArray[1] * detectionBoxes.getFloat(0, n, 1)); + int y1 = (int) (shapeArray[0] * detectionBoxes.getFloat(0, n, 0)); + int x2 = (int) (shapeArray[1] * detectionBoxes.getFloat(0, n, 3)); + int y2 = (int) (shapeArray[0] * detectionBoxes.getFloat(0, n, 2)); + graphics2D.setPaint(Color.RED); + graphics2D.setStroke(new BasicStroke(5)); + graphics2D.drawRect(x1, y1, x2 - x1, y2 - y1); + graphics2D.setPaint(Color.BLACK); + //add a label with percentage score + graphics2D.drawString(cocoTreeMap.get(classVal - 1) + " " + + (new DecimalFormat("#.##").format(detectionScore * 100)) + + "%", x1, y1); + } + } + //TODO tf.image.encodeJpeg + ImageIO.write(bufferedImage, "jpg", new File(outputImagePath)); + } catch (IOException e) { + System.err.println("Exception with writing image " + e.getMessage()); + } + } + } + } + } + } + } +} diff --git a/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/Readme.md b/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/Readme.md new file mode 100644 index 0000000..adb381b --- /dev/null +++ b/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/Readme.md @@ -0,0 +1,16 @@ +# FasterRcnnInception + +Download the model from https://tfhub.dev/tensorflow/faster_rcnn/inception_resnet_v2_1024x1024/1 + +Unzip then untar the model to a local folder - I've used models/faster_rcnn_inception_resnet_v2_1024x1024. + +Create a testimages folder then add some test images into a testimages folder + +To run the example add the input image and output image as parameters: + +FasterRcnnInception testimages/image2.jpg image2rcnn.jpg + +### Example output +Using the image2.jpg image from https://github.com/tensorflow/models/tree/master/research/object_detection/test_images +![image2rcnn.jpg.](image2rcnn.jpg "Beach") + diff --git a/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/image2rcnn.jpg b/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/image2rcnn.jpg new file mode 100644 index 0000000..03a1b56 Binary files /dev/null and b/tensorflow-examples/src/main/java/org/tensorflow/model/examples/cnn/fastrcnn/image2rcnn.jpg differ