Coral Edge TPU USB のサンプルの Object Detection を試す。#4

guichristmann/edge-tpu-tiny-yolo は、入力画像の縮小時に、元画像アスペクト比を維持して縮小しているのに対して、
examples-camera/opencv/ は、単純に入力画像を、入力テンサーサイズ 300x300 に縮小しているので、
元画像アスペクト比が維持されなくなっていました。
なので、微妙に検出ボックスのサイズが大きくなっている気がします。

ですので、今回は、guichristmann/edge-tpu-tiny-yolo の入力画像の縮小と、検出ボックスの描画の処理を取り入れてみました。
注) guichristmann/edge-tpu-tiny-yolo の、ボックスの描画処理もバグがあって、横長の画像は問題ないですが、
縦長の画像ではバグっています。
今回は、そのバグも改修しています。

detect_nishi.py
# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A demo that runs object detection on camera frames using OpenCV. TEST_DATA=../all_models Run face detection model: python3 detect.py \ --model ${TEST_DATA}/mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite Run coco model: python3 detect.py \ --model ${TEST_DATA}/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite \ --labels ${TEST_DATA}/coco_labels.txt """ import argparse import collections import common import cv2 import numpy as np import os from PIL import Image import re import tflite_runtime.interpreter as tflite # add by nishi from timeit import default_timer as timer Object = collections.namedtuple('Object', ['id', 'score', 'bbox']) def load_labels(path): p = re.compile(r'\s*(\d+)(.+)') with open(path, 'r', encoding='utf-8') as f: lines = (p.match(line).groups() for line in f.readlines()) return {int(num): text.strip() for num, text in lines} class BBox(collections.namedtuple('BBox', ['xmin', 'ymin', 'xmax', 'ymax'])): """Bounding box. Represents a rectangle which sides are either vertical or horizontal, parallel to the x or y axis. """ __slots__ = () def get_output(interpreter, score_threshold, top_k, image_scale=1.0): """Returns list of detected objects.""" boxes = common.output_tensor(interpreter, 0) class_ids = common.output_tensor(interpreter, 1) scores = common.output_tensor(interpreter, 2) count = int(common.output_tensor(interpreter, 3)) def make(i): ymin, xmin, ymax, xmax = boxes[i] return Object( id=int(class_ids[i]), score=scores[i], bbox=BBox(xmin=np.maximum(0.0, xmin), ymin=np.maximum(0.0, ymin), xmax=np.minimum(1.0, xmax), ymax=np.minimum(1.0, ymax))) return [make(i) for i in range(top_k) if scores[i] >= score_threshold] img = cv2.imread('image1.jpg') print('img.shape=',img.shape) #--- copy utils.py ---- ''' letterbox_image(imagex, size): size : width , height update by nishi 2021.2.25 ''' def letterbox_image(imagex, size): # add by nishi image=cv2.cvtColor(imagex,cv2.COLOR_BGR2RGB) '''resize image with unchanged aspect ratio using padding''' iw, ih = image.shape[0:2][::-1] w, h = size # width , height scale = min(w/iw, h/ih) # test by nishi #print(">>scale = %f"%(scale)) nw = int(iw*scale) nh = int(ih*scale) image = cv2.resize(image, (nw,nh), interpolation=cv2.INTER_CUBIC) new_image = np.zeros((size[1], size[0], 3), np.uint8) new_image.fill(128) dx = (w-nw)//2 dy = (h-nh)//2 new_image[dy:dy+nh, dx:dx+nw,:] = image return new_image #--- def append_objs_to_img(cv2_im, objs, labels): height, width, channels = cv2_im.shape for obj in objs: x0, y0, x1, y1 = list(obj.bbox) x0, y0, x1, y1 = int(x0*width), int(y0*height), int(x1*width), int(y1*height) percent = int(100 * obj.score) label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) return cv2_im def draw_box_cv(cv2_im,img_orig_shape, objs, labels): net_h,net_w=NET_h,NET_w h,w=cv2_im.shape[0],cv2_im.shape[1] ratio = min(net_h / h, net_w / w) net_off_x = (net_w - w * ratio)/2 # net_ space offset x net_off_y = (net_h - h * ratio)/2 # net_ space offset y for obj in objs: x0, y0, x1, y1 = list(obj.bbox) # x0*net_w -> net_x0 , y0*net_h -> net_y0 #print("x0=%f , y0=%f , x1=%f , y1=%f" %(x0,y0,x1,y1)) tl_x = int((x0*net_w - net_off_x)/ratio) tl_y = int((y0*net_h - net_off_y)/ratio) br_x = int((x1*net_w - net_off_x)/ratio) br_y = int((y1*net_h - net_off_y)/ratio) percent = int(100 * obj.score) label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) cv2_im = cv2.rectangle(cv2_im, (tl_x, tl_y), (br_x, br_y), (0, 255, 0), 2) cv2_im = cv2.putText(cv2_im, label, (tl_x, tl_y+30), cv2.FONT_HERSHEY_SIMPLEX, .7, (255, 0, 0), 2) return cv2_im def one_shot(): import sys img = cv2.imread('image1.jpg') print('img.shape=',img.shape) cv2_im = img if True: height=NET_h width=NET_w img_orig_shape = img.shape #img_orig_shape = (img.shape[1],img.shape[0]) # Crop frame to network input shape image_data = letterbox_image(img.copy(),(width ,height)) image_data = np.expand_dims(image_data, 0) input_details = interpreter.get_input_details() interpreter.set_tensor(input_details[0]['index'], image_data) if False: cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im_rgb) #def set_input(interpreter, image, resample=Image.NEAREST): """Copies data to input tensor.""" resample=Image.NEAREST _, height, width, channels = interpreter.get_input_details()[0]['shape'] #return width, height, channels pil_image = pil_im.resize((width, height), resample) pil_image.show() sys.exit() if False: cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im_rgb) common.set_input(interpreter, pil_im) interpreter.invoke() objs = get_output(interpreter, score_threshold=args.threshold, top_k=args.top_k) #cv2_im = append_objs_to_img(cv2_im, objs, labels) cv2_im = draw_box_cv(cv2_im,img_orig_shape, objs, labels) width=cv2_im.shape[1] height=cv2_im.shape[0] resized_img = cv2.resize(cv2_im,(int(width*.4), int(height*.4))) cv2.imshow("img", resized_img) cv2.namedWindow("img", cv2.WINDOW_NORMAL) cv2.waitKey(0) def camera_stream(): cap = cv2.VideoCapture(args.camera_idx) # test by nishi for fps accum_time = 0 curr_fps = 0 fps = "FPS: ??" prev_time = timer() first_f=True ORG_PROC=False while cap.isOpened(): ret, cv2_im = cap.read() if not ret: break if first_f: print('cv2_im.shape=',cv2_im.shape) first_f=False #cv2_im = frame if ORG_PROC: cv2_im_rgb = cv2.cvtColor(cv2_im.copy(), cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im_rgb) common.set_input(interpreter, pil_im) else: height=NET_h width=NET_w img_orig_shape = cv2_im.shape # Crop frame to network input shape image_data = letterbox_image(cv2_im.copy(),(width ,height)) image_data = np.expand_dims(image_data, 0) input_details = interpreter.get_input_details() interpreter.set_tensor(input_details[0]['index'], image_data) interpreter.invoke() objs = get_output(interpreter, score_threshold=args.threshold, top_k=args.top_k) if ORG_PROC: cv2_im = append_objs_to_img(cv2_im, objs, labels) else: cv2_im = draw_box_cv(cv2_im,img_orig_shape, objs, labels) # add by nishi for fps start curr_time = timer() exec_time = curr_time - prev_time prev_time = curr_time accum_time = accum_time + exec_time curr_fps = curr_fps + 1 if accum_time > 1: accum_time = accum_time - 1 fps = "FPS: " + str(curr_fps) curr_fps = 0 cv2.putText(cv2_im, text=fps, org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.50, color=(255, 0, 0), thickness=2) cv2.imshow('frame', cv2_im) if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows() if __name__ == '__main__': default_model_dir = '../all_models' default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite' #default_model ='mobilenet_ssd_v1_coco_quant_postprocess_edgetpu.tflite' #default_model ='mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite' default_labels = 'coco_labels.txt' parser = argparse.ArgumentParser() parser.add_argument('--model', help='.tflite model path', default=os.path.join(default_model_dir,default_model)) parser.add_argument('--labels', help='label file path', default=os.path.join(default_model_dir, default_labels)) parser.add_argument('--top_k', type=int, default=3, help='number of categories with highest score to display') parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default = 0) parser.add_argument('--threshold', type=float, default=0.5, help='classifier score threshold') args = parser.parse_args() print('Loading {} with {} labels.'.format(args.model, args.labels)) interpreter = common.make_interpreter(args.model) interpreter.allocate_tensors() if False: input_fs = interpreter.get_input_details() output_fs = interpreter.get_output_details() print('>input_fs=',input_fs) print('>output_fs=',output_fs) if False: print("input_fs[0]['shape']=",interpreter.get_input_details()[0]['shape']) NET_h,NET_w=interpreter.get_input_details()[0]['shape'][1:3] labels = load_labels(args.labels) TEST_ONO_SHOT=False # image file test by nishi if TEST_ONO_SHOT: one_shot() # camera input stream else: camera_stream()

これで、検出ボックスのサイズが、幾分小さく纏まった気がします。
また、検出の正解率が、少しだけ良くなります。

 但しオリジナルの儘でも、入力画像のアスペクト比が、1:1 だと問題は、ありません。
USB カメラで、OpenCV で取り込むと、480:640 とかになるので、その場合は、影響がでます。

オリジナルのままで、USB カメラで自分の顔を判定して試してみたのですが、検出ボックスが、結構大きいので、
これでは、余りに使い物にならないと思っていました。
しかし、今回の改造版であれば、少しは実用に耐え得る気がします。