Edge TPU - Tiny YOLO v3

改造版のプログラムです。
yolo_video_tflite-tpu.py
''' Created on 2021/02/25 yolo_video_tflite-tpu.py https://gilberttanner.com/blog/convert-your-tensorflow-object-detection-model-to-tensorflow-lite https://github.com/peace195/tensorflow-lite-YOLOv3/blob/master/tflite_example.py https://github.com/guichristmann/edge-tpu-tiny-yolo @author: nishi ''' import colorsys import os from timeit import default_timer as timer import sys import tensorflow as tf tf_ver = tf.__version__[0] #if tf_ver=='1': # tf.compat.v1.enable_eager_execution() import tflite_runtime.interpreter as tflite from keras import backend as K from yolo3.model import yolo_eval import argparse import time import cv2 #import re from PIL import Image, ImageDraw, ImageFont import numpy as np # guichristmann/edge-tpu-tiny-yolo/utils.py from utils import * import platform EDGETPU_SHARED_LIB = { 'Linux': 'libedgetpu.so.1', 'Darwin': 'libedgetpu.1.dylib', 'Windows': 'edgetpu.dll' }[platform.system()] def draw_boxes(image, boxes, scores, classes, class_names): i = 0 for topleft, botright in boxes: # Detected class cl = int(classes[i]) # This stupid thing below is needed for opencv to use as a color color = tuple(map(int, colors[cl])) # Box coordinates topleft = (int(topleft[0]), int(topleft[1])) botright = (int(botright[0]), int(botright[1])) # Draw box and class cv2.rectangle(image, topleft, botright, color, 2) textpos = (topleft[0]-2, topleft[1] - 3) score = scores[i] * 100 cl_name = class_names[cl] text = f"{cl_name} ({score:.1f}%)" cv2.putText(image, text, textpos, cv2.FONT_HERSHEY_DUPLEX, 0.45, color, 1, cv2.LINE_AA) i += 1 def draw_box_cv(cv2_im, img_orig_shape,boxes, scores, classes, class_names): net_h,net_w=NET_h,NET_w h,w=cv2_im.shape[0],cv2_im.shape[1] ratio = min(net_h / h, net_w / w) net_off_x = (net_w - w * ratio)/2 # net_ space offset x net_off_y = (net_h - h * ratio)/2 # net_ space offset y #for obj in objs: for i,box in enumerate(boxes): #x0, y0, x1, y1 = list(obj.bbox) # x0*net_w -> net_x0 , y0*net_h -> net_y0 x0, y0 = box[0] x1, y1 = box[1] #print("x0=%f , y0=%f , x1=%f , y1=%f" %(x0,y0,x1,y1)) tl_x = int((x0*net_w - net_off_x)/ratio) tl_y = int((y0*net_h - net_off_y)/ratio) br_x = int((x1*net_w - net_off_x)/ratio) br_y = int((y1*net_h - net_off_y)/ratio) score = int(100 * scores[i]) # Detected class cl = int(classes[i]) color1 = tuple(map(int, colors[cl])) #color1 = (0, 255, 0) color2= (255, 0, 0) cl_name = class_names[cl] text = f"{cl_name} ({score:.1f}%)" cv2_im = cv2.rectangle(cv2_im, (tl_x, tl_y), (br_x, br_y), color1, 2) cv2_im = cv2.putText(cv2_im, text, (tl_x-2, tl_y-3), cv2.FONT_HERSHEY_DUPLEX, .45, color1, 1) return cv2_im def make_interpreter(model_file): model_file, *device = model_file.split('@') return tflite.Interpreter( model_path=model_file, experimental_delegates=[ tflite.load_delegate(EDGETPU_SHARED_LIB, {'device': device[0]} if device else {}) ]) def detect_objects(interpreter, input_data,img_orig_shape): input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() interpreter.set_tensor(input_details[0]['index'], input_data) interpreter.invoke() # Retrieve outputs of the network out1 = interpreter.get_tensor(output_details[0]['index']) out2 = interpreter.get_tensor(output_details[1]['index']) # If this is a quantized model, dequantize the outputs #if args.quant: if True: # Dequantize output o1_scale, o1_zero = output_details[0]['quantization'] out1 = (out1.astype(np.float32) - o1_zero) * o1_scale o2_scale, o2_zero = output_details[1]['quantization'] out2 = (out2.astype(np.float32) - o2_zero) * o2_scale # Get boxes from outputs of network if ORG_PROG: _boxes1, _scores1, _classes1 = featuresToBoxes(out1, anchors[[3, 4, 5]], n_classes, net_input_shape, img_orig_shape, threshold) _boxes2, _scores2, _classes2 = featuresToBoxes(out2, anchors[[1, 2, 3]], n_classes, net_input_shape, img_orig_shape, threshold) else: _boxes1, _scores1, _classes1 = featuresToBoxes2(out1, anchors[[3, 4, 5]], n_classes, net_input_shape, threshold) _boxes2, _scores2, _classes2 = featuresToBoxes2(out2, anchors[[1, 2, 3]], n_classes, net_input_shape, threshold) # This is needed to be able to append nicely when the output layers don't # return any boxes if _boxes1.shape[0] == 0: _boxes1 = np.empty([0, 2, 2]) _scores1 = np.empty([0,]) _classes1 = np.empty([0,]) if _boxes2.shape[0] == 0: _boxes2 = np.empty([0, 2, 2]) _scores2 = np.empty([0,]) _classes2 = np.empty([0,]) boxes = np.append(_boxes1, _boxes2, axis=0) scores = np.append(_scores1, _scores2, axis=0) classes = np.append(_classes1, _classes2, axis=0) if len(boxes) > 0: boxes, scores, classes = nms_boxes(boxes, scores, classes) return boxes, scores, classes def camera_stream(net_h,net_w): # Initialize video stream video = cv2.VideoCapture(args.video) time.sleep(1) accum_time = 0 curr_fps = 0 fps = "FPS: ??" prev_time = timer() first_f=True while(video.isOpened()): try: ret, frame = video.read() img_orig_shape = frame.shape if first_f: print('frame.shape=',frame.shape) first_f=False if False: frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame_resized = cv2.resize(frame_rgb, (net_w, net_h)) input_data = np.expand_dims(frame_resized, axis=0) boxes, scores, pred_classes = detect_objects(interpreter, input_data) else: # Crop frame to network input shape img = letterbox_image(frame.copy(), (net_w, net_h)) # Add batch dimension img = np.expand_dims(img, 0) #if not args.quant: # # Normalize image from 0 to 1 # img = np.divide(img, 255.).astype(np.float32) boxes, scores, pred_classes = detect_objects(interpreter, img,img_orig_shape) #draw_image(image, results, labels, image.size) if len(boxes) > 0: if ORG_PROG: draw_boxes(frame, boxes, scores, pred_classes, classes) else: cv2_im = draw_box_cv(frame, img_orig_shape,boxes, scores, pred_classes, classes) curr_time = timer() exec_time = curr_time - prev_time prev_time = curr_time accum_time = accum_time + exec_time curr_fps = curr_fps + 1 if accum_time > 1: accum_time = accum_time - 1 fps = "FPS: " + str(curr_fps) curr_fps = 0 cv2.putText(frame, text=fps, org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.50, color=(255, 0, 0), thickness=2) cv2.imshow("Image", frame) if( cv2.waitKey( 5 ) & 0xFF == ord( 'q' ) ): fps.stop() break except KeyboardInterrupt: break cv2.destroyAllWindows() video.release() time.sleep(2) if __name__ == '__main__': parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-m', '--model', type=str, default='xxx.tflite', required=False, help='File path of .tflite file.') parser.add_argument('-l', '--labels', type=str, default='model_data/voc_classes.txt',required=False, help='File path of labels file.') parser.add_argument('-t', '--threshold', type=float, default=0.4, required=False, help='Score threshold for detected objects.') parser.add_argument('-v', '--video', type=str, default=0, required=False, help='Path to video') parser.add_argument('-e', '--use_edgetpu', action='store_true', default=False, help='Use EdgeTPU') args = parser.parse_args() classes_path='model_data/coco_classes.txt' #class_names = get_class(classes_path) classes = get_classes(classes_path) n_classes = len(classes) anchors_path='E:/local/coral/edge-tpu-tiny-yolo/cfg/tiny_yolo_anchors.txt' anchors = get_anchors(anchors_path) #model_path='tpu/yyy_edgetpu.tflite' #model_path='tpu/yyy2_edgetpu.tflite' #model_path='tpu/yyy3_edgetpu.tflite' model_path='E:/local/coral/edge-tpu-tiny-yolo/models/quant_coco-tiny-v3-relu_edgetpu.tflite' # Generate random colors for each detection colors = np.random.uniform(30, 255, size=(len(classes), 3)) INT8_type = False interpreter = make_interpreter(model_path) interpreter.allocate_tensors() input_details = interpreter.get_input_details() NET_h = input_details[0]['shape'][1] NET_w = input_details[0]['shape'][2] net_input_shape=input_details[0]['shape'] print('>NET_h=',NET_h) print('>NET_w=',NET_w) print('>net_input_shape=',net_input_shape) ORG_PROG=False # Perform inference threshold=0.15 camera_stream(NET_h,NET_w)

あと、オリジナルの utils.py だと問題があるので、改造しました。
utils.py
featuresToBoxes2() を使います。
''' copy from guichristmann/edge-tpu-tiny-yolo utils.py update by nishi 2021.2.25 ''' import numpy as np import os import cv2 # Maximum number of boxes. Only the top scoring ones will be considered. MAX_BOXES = 30 def sigmoid(x): return 1. / (1 + np.exp(-x)) ''' letterbox_image(imagex, size): size : width , height update by nishi 2021.2.25 ''' def letterbox_image(imagex, size): # add by nishi image=cv2.cvtColor(imagex,cv2.COLOR_BGR2RGB) '''resize image with unchanged aspect ratio using padding''' iw, ih = image.shape[0:2][::-1] #print('iw=',iw,',ih=',ih) w, h = size # width , height scale = min(w/iw, h/ih) # test by nishi #print(">>scale = %f"%(scale)) nw = int(iw*scale) nh = int(ih*scale) image = cv2.resize(image, (nw,nh), interpolation=cv2.INTER_CUBIC) new_image = np.zeros((size[1], size[0], 3), np.uint8) new_image.fill(128) dx = (w-nw)//2 dy = (h-nh)//2 new_image[dy:dy+nh, dx:dx+nw,:] = image return new_image ''' orginal dosn't work correctly update by nishi 2021.2.25 ''' def featuresToBoxes(outputs, anchors, n_classes, net_input_shape, img_orig_shape, threshold): grid_shape = outputs.shape[1:3] n_anchors = len(anchors) # Numpy screwaround to get the boxes in reasonable amount of time grid_y = np.tile(np.arange(grid_shape[0]).reshape(-1, 1), grid_shape[0]).reshape(1, grid_shape[0], grid_shape[0], 1).astype(np.float32) grid_x = grid_y.copy().T.reshape(1, grid_shape[0], grid_shape[1], 1).astype(np.float32) outputs = outputs.reshape(1, grid_shape[0], grid_shape[1], n_anchors, -1) _anchors = anchors.reshape(1, 1, 3, 2).astype(np.float32) # Get box parameters from network output and apply transformations bx = (sigmoid(outputs[..., 0]) + grid_x) / grid_shape[0] by = (sigmoid(outputs[..., 1]) + grid_y) / grid_shape[1] # Should these be inverted? bw = np.multiply(_anchors[..., 0] / net_input_shape[1], np.exp(outputs[..., 2])) bh = np.multiply(_anchors[..., 1] / net_input_shape[2], np.exp(outputs[..., 3])) # Get the scores scores = sigmoid(np.expand_dims(outputs[..., 4], -1)) * \ sigmoid(outputs[..., 5:]) scores = scores.reshape(-1, n_classes) # Reshape boxes and scale back to original image size #ratio = net_input_shape[2] / img_orig_shape[1] # changed by nishi ratio = min(net_input_shape[2] / img_orig_shape[1], net_input_shape[1] / img_orig_shape[0]) #print("ratio = %f" % ratio) letterboxed_height = ratio * img_orig_shape[0] scale_h = net_input_shape[1] / letterboxed_height # add by nishi letterboxed_width = ratio * img_orig_shape[1] scale_w = net_input_shape[2] / letterboxed_width #print("scale_h = %f" % scale_h) #print("scale_w = %f" % scale_w) offset_y = (net_input_shape[1] - letterboxed_height) / 2 / net_input_shape[1] # add by nishi offset_x = (net_input_shape[2] - letterboxed_width) / 2 / net_input_shape[2] #bx = bx.flatten() # changed by nishi bx = (bx.flatten() - offset_x) * scale_w by = (by.flatten() - offset_y) * scale_h #bw = bw.flatten() # changed by nishi bw = bw.flatten() * scale_w bh = bh.flatten() * scale_h half_bw = bw / 2. half_bh = bh / 2. tl_x = np.multiply(bx - half_bw, img_orig_shape[1]) tl_y = np.multiply(by - half_bh, img_orig_shape[0]) br_x = np.multiply(bx + half_bw, img_orig_shape[1]) br_y = np.multiply(by + half_bh, img_orig_shape[0]) # Get indices of boxes with score higher than threshold indices = np.argwhere(scores >= threshold) selected_boxes = [] selected_scores = [] for i in indices: i = tuple(i) selected_boxes.append( ((tl_x[i[0]], tl_y[i[0]]), (br_x[i[0]], br_y[i[0]])) ) selected_scores.append(scores[i]) selected_boxes = np.array(selected_boxes) selected_scores = np.array(selected_scores) selected_classes = indices[:, 1] return selected_boxes, selected_scores, selected_classes ''' orginal dosn't work correctly update by nishi 2021.3.2 ''' def featuresToBoxes2(outputs, anchors, n_classes, net_input_shape, threshold): grid_shape = outputs.shape[1:3] n_anchors = len(anchors) # Numpy screwaround to get the boxes in reasonable amount of time grid_y = np.tile(np.arange(grid_shape[0]).reshape(-1, 1), grid_shape[0]).reshape(1, grid_shape[0], grid_shape[0], 1).astype(np.float32) grid_x = grid_y.copy().T.reshape(1, grid_shape[0], grid_shape[1], 1).astype(np.float32) outputs = outputs.reshape(1, grid_shape[0], grid_shape[1], n_anchors, -1) _anchors = anchors.reshape(1, 1, 3, 2).astype(np.float32) # Get box parameters from network output and apply transformations bx = (sigmoid(outputs[..., 0]) + grid_x) / grid_shape[0] by = (sigmoid(outputs[..., 1]) + grid_y) / grid_shape[1] # Should these be inverted? bw = np.multiply(_anchors[..., 0] / net_input_shape[1], np.exp(outputs[..., 2])) bh = np.multiply(_anchors[..., 1] / net_input_shape[2], np.exp(outputs[..., 3])) # Get the scores scores = sigmoid(np.expand_dims(outputs[..., 4], -1)) * \ sigmoid(outputs[..., 5:]) scores = scores.reshape(-1, n_classes) bx = bx.flatten() by = by.flatten() bw = bw.flatten() bh = bh.flatten() half_bw = bw / 2. half_bh = bh / 2. tl_x = bx - half_bw tl_y = by - half_bh br_x = bx + half_bw br_y = by + half_bh # Get indices of boxes with score higher than threshold indices = np.argwhere(scores >= threshold) selected_boxes = [] selected_scores = [] for i in indices: i = tuple(i) selected_boxes.append( ((tl_x[i[0]], tl_y[i[0]]), (br_x[i[0]], br_y[i[0]])) ) selected_scores.append(scores[i]) selected_boxes = np.array(selected_boxes) selected_scores = np.array(selected_scores) selected_classes = indices[:, 1] return selected_boxes, selected_scores, selected_classes def get_anchors(path): anchors_path = os.path.expanduser(path) with open(anchors_path) as f: anchors = f.readline() anchors = [float(x) for x in anchors.split(',')] return np.array(anchors).reshape(-1, 2) def get_classes(path): classes_path = os.path.expanduser(path) with open(classes_path) as f: classes = [line.strip('\n') for line in f.readlines()] return classes def nms_boxes(boxes, scores, classes): present_classes = np.unique(classes) assert(boxes.shape[0] == scores.shape[0]) assert(boxes.shape[0] == classes.shape[0]) # Sort based on score indices = np.arange(len(scores)) scores, sorted_is = (list(l) for l in zip(*sorted(zip(scores, indices), reverse=True))) boxes = list(boxes[sorted_is]) classes = list(classes[sorted_is]) # Run nms for each class i = 0 while True: if len(boxes) == 1 or i >= len(boxes) or i == MAX_BOXES: break # Get box with highest score best_box = boxes[i] best_cl = classes[i] # Iterate over other boxes to_remove = [] for j in range(i+1, len(boxes)): other_cl = classes[j] #if other_cl != best_cl: # continue other_box = boxes[j] box_iou = iou(best_box, other_box) if box_iou > 0.15: to_remove.append(j) if len(to_remove) == 0: break else: # Remove boxes for r in to_remove[::-1]: del boxes[r] del scores[r] del classes[r] i += 1 return boxes[:MAX_BOXES], scores[:MAX_BOXES], classes[:MAX_BOXES] def iou(box1, box2): xi1 = max(box1[0][0], box2[0][0]) yi1 = max(box1[0][1], box2[0][1]) xi2 = min(box1[1][0], box2[1][0]) yi2 = min(box1[1][1], box2[1][1]) inter_area = (xi2 - xi1)*(yi2 - yi1) # Formula: Union(A,B) = A + B - Inter(A,B) box1_area = (box1[1][1] - box1[0][1])*(box1[1][0]- box1[0][0]) box2_area = (box2[1][1] - box2[0][1])*(box2[1][0]- box2[0][0]) union_area = (box1_area + box2_area) - inter_area # compute the IoU IoU = inter_area / union_area return IoU