tensorflowでMASK R-CNNによるSemantic Segmentation

C:\app\tensorflow\models-master\research\object_detection
ディレクトリで作業します。
おんちゃんは、例によって、Mjpeg-streamerを、OpenCV3 で受信できるように組み込んでみました。

object_detection_mask_rcnn_nishi_cam.py
''' object_detection_mask_rcnn_nishi_cam.py tensorflowでMASK R-CNNによるSemantic Segmentation http://whoopsidaisies.hatenablog.com/entry/2018/02/27/184448 Created on 2019/07/01 @author: nishi ''' # coding: utf-8 import os import six.moves.urllib as urllib import sys import tarfile import tensorflow as tf import zipfile import numpy as np from PIL import Image from object_detection.utils import ops as utils_ops #sys.path.append('[modelsのパス]/research/object_detection') #sys.path.append('C:/app/TensorFlow/models-master/research/object_detection') sys.path.append("..") from object_detection.utils import label_map_util from object_detection.utils import visualization_utils as vis_util import cv2 BASE_DIR='object_detection' #MODEL_NAME = 'mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28' MODEL_NAME = 'mask_rcnn_inception_v2_coco_2018_01_28' #MODEL_NAME = 'mask_rcnn_resnet50_atrous_coco_2018_01_28' MODEL_FILE = MODEL_NAME + '.tar.gz' DOWNLOAD_BASE = 'http://download.tensorflow.org/models/object_detection/' ''' モデルファイルのダウンロード Tensorflow detection model zooから物体検出やセマンティックセグメンテーションのトレーニング済みモデルをダウンロード出来る。 とりあえず一番mAPの高いmask_rcnn_inception_resnet_v2_atrous_cocoというモデルを使う。 wget http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz tar -zxvf mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz 実行方法 >activate tensorflow1.10 >set PYTHONPATH=C:\app\TensorFlow\models-master\research;C:\app\TensorFlow\models-master\research\slim;%PYTHONPATH% >cd C:\app\TensorFlow\models-master\research >python object_detection\object_detection_mask_rcnn_nishi_cam.py ''' ''' Download Model 注) 実行ディレクトリにダウンロードされます。後で、object_detection ディレクトリにコピーして。 ''' def model_downloader(): opener = urllib.request.URLopener() opener.retrieve(DOWNLOAD_BASE + MODEL_FILE, MODEL_FILE) tar_file = tarfile.open(MODEL_FILE) for file in tar_file.getmembers(): file_name = os.path.basename(file.name) if 'frozen_inference_graph.pb' in file_name: tar_file.extract(file, os.getcwd()) #model_downloader() #exit() # 学習済モデルの読み込み #PATH_TO_CKPT = 'mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28/frozen_inference_graph.pb' PATH_TO_CKPT = os.path.join(BASE_DIR,MODEL_NAME,'frozen_inference_graph.pb') detection_graph = tf.Graph() with detection_graph.as_default(): od_graph_def = tf.GraphDef() with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: serialized_graph = fid.read() od_graph_def.ParseFromString(serialized_graph) tf.import_graph_def(od_graph_def, name='') # ラベルの読み込み #PATH_TO_LABELS = '[modelsへのパス]/research/object_detection/data/mscoco_label_map.pbtxt' PATH_TO_LABELS = os.path.join(BASE_DIR,'data', 'mscoco_label_map.pbtxt') NUM_CLASSES = 90 label_map = label_map_util.load_labelmap(PATH_TO_LABELS) categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True) category_index = label_map_util.create_category_index(categories) # 画像の読み込みとnumpy配列への変換 def load_image_into_numpy_array(image): (im_width, im_height) = image.size return np.array(image.getdata()).reshape( (im_height, im_width, 3)).astype(np.uint8) if False: #filename = '[画像ファイルのパス]' filename= 'object_detection/test_images/image2.jpg' image = Image.open(filename) image_np = load_image_into_numpy_array(image) ''' USBカメラオープン ''' if False: video = cv2.VideoCapture(0) # Exit if video not opened. if not video.isOpened(): print("Could not open video") sys.exit() ''' Web CAM(Mjpeg-streamer) オープン ''' if True: #URL="http://200.36.58.250/mjpg/video.mjpg?resolution=320x240" URL="http://200.36.58.250/mjpg/video.mjpg?resolution=640x480" video = cv2.VideoCapture(URL) # Exit if video not opened. if not video.isOpened(): print("Could not open Web CAM") sys.exit() # セマンティックセグメンテーションの処理 with detection_graph.as_default(): with tf.Session() as sess: # 入出力用テンソルのハンドルを取得 image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0') tensor_dict = {} tensor_dict['num_detections'] = tf.get_default_graph().get_tensor_by_name('num_detections:0') tensor_dict['detection_boxes'] = tf.get_default_graph().get_tensor_by_name('detection_boxes:0') tensor_dict['detection_scores'] = tf.get_default_graph().get_tensor_by_name('detection_scores:0') tensor_dict['detection_classes'] = tf.get_default_graph().get_tensor_by_name('detection_classes:0') tensor_dict['detection_masks'] = tf.get_default_graph().get_tensor_by_name('detection_masks:0') # バッチ内の最初の画像の結果を取り出す detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0]) detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0]) # 各検出ボックスのマスクを画像全体上のマスクへ変換 real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32) detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1]) detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1]) #for image_path in TEST_IMAGE_PATHS: while True: #image = Image.open(image_path) # Read first frame. ok, image_np = video.read() if not ok: print('Cannot read video file') sys.exit() detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks( detection_masks, detection_boxes, image_np.shape[0], image_np.shape[1]) detection_masks_reframed = tf.cast( tf.greater(detection_masks_reframed, 0.5), tf.uint8) # バッチ分の次元を追加 tensor_dict['detection_masks'] = tf.expand_dims( detection_masks_reframed, 0) # 実行 output_dict = sess.run(tensor_dict, feed_dict={image_tensor: np.expand_dims(image_np, 0)}) # バッチ分の次元の削除と型変換 output_dict['num_detections'] = int(output_dict['num_detections'][0]) output_dict['detection_classes'] = output_dict[ 'detection_classes'][0].astype(np.uint8) output_dict['detection_boxes'] = output_dict['detection_boxes'][0] output_dict['detection_scores'] = output_dict['detection_scores'][0] output_dict['detection_masks'] = output_dict['detection_masks'][0] print('num=',output_dict['num_detections']) # 画像にマスクとバウンディングボックスを書き込んで出力 vis_util.visualize_boxes_and_labels_on_image_array( image_np, output_dict['detection_boxes'], output_dict['detection_classes'], output_dict['detection_scores'], category_index, instance_masks=output_dict.get('detection_masks'), use_normalized_coordinates=True, line_thickness=0) #plt.figure(figsize=IMAGE_SIZE) #plt.imshow(image_np) #plt.show() cv2.imshow('img',image_np) #---------------- # プログラムの終了は、 q を入力してください。 #---------------- if cv2.waitKey(1) & 0xFF == ord('q'): break #Image.fromarray(image_np).save('out.png') video.release() cv2.destroyAllWindows()

一番軽そうなモデルで試してみましたが、結構遅いです。

Pytorch 1.0 Object Detection faster-rcnn の試し
http://www.netosa.com/blog/2019/05/pytorch-10-object-detection-faster-rcnn.html

の方が断然、速いみたい。