Tensorflow1.15 pointnet sem_seg #2

1. 早速修正した、コードを掲載します。
sem_seg/batch_inference_nishi.py
import argparse import os import sys BASE_DIR = os.path.dirname(os.path.abspath(__file__)) ROOT_DIR = os.path.dirname(BASE_DIR) sys.path.append(BASE_DIR) sys.path.append(ROOT_DIR) sys.path.append(os.path.join(ROOT_DIR, 'utils')) from model import * import indoor3d_util # add by nishi import provider #import open3d as o3d parser = argparse.ArgumentParser() parser.add_argument('--gpu', type=int, default=0, help='GPU to use [default: GPU 0]') parser.add_argument('--batch_size', type=int, default=1, help='Batch Size during training [default: 1]') parser.add_argument('--num_point', type=int, default=4096, help='Point number [default: 4096]') #parser.add_argument('--model_path', required=True, help='model checkpoint file path') parser.add_argument('--model_path', default='trained_models-back1/eval_acc_model-21.ckpt', help='model checkpoint file path') #parser.add_argument('--dump_dir', required=True, help='dump folder path') parser.add_argument('--dump_dir', default='log6/dump', help='dump folder path') #parser.add_argument('--output_filelist', required=True, help='TXT filename, filelist, each line is an output for a room') parser.add_argument('--output_filelist', default='log6/output_filelist.txt', help='TXT filename, filelist, each line is an output for a room') #parser.add_argument('--room_data_filelist', required=True, help='TXT filename, filelist, each line is a test room data label file.') parser.add_argument('--room_data_filelist', default='meta/area6_data_label.txt', help='TXT filename, filelist, each line is a test room data label file.') parser.add_argument('--no_clutter', action='store_true', help='If true, donot count the clutter class') #parser.add_argument('--visu', action='store_true', help='Whether to output OBJ file for prediction visualization.') parser.add_argument('--visu', default=1, help='Whether to output OBJ file for prediction visualization.') parser.add_argument('--test_area', type=int, default=6, help='Which area to use for test, option: 1-6 [default: 6]') FLAGS = parser.parse_args() BATCH_SIZE = FLAGS.batch_size NUM_POINT = FLAGS.num_point MODEL_PATH = FLAGS.model_path GPU_INDEX = FLAGS.gpu DUMP_DIR = FLAGS.dump_dir if not os.path.exists(DUMP_DIR): #os.mkdir(DUMP_DIR) os.makedirs(DUMP_DIR) data_dir = 'E:/tmp/pointnet/sem_seg/' LOG_FOUT = open(os.path.join(DUMP_DIR, 'log_evaluate.txt'), 'w') LOG_FOUT.write(str(FLAGS)+'\n') if True: data_dir = 'E:/tmp/pointnet/sem_seg/' #hdf5_data_dir = os.path.join(BASE_DIR, './indoor3d_sem_seg_hdf5_data') hdf5_data_dir = data_dir +'indoor3d_sem_seg_hdf5_data' ALL_FILES = provider.getDataFiles(hdf5_data_dir+'/all_files.txt') room_filelist = [line.rstrip() for line in open(hdf5_data_dir+'/room_filelist.txt')] # Load ALL data data_batch_list = [] label_batch_list = [] for h5_filename in ALL_FILES: print('h5_filename=',h5_filename) data_batch, label_batch = provider.loadDataFile(data_dir+h5_filename) data_batch_list.append(data_batch) label_batch_list.append(label_batch) data_batches = np.concatenate(data_batch_list, 0) label_batches = np.concatenate(label_batch_list, 0) print('>data_batches.shape=',data_batches.shape) print('>label_batches.shape=',label_batches.shape) test_area = 'Area_'+str(FLAGS.test_area) train_idxs = [] test_idxs = [] for i,room_name in enumerate(room_filelist): if test_area in room_name: test_idxs.append(i) else: train_idxs.append(i) #train_data = data_batches[train_idxs,...] #train_label = label_batches[train_idxs] test_data = data_batches[test_idxs,...] test_label = label_batches[test_idxs] #print('>train_data.shape=',train_data.shape,' train_label.shape=', train_label.shape) print('>test_data.shape=',test_data.shape,' test_label.shape=', test_label.shape) test_data_cnt=test_data.shape[0] else: #ROOM_PATH_LIST = [os.path.join(ROOT_DIR,line.rstrip()) for line in open(FLAGS.room_data_filelist)] ROOM_PATH_LIST = [os.path.join(data_dir,line.rstrip()) for line in open(FLAGS.room_data_filelist)] ROOM_PATH_LIST = [os.path.join(data_dir,line.rstrip()) for line in open(FLAGS.room_data_filelist)] NUM_CLASSES = 13 #def view_pcd(point_cloud): # pcd = o3d.geometry.PointCloud() # pcd.points = o3d.utility.Vector3dVector(point_cloud[:,:3]) # #pcd.colors = o3d.utility.Vector3dVector(point_cloud[:,3:6]/255) # pcd.colors = o3d.utility.Vector3dVector(point_cloud[:,3:6]) # pcd.normals = o3d.utility.Vector3dVector(point_cloud[:,6:9]) # # o3d.visualization.draw_geometries([pcd]) def log_string(out_str): LOG_FOUT.write(out_str+'\n') LOG_FOUT.flush() print(out_str) def evaluate(): is_training = False with tf.device('/gpu:'+str(GPU_INDEX)): pointclouds_pl, labels_pl = placeholder_inputs(BATCH_SIZE, NUM_POINT) is_training_pl = tf.placeholder(tf.bool, shape=()) # simple model pred = get_model(pointclouds_pl, is_training_pl) loss = get_loss(pred, labels_pl) pred_softmax = tf.nn.softmax(pred) # Add ops to save and restore all the variables. saver = tf.train.Saver() # Create a session config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True #config.log_device_placement = True # changed by nishi config.log_device_placement = False sess = tf.Session(config=config) # Restore variables from disk. saver.restore(sess, MODEL_PATH) log_string("Model restored.") ops = {'pointclouds_pl': pointclouds_pl, 'labels_pl': labels_pl, 'is_training_pl': is_training_pl, 'pred': pred, 'pred_softmax': pred_softmax, 'loss': loss} total_correct = 0 total_seen = 0 test_max=100 fout_out_filelist = open(FLAGS.output_filelist, 'w') for cur_no in range(test_max): out_data_label_filename = str(cur_no) + '_pred.txt' out_data_label_filename = os.path.join(DUMP_DIR, out_data_label_filename) out_gt_label_filename = str(cur_no) + '_gt.txt' out_gt_label_filename = os.path.join(DUMP_DIR, out_gt_label_filename) #print(cur_no, out_data_label_filename) a, b = eval_one_epoch(sess, ops, cur_no,test_data,test_label, out_data_label_filename, out_gt_label_filename) total_correct += a total_seen += b fout_out_filelist.write(out_data_label_filename+'\n') fout_out_filelist.close() log_string('total_seen: %d'% (total_seen)) log_string('all room eval accuracy: %f'% (total_correct / float(total_seen))) def eval_one_epoch(sess, ops, cur_no,test_data,test_label, out_data_label_filename, out_gt_label_filename): error_cnt = 0 is_training = False total_correct = 0 total_seen = 0 loss_sum = 0 total_seen_class = [0 for _ in range(NUM_CLASSES)] total_correct_class = [0 for _ in range(NUM_CLASSES)] if FLAGS.visu == True: fout = open(os.path.join(DUMP_DIR, str(cur_no) +'_pred.obj'), 'w') fout_gt = open(os.path.join(DUMP_DIR, str(cur_no) +'_gt.obj'), 'w') fout_data_label = open(out_data_label_filename, 'w') fout_gt_label = open(out_gt_label_filename, 'w') if False: room_path='' current_data, current_label = indoor3d_util.room2blocks_wrapper_normalized(room_path, NUM_POINT) current_data = current_data[:,0:NUM_POINT,:] current_label = np.squeeze(current_label) # Get room dimension.. data_label = np.load(room_path) data = data_label[:,0:6] max_room_x = max(data[:,0]) max_room_y = max(data[:,1]) max_room_z = max(data[:,2]) else: current_data = test_data current_label = test_label max_room_x=1.0 max_room_y=1.0 max_room_z=1.0 file_size = current_data.shape[0] num_batches = file_size // BATCH_SIZE #print(file_size) for batch_idx in range(cur_no,cur_no+1): start_idx = batch_idx * BATCH_SIZE end_idx = (batch_idx+1) * BATCH_SIZE cur_batch_size = end_idx - start_idx feed_dict = {ops['pointclouds_pl']: current_data[start_idx:end_idx, :, :], ops['labels_pl']: current_label[start_idx:end_idx], ops['is_training_pl']: is_training} loss_val, pred_val = sess.run([ops['loss'], ops['pred_softmax']], feed_dict=feed_dict) if FLAGS.no_clutter: pred_label = np.argmax(pred_val[:,:,0:12], 2) # BxN else: pred_label = np.argmax(pred_val, 2) # BxN p_cloud=current_data[start_idx:end_idx, :, :] #view_pcd(p_cloud[0]) if True: # Save prediction labels to OBJ file for b in range(BATCH_SIZE): pts = current_data[start_idx+b, :, :] l = current_label[start_idx+b,:] pts[:,6] *= max_room_x pts[:,7] *= max_room_y pts[:,8] *= max_room_z pts[:,3:6] *= 255.0 pred = pred_label[b, :] for i in range(NUM_POINT): color = indoor3d_util.g_label2color[pred[i]] color_gt = indoor3d_util.g_label2color[current_label[start_idx+b, i]] if FLAGS.visu: fout.write('v %f %f %f %d %d %d\n' % (pts[i,6], pts[i,7], pts[i,8], color[0], color[1], color[2])) fout_gt.write('v %f %f %f %d %d %d\n' % (pts[i,6], pts[i,7], pts[i,8], color_gt[0], color_gt[1], color_gt[2])) fout_data_label.write('%f %f %f %d %d %d %f %d\n' % (pts[i,6], pts[i,7], pts[i,8], pts[i,3], pts[i,4], pts[i,5], pred_val[b,i,pred[i]], pred[i])) fout_gt_label.write('%d\n' % (l[i])) correct = np.sum(pred_label == current_label[start_idx:end_idx,:]) total_correct += correct total_seen += (cur_batch_size*NUM_POINT) loss_sum += (loss_val*BATCH_SIZE) for i in range(start_idx, end_idx): for j in range(NUM_POINT): l = current_label[i, j] total_seen_class[l] += 1 total_correct_class[l] += (pred_label[i-start_idx, j] == l) #log_string('eval mean loss: %f' % (loss_sum / float(total_seen/NUM_POINT))) #log_string('eval accuracy: %f'% (total_correct / float(total_seen))) fout_data_label.close() fout_gt_label.close() if FLAGS.visu== True: fout.close() fout_gt.close() return total_correct, total_seen if __name__=='__main__': with tf.Graph().as_default(): evaluate() LOG_FOUT.close()

実行は、
> cd sem_seg
> python batch_inference_nishi.py

そうすると、
log6/dump ディレクトリーの下に、
?_gt.obj
?_gt.txt
?_pred.obj
?_pred.txt
等ファイルが作成されます。
特に、?_pred.txt に、predict した結果が残されていました。
ポイント毎に
x,y,z,r,g,b,acc,class
が出力されるので、これを、open3d 等で表示してみると確認できます。

2. 表示プログラム
早速、?_gt.obj , ?_pred.obj , ?_pred.txt を、open3d で表示してみました。
sem_seg/view_obj_nishi2.py
''' view_obj_nishi2.py Created on 2021/01/16 https://ichi.pro/python-de-tengun-kara-3-d-messhu-o-seiseisuru-tame-no-5-suteppu-gaido-60176041760954 @author: nishi ''' import open3d as o3d #from cv2 import split import numpy as np def conv_obj2pcd(path): data=[] with open(path) as f: for line in f: toks = line.split() if not toks: continue if toks[0] == 'v': data.append( [ float(v) for v in toks[1:]] ) p_cloud = np.array(data) #print(point_cloud[0:2]) #print(point_cloud.shape) return p_cloud # ?_pred.txt # x,y,z,r,g,b,acc,class def conv_txt2pcd(path): data=[] with open(path) as f: for line in f: toks = line.split() if not toks: continue data.append( [ float(v) for v in toks] ) p_cloud = np.array(data) return p_cloud def dev_class(point_cloud,acc=0.0): c_tbl={} for c in point_cloud: if c[6] < acc: next i=int(c[7]) if i in c_tbl: c_tbl[i].append(c) else: c_tbl[i] =[c] for (k,v) in c_tbl.items(): c_tbl[k] = np.array(v) return c_tbl def count_class(point_cloud): c_tbl={} for c in point_cloud[:,7]: if c in c_tbl: c_tbl[c] +=1 else: c_tbl[c] =1 return c_tbl def view_pcd_cls(cls,p_cloud): print('cls=',int(cls)) data=[] for dt in p_cloud: if dt[7] == cls: data.append(dt) point_cloud = np.array(data) print('point_cloud.shape=',point_cloud.shape) view_pcd(point_cloud,color=1) def view_pcd(point_cloud,color=0): pcd = o3d.geometry.PointCloud() pcd.points = o3d.utility.Vector3dVector(point_cloud[:,:3]) if color==0: pcd.colors = o3d.utility.Vector3dVector(point_cloud[:,3:6]) else: pcd.colors = o3d.utility.Vector3dVector(point_cloud[:,3:6]/255) #pcd.normals = o3d.utility.Vector3dVector(point_cloud[:,6:9]) o3d.visualization.draw_geometries([pcd]) test_id=0 if __name__=='__main__': st=0 for i in range(st,st+10): #path = "test_results/1003_diff.obj" #path = "test_results/1003_gt.obj" #path = "test_results/1003_pred.obj" #path="test_results/"+str(i)+"_gt.obj" #path="test_results/"+str(i)+"_pred.obj" #path="test_results/"+str(i)+"_diff.obj" if test_id==0: if True: path='log6/dump/'+str(i)+'_gt.obj' #path='log6/dump/Area_6_conferenceRoom_1_gt.obj' p_cloud=conv_obj2pcd(path) view_pcd(p_cloud) if True: path='log6/dump/'+str(i)+'_pred.obj' #path='log6/dump/Area_6_conferenceRoom_1_gt.obj' p_cloud=conv_obj2pcd(path) view_pcd(p_cloud) if test_id==3: path='log6/dump/'+str(i)+'_pred.txt' p_cloud = conv_txt2pcd(path) print('p_cloud.shape=',p_cloud.shape) c_tbl = count_class(p_cloud) for cls in c_tbl.keys(): view_pcd_cls(cls,p_cloud) if test_id ==4: path='log6/dump/'+str(i)+'_pred.txt' p_cloud = conv_txt2pcd(path) c_tbl=dev_class(p_cloud,acc=0.6) # 8 <- 'chair' if 8 in c_tbl: j=c_tbl[8] view_pcd(c_tbl[8],color=1)

test_id=4
で実行すると、class=8 -> chair
だけを検出しているポイントだけを確認できます。

3. 感想
1) 結局、全ての判定ポイント(num_point=4096) 毎に、それぞれのクラスを判定して
クラス番号を付与するプログラムと言う事らしいです。

つまり、複数の同じ物があれば、それの全てのポイントに、同じクラス番号が付与される事のようです。

2) 入力は、Depth カメラで撮影した PointCloud に近い画像の様なので、
実際のアプリケーションでも、利用できそうです。

次は、独自データで学習しようとした場合、その学習データは、どうやって用意すれば良いのか?
を考えないといかんぞね。
たとえば、インテルの Depth Camera と、表示用のLCD を、Raspi に繋げて、
電池駆動で持ち運び出来るようにして、
スイッチを押したら、その時の RGB-D と Point Cloud データを、保存するプログラムを、動かして、
屋外に持ち出して撮影する。
と、言う事か?