Keras CNN Sound Classify #6

先ず、Tensorflow Lite 向けに学習済モデルを、変換します。

h5toTF.py
# -*- coding:utf-8 -*- ''' Created on 2020/05/27 h5toTF.py https://www.tensorflow.org/lite/convert/python_api? @author: nishi ''' import os import keras from keras.models import Model #from keras.applications import MobileNetV2 from keras.applications.inception_v3 import InceptionV3 import tensorflow as tf #np.random.seed(0) num_calibration_steps=200 def representative_dataset_gen(): for _ in range(num_calibration_steps): # Get sample input data as a numpy array in a method of your choosing. yield [input] test_name='test4f' #test_name='test4h' test_id=4 classes = 50 #input_shape=(128,1379,1) input_shape=(128,1723,1) #inputs = Input(shape=(128,1723,1)) if True: if test_name=='test4h': input_shape=(256,862,1) model = InceptionV3(include_top=True, weights=None, input_tensor=None, input_shape=input_shape, pooling=None, classes=classes) if test_name=='test4f': weights_load_path = 'save/sound_trained_weights-inceptv3-test4f_f16-12_5.h5' if test_name=='test4h': weights_load_path = 'save/sound_trained_weights-inceptv3-test4h3_f32-9_3.h5' #weights_load_path = 'save/sound_trained_weights-inceptv3-test4h4_f32-11_1.h5' print('load',weights_load_path) model.load_weights(weights_load_path, by_name=True) model.save('xxx.h5', include_optimizer=False) # モデルを変換 converter = tf.lite.TFLiteConverter.from_keras_model_file('xxx.h5') # Quantizing models for CPU model size # https://www.tensorflow.org/lite/convert/quantization?hl=ja # https://github.com/tensorflow/tensorflow/issues/31784 # https://www.tensorflow.org/lite/performance/post_training_quantization?hl=ja if True: converter.optimizations = [tf.lite.Optimize.DEFAULT] converter.target_spec.supported_types = [tf.lite.constants.FLOAT16] if False: converter.optimizations = [tf.lite.Optimize.DEFAULT] def representative_dataset_gen(): for _ in range(num_calibration_steps): # Get sample input data as a numpy array in a method of your choosing. yield [input] converter.representative_dataset = representative_dataset_gen converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] converter.inference_input_type = tf.int8 # or tf.uint8 converter.inference_output_type = tf.int8 # or tf.uint8 tflite_model = converter.convert() open("xxx.tflite", "wb").write(tflite_model) def save_tflite(): converter = tf.lite.TFLiteConverter.from_keras_model_file("model.h5",output_arrays=['detection_layer_30/BiasAdd']) tflite_model = converter.convert() open("model.tflite", "wb").write(tflite_model)

次に、Tensorflow Lite での、マイク入力での判定プログラムです

sound-predict3-tflite.py
# -*- coding:utf-8 -*- ''' Created on 2020/01/19 update on 2020/05/26 sound-predict3-tflite.py https://blog.imind.jp/entry/2019/08/11/182853 https://qiita.com/__Attsun__/items/e033d689c336315435b3 https://musicinformationretrieval.com/ipython_audio.html @author: nishi ''' import tensorflow as tf import matplotlib.pyplot as plt import librosa import librosa.display import numpy as np #import matplotlib as mpl import pyaudio #import os np.random.seed(0) test_name='test4f' #test_name='test4h' test_id=4 class_indx={ 0: 'dog', 1: 'rooster', 2: 'pig', 3: 'cow', 4: 'frog', 5: 'cat', 6: 'hen', 7: 'insects', 8: 'sheep', 9: 'crow', 10: 'rain', 11: 'sea_waves', 12: 'crackling_fire', 13: 'crickets', 14: 'chirping_birds', 15: 'water_drops', 16: 'wind', 17: 'pouring_water', 18: 'toilet_flush', 18: 'toilet_flush', 19: 'thunderstorm', 20: 'crying_baby', 21: 'sneezing', 22: 'clapping', 23: 'breathing', 24: 'coughing', 25: 'footsteps', 26: 'laughing', 27: 'brushing_teeth', 28: 'snoring', 29: 'drinking_sipping', 30: 'door_wood_knock', 31: 'mouse_click', 32: 'keyboard_typing', 33: 'door_wood_creaks', 34: 'can_opening', 35: 'washing_machine', 36: 'vacuum_cleaner', 37: 'clock_alarm', 38: 'clock_tick', 39: 'glass_breaking', 40: 'helicopter', 41: 'chainsaw', 42: 'siren', 43: 'car_horn', 44: 'engine', 45: 'train', 46: 'church_bells', 47: 'airplane', 48: 'fireworks', 49: 'hand_saw' } #f_list=['1-137-A-32.wav','1-13572-A-46.wav','1-50455-A-44.wav'] # load a wave data def load_wave_data(audio_dir, file_name): file_path = os.path.join(audio_dir, file_name) x, fs = librosa.load(file_path, sr=44100) return x,fs # change wave data to mel-stft def calculate_melsp_org(x, n_fft=1024, hop_length=128): stft = np.abs(librosa.stft(x, n_fft=n_fft, hop_length=hop_length))**2 log_stft = librosa.power_to_db(stft) melsp = librosa.feature.melspectrogram(S=log_stft,n_mels=128) return melsp # for load-datasep.y 改 2020.4.2 def calculate_melsp1(x, n_fft=1024, hop_length=128,n_mels=128): stft = np.abs(librosa.stft(x, n_fft=n_fft, hop_length=hop_length))**2 #log_stft = librosa.power_to_db(stft) # こちらが、オリジナル log_stft = librosa.power_to_db(stft,ref=0.0) melsp = librosa.feature.melspectrogram(S=log_stft,n_mels=n_mels) #melsp -= melsp.min() melsp /= melsp.max() return melsp # for load-dataset2.py def calculate_melsp2(x, n_fft=1024, hop_length=128,sr=44100): S = librosa.feature.melspectrogram(y=x,sr=sr, n_mels=128,n_fft=n_fft, hop_length=hop_length) db_D = librosa.amplitude_to_db(S, ref=0.0) db_D -= db_D.min() return db_D def load_melsp(dx,test=3): dx=dx[np.newaxis,:,:] #print(dx.shape) #print(dx.dtype) #x = x.astype('float32') # data type convert float63 -> float32 #x = x.astype('float32') dx_shape = dx.shape # reshape # (-1,128,1723) -> (-1,128,1723,1) dx = np.reshape(dx, (-1,dx_shape[1],dx_shape[2],1)) print(dx.shape) # normalize if test == 2: dx_max=dx.max() dx_min=dx.min() print('>dx_max=',dx_max,', dx_min=',dx_min) if dx_min < 0.0 : dx_min *= -1.0 dx += dx_min dx_max += dx_min #dx /= dx_max dx /= 6.5 #if dx_max > 4.0: print('>dx_max=',dx_max) if test==3: dx /= 80.0 #dx /= 4.1 dx_max=dx.max() dx_min=dx.min() print('>dx_max=',dx_max) print('>dx_min=',dx_min) if test ==4: dx_max=dx.max() dx_min=dx.min() print('>dx_max=',dx_max) print('>dx_min=',dx_min) return dx classes = 50 #input_shape=(128,1379,1) input_shape=(128,1723,1) if test_name=='test4h': input_shape=(256,862,1) dx_mx=input_shape[1] #weights_load_path = 'save/sound_trained_weights-inceptv3-test4_f16-37_3.h5' #weights_load_path = 'save/sound_trained_weights-inceptv3-test4b_f16-39_3.h5' #weights_load_path = 'save/sound_trained_weights-inceptv3-test4c_f16-5_6.h5' #weights_load_path = 'save/sound_trained_weights-inceptv3-test4e_f16-6_5.h5' if test_name=='test4f': weights_load_path = 'save/sound_trained_weights-inceptv3-test4f_f16-12_5.h5' #weights_load_path = 'save/sound_trained_weights-inceptv3-test4g2_f16-8_2.h5' if test_name=='test4h': #weights_load_path = 'save/sound_trained_weights-inceptv3-test4h_f32-19_1.h5' #weights_load_path = 'save/sound_trained_weights-inceptv3-test4h2_f32-24_1.h5' weights_load_path = 'save/sound_trained_weights-inceptv3-test4h3_f32-9_3.h5' #weights_load_path = 'save/sound_trained_weights-inceptv3-test4h4_f32-11_1.h5' # 読込 itp = tf.lite.Interpreter(model_path="xxx.tflite") itp.allocate_tensors() # 入出力のdetail確認 input_fs = itp.get_input_details() #=> [{'name': 'conv2d_6_input', 'index': 7, 'shape': array([ 1, 28, 28, 1], dtype=int32), 'dtype': <class 'numpy.float32'>, 'quantization': (0.0, 0)}] output_fs = itp.get_output_details() print('input_fs=',input_fs) #input_fs= [{'name': 'input_1', 'index': 295, 'shape': array([ 1, 128, 1723, 1]), 'dtype': <class 'numpy.float32'>, 'quantization': (0.0, 0)}] print('output_fs=',output_fs) #output_fs= [{'name': 'predictions/Softmax', 'index': 315, 'shape': array([ 1, 50]), 'dtype': <class 'numpy.float32'>, 'quantization': (0.0, 0)}] #sys.exit() #CHUNK=1024 #CHUNK=44100 #CHUNK=2**11 # 2048 RATE=44100 # Sampling Rate CHUNK=int(RATE/5) # 0.2 [sec] CHUNK predict_dt = 5*5 # predict data length -> 5 [sec] #predict_dt = 5*4 # predict data length -> 4 [sec] dbl_off = 2 # double check offset #dbl_off = 0 # double check offset #level_th=1000.0 # predict start Sound Level #level_th=2000.0 # predict start Sound Level #level_th=1500.0 # predict start Sound Level level_th=2300.0 # predict start Sound Level p=pyaudio.PyAudio() stream=p.open(format = pyaudio.paInt16, channels = 1, rate = RATE, frames_per_buffer = CHUNK, input = True, output = False) # inputとoutputを同時にTrueにする smaple_f=0 hop_length=128 n_mels=128 if test_name=='test4h': hop_length=256 n_mels=256 while stream.is_active(): input = stream.read(CHUNK) #output = stream.write(input) if smaple_f==0: #print(len(input)) dx = np.frombuffer(input, dtype='int16').astype('float32') #print(dx.shape) #print(dx.dtype) level_max = dx.max() #print('level=',level_max) if level_max < level_th: continue print('level=',level_max) smaple_f=1 data = [] dcnt=0 data.append(input) dcnt+=1 if dcnt >= predict_dt+dbl_off: print('.',end='') data = b''.join(data) #dx = np.frombuffer(data,dtype="int16").astype('float32')/2**15 if test_name=='test4h': dx = np.frombuffer(data,dtype="int16").astype('float32')/2**15 else: dx = np.frombuffer(data,dtype="int16").astype('float16')/2**15 melsp = calculate_melsp1(dx,hop_length=hop_length,n_mels=n_mels) dt=load_melsp(melsp,test=test_id) if dbl_off >0: # (-1,128,1723) -> (-1,128,1723,1) dx_st=dt.shape[2]-dx_mx #v = model.predict(dt[:,:,:dx_mx,:]) itp.set_tensor(input_fs[0]['index'], dt[:,:,:dx_mx,:]) itp.invoke() v=itp.get_tensor(output_fs[0]['index'])[0] #v2 = model.predict(dt[:,:,dx_st:,:]) itp.set_tensor(input_fs[0]['index'], dt[:,:,dx_st:,:]) itp.invoke() v2=itp.get_tensor(output_fs[0]['index'])[0] i=v.argmax() pred=v.max() s=class_indx[i] print('>',i,':',s,' ',pred) i=v2.argmax() pred=v2.max() s=class_indx[i] print('>',i,':',s,' ',pred) else: #input_data = x_test.astype(np.float32)[i:i+1] itp.set_tensor(input_fs[0]['index'], dt) itp.invoke() v=itp.get_tensor(output_fs[0]['index'])[0] #v = model.predict(dt) i=v.argmax() pred=v.max() s=class_indx[i] print('>',i,':',s,' ',pred) if False: #plt.figure(figsize=(14, 5)) #plt.figure(figsize=(11, 5)) librosa.display.waveplot(dx, sr=RATE) plt.show() if False: X = librosa.stft(dx) Xdb = librosa.amplitude_to_db(abs(X)) #plt.figure(figsize=(14, 5)) librosa.display.specshow(Xdb, sr=RATE, x_axis='time', y_axis='hz') plt.show() if False: plt.figure(figsize=(14, 5)) librosa.display.specshow(melsp, sr=RATE, x_axis="time", y_axis="mel", hop_length=128) plt.colorbar(format='%+2.0f dB') plt.title("sound-predict3") plt.show() smaple_f=0 data=None stream.stop_stream() stream.start_stream() stream.stop_stream() stream.close() p.terminate()

Tensorflow 1.15.0 の Lite は、結構早いみたいです。
この後は、 Rasspbery pi 3b + google-coral で試してみたいです。