Keras CNN Sound Classify

1. データのダウンロード
→ F:\tmp\ECS-50

2.テストと学習用の クラス、スペクトラムデータを、npz で作成。
プログラムは下記になります。

load-dataset.py
''' sound classify load-dataset.py https://qiita.com/cvusk/items/61cdbce80785eaf28349 ''' import os import random import numpy as np import pandas as pd import librosa import librosa.display import matplotlib.pyplot as plt import seaborn as sn from sklearn import model_selection from sklearn import preprocessing import IPython.display as ipd # define directories #base_dir = "./" base_dir = "F:/tmp/ESC-50" esc_dir = os.path.join(base_dir, "ESC-50-master") meta_file = os.path.join(esc_dir, "meta/esc50.csv") audio_dir = os.path.join(esc_dir, "audio/") # load metadata meta_data = pd.read_csv(meta_file) # get data size data_size = meta_data.shape print(data_size) # arrange target label and its name class_dict = {} for i in range(data_size[0]): if meta_data.loc[i,"target"] not in class_dict.keys(): class_dict[meta_data.loc[i,"target"]] = meta_data.loc[i,"category"] #---------- # load a wave data def load_wave_data(audio_dir, file_name): file_path = os.path.join(audio_dir, file_name) x, fs = librosa.load(file_path, sr=44100) return x,fs # change wave data to mel-stft def calculate_melsp(x, n_fft=1024, hop_length=128): stft = np.abs(librosa.stft(x, n_fft=n_fft, hop_length=hop_length))**2 log_stft = librosa.power_to_db(stft) melsp = librosa.feature.melspectrogram(S=log_stft,n_mels=128) return melsp # display wave in plots def show_wave(x): plt.plot(x) plt.show() # display wave in heatmap def show_melsp(melsp, fs): librosa.display.specshow(melsp, sr=fs) plt.colorbar() plt.show() # example data x, fs = load_wave_data(audio_dir, meta_data.loc[0,"filename"]) melsp = calculate_melsp(x) print("wave size:{0}\nmelsp size:{1}\nsamping rate:{2}".format(x.shape, melsp.shape, fs)) show_wave(x) show_melsp(melsp, fs) #------------ # data augmentation: add white noise def add_white_noise(x, rate=0.002): return x + rate*np.random.randn(len(x)) # data augmentation: shift sound in timeframe def shift_sound(x, rate=2): return np.roll(x, int(len(x)//rate)) # data augmentation: stretch sound def stretch_sound(x, rate=1.1): input_length = len(x) x = librosa.effects.time_stretch(x, rate) if len(x)>input_length: return x[:input_length] else: return np.pad(x, (0, max(0, input_length - len(x))), "constant") #-------------- import librosa import librosa.display # change wave data to mel-stft def calculate_melsp(x, n_fft=1024, hop_length=128): stft = np.abs(librosa.stft(x, n_fft=n_fft, hop_length=hop_length))**2 log_stft = librosa.power_to_db(stft) melsp = librosa.feature.melspectrogram(S=log_stft,n_mels=128) return melsp #----------- # get training dataset and target dataset x = list(meta_data.loc[:,"filename"]) y = list(meta_data.loc[:, "target"]) x_train, x_test, y_train, y_test = model_selection.train_test_split(x, y, test_size=0.25, stratify=y) print("x train:{0}\ny train:{1}\nx test:{2}\ny test:{3}".format(len(x_train), len(y_train), len(x_test), len(y_test))) """output x train:1500 y train:1500 x test:500 y test:500 """ # showing the classes are equally splitted a = np.zeros(50) for c in y_test: a[c] += 1 print(a) """output [10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10. 10.] """ #----------- freq = 128 time = 1723 # save wave data in npz, with augmentation def save_np_data(filename, x, y, aug=None, rates=None): np_data = np.zeros(freq*time*len(x)).reshape(len(x), freq, time) np_targets = np.zeros(len(y)) for i in range(len(y)): _x, fs = load_wave_data(audio_dir, x[i]) if aug is not None: _x = aug(x=_x, rate=rates[i]) _x = calculate_melsp(_x) np_data[i] = _x np_targets[i] = y[i] np.savez(filename, x=np_data, y=np_targets) # save test dataset if not os.path.exists("esc_melsp_test.npz"): save_np_data("esc_melsp_test.npz", x_test, y_test) # save raw training dataset if not os.path.exists("esc_melsp_train_raw.npz"): save_np_data("esc_melsp_train_raw.npz", x_train, y_train) # save training dataset with white noise if not os.path.exists("esc_melsp_train_wn.npz"): rates = np.random.randint(1,50,len(x_train))/10000 save_np_data("esc_melsp_train_wn.npz", x_train, y_train, aug=add_white_noise, rates=rates) # save training dataset with sound shift if not os.path.exists("esc_melsp_train_ss.npz"): rates = np.random.choice(np.arange(2,6),len(y_train)) save_np_data("esc_melsp_train_ss.npz", x_train, y_train, aug=shift_sound, rates=rates) # save training dataset with stretch if not os.path.exists("esc_melsp_train_st.npz"): rates = np.random.choice(np.arange(80,120),len(y_train))/100 save_np_data("esc_melsp_train_st.npz", x_train, y_train, aug=stretch_sound, rates=rates) # save training dataset with combination of white noise and shift or stretch if not os.path.exists("esc_melsp_train_com.npz"): np_data = np.zeros(freq*time*len(x_train)).reshape(len(x_train), freq, time) np_targets = np.zeros(len(y_train)) for i in range(len(y_train)): x, fs = load_wave_data(audio_dir, x_train[i]) x = add_white_noise(x=x, rate=np.random.randint(1,50)/1000) if np.random.choice((True,False)): x = shift_sound(x=x, rate=np.random.choice(np.arange(2,6))) else: x = stretch_sound(x=x, rate=np.random.choice(np.arange(80,120))/100) x = calculate_melsp(x) np_data[i] = x np_targets[i] = y_train[i] np.savez("esc_melsp_train_com.npz", x=np_data, y=np_targets)

3. 学習の実行
プログラムは、かきになります。

train.py
''' sound classify train.py https://qiita.com/cvusk/items/61cdbce80785eaf28349 ''' import os import random import numpy as np #import pandas as pd #import tensorflow as tf #import tensorflow.keras.backend as K import keras from keras.models import Model from keras.layers import Input, Dense, Dropout, Activation from keras.layers import Conv2D, GlobalAveragePooling2D from keras.layers import BatchNormalization, Add from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint from keras.optimizers import Adam import matplotlib.pyplot as plt # https://www.haya-programming.com/entry/2018/12/28/004211 npz_list = ['esc_melsp_test.npz', 'esc_melsp_train_raw.npz', 'esc_melsp_train_wn.npz', 'esc_melsp_train_ss.npz', 'esc_melsp_train_st.npz', 'esc_melsp_train_com.npz' ] classes = 50 test_id=3 log_dir = 'logs/test3/' class MixupGenerator(): def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None): self.X_train = X_train self.y_train = y_train self.batch_size = batch_size self.alpha = alpha self.shuffle = shuffle self.sample_num = len(X_train) self.datagen = datagen def __call__(self): while True: indexes = self.__get_exploration_order() itr_num = int(len(indexes) // (self.batch_size * 2)) for i in range(itr_num): batch_ids = indexes[i * self.batch_size * 2:(i + 1) * self.batch_size * 2] X, y = self.__data_generation(batch_ids) yield X, y def __get_exploration_order(self): indexes = np.arange(self.sample_num) if self.shuffle: np.random.shuffle(indexes) return indexes def __data_generation(self, batch_ids): _, h, w, c = self.X_train.shape _, class_num = self.y_train.shape X1 = self.X_train[batch_ids[:self.batch_size]] X2 = self.X_train[batch_ids[self.batch_size:]] y1 = self.y_train[batch_ids[:self.batch_size]] y2 = self.y_train[batch_ids[self.batch_size:]] l = np.random.beta(self.alpha, self.alpha, self.batch_size) X_l = l.reshape(self.batch_size, 1, 1, 1) y_l = l.reshape(self.batch_size, 1) X = X1 * X_l + X2 * (1 - X_l) y = y1 * y_l + y2 * (1 - y_l) if self.datagen: for i in range(self.batch_size): X[i] = self.datagen.random_transform(X[i]) return X, y def load_npz(path,classes = 50,test=3): npz_dt= np.load(path) #print('list(npz_dt)=',list(npz_dt)) x = npz_dt['x'].astype('float32') # data type convert float63 -> float32 #x = x.astype('float32') x_shape = x.shape # reshape # (-1,128,1723) -> (-1,128,1723,1) x = np.reshape(x, (-1,x_shape[1],x_shape[2],1)) # normalize if test == 2: x_max=x.max() x_min=x.min() if x_min < 0.0 : x_min *= -1.0 x += x_min x_max += x_min #x /= x_max x /= 6.5 #if x_max > 4.0: print('>x_max=',x_max) if test ==3: x /= 4.1 x_max=x.max() x_min=x.min() print('>x_max=',x_max) print('>x_min=',x_min) if test ==4: x_max=x.max() x_min=x.min() print('>x_max=',x_max) print('>x_min=',x_min) y = npz_dt['y'] # redefine target data into one hot vector y = keras.utils.to_categorical(y, classes) return x ,y # load test data x_test,y_test = load_npz(npz_list[0],classes=classes,test=test_id) #print('x_test.max()=',x_test.max()) #print('x_test.min()=',x_test.min()) #print('x_test.dtype',x_test.dtype) #print('y_test.dtype',y_test.dtype) def cba(inputs, filters, kernel_size, strides): x = Conv2D(filters, kernel_size=kernel_size, strides=strides, padding='same')(inputs) x = BatchNormalization()(x) x = Activation("relu")(x) return x # define CNN #print('x_train.shape=',x_train.shape) #x_train.shape= (1500, 128, 1723, 1) print('>x_test.shape=',x_test.shape) print('>x_test.dtype=',x_test.dtype) inputs = Input(shape=(x_test.shape[1:])) x_1 = cba(inputs, filters=32, kernel_size=(1,8), strides=(1,2)) x_1 = cba(x_1, filters=32, kernel_size=(8,1), strides=(2,1)) x_1 = cba(x_1, filters=64, kernel_size=(1,8), strides=(1,2)) x_1 = cba(x_1, filters=64, kernel_size=(8,1), strides=(2,1)) x_2 = cba(inputs, filters=32, kernel_size=(1,16), strides=(1,2)) x_2 = cba(x_2, filters=32, kernel_size=(16,1), strides=(2,1)) x_2 = cba(x_2, filters=64, kernel_size=(1,16), strides=(1,2)) x_2 = cba(x_2, filters=64, kernel_size=(16,1), strides=(2,1)) x_3 = cba(inputs, filters=32, kernel_size=(1,32), strides=(1,2)) x_3 = cba(x_3, filters=32, kernel_size=(32,1), strides=(2,1)) x_3 = cba(x_3, filters=64, kernel_size=(1,32), strides=(1,2)) x_3 = cba(x_3, filters=64, kernel_size=(32,1), strides=(2,1)) x_4 = cba(inputs, filters=32, kernel_size=(1,64), strides=(1,2)) x_4 = cba(x_4, filters=32, kernel_size=(64,1), strides=(2,1)) x_4 = cba(x_4, filters=64, kernel_size=(1,64), strides=(1,2)) x_4 = cba(x_4, filters=64, kernel_size=(64,1), strides=(2,1)) x = Add()([x_1, x_2, x_3, x_4]) x = cba(x, filters=128, kernel_size=(1,16), strides=(1,2)) x = cba(x, filters=128, kernel_size=(16,1), strides=(2,1)) x = GlobalAveragePooling2D()(x) x = Dense(classes)(x) x = Activation("softmax")(x) model = Model(inputs, x) #logging = TensorBoard(log_dir=log_dir,write_graph=True) logging = TensorBoard(log_dir=log_dir) next_epochs=75 epochs=5 batch_size=6 weights_load_path = 'sound_trained_weights-test3-5_3.h5' weights_sav_path = 'sound_trained_weights-test3-6' if True: model.load_weights(weights_load_path, by_name=True) if next_epochs >= 45: # initiate Adam optimizer #opt = Adam(lr=0.00001, decay=1e-6, amsgrad=True) opt=Adam(lr=1e-4) # 0.001 else: opt=Adam(lr=1e-3) # 0.01 # Let's train the model using Adam with amsgrad model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) #model.summary() #for id in range(1,6): for id in range(1,4): print('npz=',npz_list[id]) # load train data x_train,y_train = load_npz(npz_list[id],classes=classes,test=test_id) if False: hist=model.fit(x_train,y_train, #epochs=50, epochs=next_epochs+epochs, initial_epoch=next_epochs, batch_size=batch_size, validation_data=(x_test, y_test), callbacks=[logging]) if True: training_generator = MixupGenerator(x_train, y_train,batch_size=batch_size)() hist=model.fit_generator(generator=training_generator, steps_per_epoch=x_train.shape[0] // batch_size, validation_data=(x_test, y_test), epochs=next_epochs+epochs, initial_epoch=next_epochs, # verbose=1, callbacks=[logging]) sav_p= "%s_%d.h5" % (weights_sav_path, id) print('sav_p=',sav_p) model.save_weights(sav_p) #------------- if False: hist_dic=hist.history #print(hist_dic.keys()) #dict_keys(['val_loss', 'val_accuracy', 'loss', 'accuracy'] acc = hist_dic['accuracy'] val_acc = hist_dic['val_accuracy'] loss = hist_dic['loss'] val_loss = hist_dic['val_loss'] epochs = range(1, len(acc) + 1) # "bo" is for "blue dot" plt.plot(epochs, loss, 'bo', label='Training loss') # b is for "solid blue line" plt.plot(epochs, val_loss, 'b', label='Validation loss') plt.title('Training and validation loss') plt.xlabel('Epochs') plt.ylabel('Loss') plt.legend() plt.show() plt.clf() plt.plot(epochs, acc, 'bo', label='Training acc') # b is for "solid blue line" plt.plot(epochs, val_acc, 'b', label='Validation acc') plt.title('Training and validation accuracy') plt.xlabel('Epochs') plt.ylabel('Accuracy') plt.legend() plt.show() x_train=None y_train=None next_epochs+=epochs

補足
学習用のデータは、
下記の３個を使用しました。残りの２個も使うと、正解率が、かなり悪くなったので、のぞきました。

esc_melsp_train_raw.npz
esc_melsp_train_wn.npz
esc_melsp_train_ss.npz

5エポックずつ学習させました。
音声のスペクトラムデータは、 float32 にして、
+-の信号を、一律、4.1 で割って、-1.0 ～ +1.0 の値に正規化して学習させてみました。

注１） 正規化の方法は、幾つか試しましたが、+- の儘でも学習できるようです。 2010.1.30

75 エポック迄学習させました。
1エポック 約5分 になります。

正解率: 0.8520 になりました。
オリジナルは、 0.82 程行ったようなので、良い結果だと思います。

npz= esc_melsp_train_ss.npz >x_max= 0.59915954 >x_min= -0.9708573 Epoch 71/75 250/250 [==============================] - 320s 1s/step - loss: 0.8098 - accuracy: 0.9380 - val_loss: 0.4958 - val_accuracy: 0.8680 Epoch 72/75 250/250 [==============================] - 318s 1s/step - loss: 0.7647 - accuracy: 0.9300 - val_loss: 0.4849 - val_accuracy: 0.8740 Epoch 73/75 250/250 [==============================] - 318s 1s/step - loss: 0.8152 - accuracy: 0.9313 - val_loss: 0.4924 - val_accuracy: 0.8740 Epoch 74/75 250/250 [==============================] - 319s 1s/step - loss: 0.7961 - accuracy: 0.9293 - val_loss: 0.4856 - val_accuracy: 0.8740 Epoch 75/75 250/250 [==============================] - 319s 1s/step - loss: 0.7780 - accuracy: 0.9333 - val_loss: 0.4957 - val_accuracy: 0.8520