Commit 1be0eff2 authored by zpgeng's avatar zpgeng
Browse files

Update the best model selection and early stopping callbacks.

parent 1adf2398
......@@ -46,15 +46,19 @@ def run(trainX, trainY):
# save our trained model
# PATH = '../cifar_net.pth'
# torch.save(net.state_dict(), PATH)
# Newly added lines below
save_logs(hist, config['model_dir'], config['model'], pytorch=True)
save_model_param(net, config['model_dir'], config['model'], pytorch=True)
plot_loss_torch(hist.history['loss'])
plot_loss_torch(hist) # Require debugging
torch.save(net.state_dict(), config['model_dir'] + '/' + config['model'] + '_' + 'model.pth')
# -------------- SEE BELOW -----------------------------------
# Should incoporate the best model function during training
# @Oriel: you can check this link for reference:
# https://discuss.pytorch.org/t/save-the-best-model/30430
def train(trainloader, net, optimizer, criterion, epoch=50):
def train(trainloader, net, optimizer, criterion, nb_epoch=50):
loss=[]
for epoch in range(2): # loop over the dataset multiple times
for epoch in range(nb_epoch): # loop over the dataset multiple times
running_loss = 0.0
loss_values=[]
for i, data in enumerate(trainloader, 0):
......
......@@ -17,8 +17,6 @@ sns.set_style('darkgrid')
def run(trainX, trainY):
if config['split']:
config['model']=config['model']+'_cluster'
classifier = Classifier_DEEPEYE_LSTM(output_directory=config['root_dir'],
input_shape=config['deepeye-lstm']['input_shape'])
hist = classifier.fit(trainX, trainY)
......@@ -198,21 +196,19 @@ class Classifier_DEEPEYE_LSTM:
return model
def fit(self, lstm_x, y):
# Add early stopping and reduced learning rata to mitigate overfitting
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=15,
min_lr=0.0001)
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=50)
csv_logger = CSVLogger(config['batches_log'], append=True, separator=';')
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
if self.batch_size is None:
mini_batch_size = int(min(deepeye_x.shape[0] / 10, 16))
else:
mini_batch_size = self.batch_size
#self.model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(learning_rate=self.lr), metrics=['accuracy'])
self.model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.SGD(learning_rate=self.lr,
momentum=self.momentum, nesterov=True), metrics=['accuracy'])
#self.model.compile(loss='binary_crossentropy',
#optimizer=keras.optimizers.RMSprop(learning_rate=self.lr, momentum=self.momentum), metrics=['accuracy'])
hist = self.model.fit(lstm_x, y, batch_size=mini_batch_size, verbose=1, validation_split=0.25,
epochs=self.nb_epochs, shuffle=True, callbacks=[early_stop, csv_logger])
return hist
self.model.compile(loss='binary_crossentropy',optimizer=keras.optimizers.RMSprop(learning_rate=self.lr,
momentum=self.momentum), metrics=['accuracy'])
hist = self.model.fit(lstm_x, y, batch_size=mini_batch_size, verbose=1, validation_split=0.2,
epochs=self.nb_epochs, callbacks=[early_stop, csv_logger, ckpt])
return hist
\ No newline at end of file
......@@ -14,7 +14,7 @@ def run(trainX, trainY):
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
#save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
class Classifier_Xception:
......@@ -69,5 +69,9 @@ class Classifier_Xception:
def fit(self, xception_x, y):
csv_logger = CSVLogger(config['batches_log'], append=True, separator=';')
hist = self.model.fit(xception_x, y, verbose=1, validation_split=0.2, epochs=35, callbacks=[csv_logger])
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
hist = self.model.fit(xception_x, y, verbose=1, validation_split=0.2, epochs=35,
callbacks=[csv_logger, ckpt, early_stop])
return hist
......@@ -15,19 +15,13 @@ sns.set_style('darkgrid')
def run(trainX, trainY):
if config['split']:
config['model']=config['model']+'_cluster'
classifier = Classifier_DEEPEYE(output_directory=config['root_dir'], input_shape=config['deepeye']['input_shape'])
hist = classifier.fit(deepeye_x=trainX, y=trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
class Classifier_DEEPEYE:
"""
Inputs:
......@@ -49,7 +43,7 @@ class Classifier_DEEPEYE:
def __init__(self, output_directory, input_shape, verbose=False, build=True,
batch_size=64, nb_filters=32, use_residual=True, use_bottleneck=True, depth=6,
kernel_size=41, nb_epochs=1500):
kernel_size=41, nb_epochs=100):
self.output_directory = output_directory
self.nb_filters = nb_filters
......@@ -92,7 +86,6 @@ class Classifier_DEEPEYE:
return model
@staticmethod
def _eeg_preprocessing(input_tensor, F1=8, D=2, kernLength=250, c = None):
"""
Static method since this function does not receive any reference argument from this class.
......@@ -196,5 +189,9 @@ class Classifier_DEEPEYE:
def fit(self, deepeye_x, y):
self.model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
csv_logger = CSVLogger(config['batches_log'], append=True, separator=';')
hist = self.model.fit(deepeye_x, y, verbose=1, validation_split=0.2, epochs=1, callbacks=[csv_logger])
return hist
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
hist = self.model.fit(deepeye_x, y, verbose=1, validation_split=0.2, epochs=self.nb_epochs,
callbacks=[csv_logger, ckpt, early_stop])
return hist
\ No newline at end of file
......@@ -9,18 +9,16 @@ from keras.callbacks import CSVLogger
def run(trainX, trainY):
logging.info("Starting DeepEye3.")
if config['split']:
config['model'] = config['model'] + '_cluster'
classifier = Classifier_DEEPEYE3(input_shape=config['deepeye3']['input_shape'])
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
# save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
class Classifier_DEEPEYE3:
def __init__(self, input_shape, verbose=True, build=True, batch_size=64, nb_filters=32,
use_residual=True, use_bottleneck=True, depth=6, kernel_size=40, nb_epochs=1500):
use_residual=True, use_bottleneck=True, depth=9, kernel_size=40, nb_epochs=100):
self.nb_filters = nb_filters
self.use_residual = use_residual
......@@ -31,6 +29,7 @@ class Classifier_DEEPEYE3:
self.batch_size = batch_size
self.bottleneck_size = 32
self.nb_epochs = nb_epochs
self.residual_jump = 3
self.verbose = verbose
if build:
......@@ -41,53 +40,33 @@ class Classifier_DEEPEYE3:
if self.verbose:
self.model.summary()
def split_model(self, input_shape):
input_layer = tf.keras.layers.Input(input_shape)
output = []
# run inception over the cluster
for c in config['cluster'].keys():
a = [input_shape[0]]
a.append(len(config['cluster'][c]))
input_shape = tuple(a)
output.append(self._build_model(input_shape,
X=tf.transpose(tf.nn.embedding_lookup(tf.transpose(input_layer),
config['cluster'][c]))))
# append the results and perform 1 dense layer with last_channel dimension and the output layer
x = tf.keras.layers.Concatenate()(output)
dense = tf.keras.layers.Dense(32, activation='relu')(x)
output_layer = tf.keras.layers.Dense(1, activation='sigmoid')(dense)
model = tf.keras.models.Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
return model
def _inception_module(self, input_tensor, nb_filters=32, use_bottleneck=True, kernel_size=40, bottleneck_size=32,
stride=1, activation='linear'):
def _inception_module(self, input_tensor, stride=1, activation='linear'):
if use_bottleneck and int(input_tensor.shape[-1]) > 1:
input_inception = tf.keras.layers.Conv1D(filters=bottleneck_size, kernel_size=1, padding='same', activation=activation, use_bias=False)(input_tensor)
if self.use_bottleneck and int(input_tensor.shape[-1]) > 1:
input_inception = tf.keras.layers.Conv1D(filters=self.bottleneck_size, kernel_size=1, padding='same',
activation=activation, use_bias=False)(input_tensor)
else:
input_inception = input_tensor
# kernel_size_s = [3, 5, 8, 11, 17]
kernel_size_s = [kernel_size // (2 ** i) for i in range(2)]
kernel_size_s = [self.kernel_size // (2 ** i) for i in range(2)]
conv_list = []
for i in range(len(kernel_size_s)):
conv_list.append(
tf.keras.layers.Conv1D(filters=nb_filters, kernel_size=kernel_size_s[i], strides=stride, padding='same', activation=activation, use_bias=False)(input_inception))
tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=kernel_size_s[i], strides=stride,
padding='same', activation=activation, use_bias=False)(input_inception))
max_pool_1 = tf.keras.layers.MaxPool1D(pool_size=10, strides=stride, padding='same')(input_tensor)
conv_6 = tf.keras.layers.Conv1D(filters=nb_filters, kernel_size=1, padding='same', activation=activation, use_bias=False)(max_pool_1)
conv_6 = tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=1, padding='same',
activation=activation, use_bias=False)(max_pool_1)
max_pool_2 = tf.keras.layers.MaxPool1D(pool_size=10, strides=stride, padding='same')(input_tensor)
conv_7 = tf.keras.layers.Conv1D(filters=nb_filters/8, kernel_size=16, padding='same', activation=activation, use_bias=False)(max_pool_2)
conv_7 = tf.keras.layers.Conv1D(filters=self.nb_filters / 8, kernel_size=16, padding='same',
activation=activation, use_bias=False)(max_pool_2)
conv_8 = tf.keras.layers.SeparableConv1D(filters=nb_filters, kernel_size=32, padding='same', activation=activation, use_bias=False, depth_multiplier=1)(input_tensor)
conv_8 = tf.keras.layers.SeparableConv1D(filters=self.nb_filters, kernel_size=32, padding='same',
activation=activation, use_bias=False, depth_multiplier=1)(input_tensor)
conv_list.append(conv_6)
conv_list.append(conv_7)
conv_list.append(conv_8)
......@@ -105,8 +84,7 @@ class Classifier_DEEPEYE3:
x = keras.layers.Activation('relu')(x)
return x
def _build_model(self, input_shape, X=[], use_residual=True, depth=9):
def _build_model(self, input_shape, X=[]):
if config['split']:
input_layer = X
else:
......@@ -115,10 +93,10 @@ class Classifier_DEEPEYE3:
x = input_layer
input_res = input_layer
for d in range(depth):
for d in range(self.depth):
x = self._inception_module(x)
if use_residual and d % 3 == 2:
if (self.use_residual) and (d % self.residual_jump == (self.residual_jump - 1)):
x = self._shortcut_layer(input_res, x)
input_res = x
......@@ -130,7 +108,34 @@ class Classifier_DEEPEYE3:
model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
return model
def split_model(self, input_shape):
input_layer = tf.keras.layers.Input(input_shape)
output = []
# run inception over the cluster
for c in config['cluster'].keys():
a = [input_shape[0]]
a.append(len(config['cluster'][c]))
input_shape = tuple(a)
output.append(self._build_model(input_shape,
X=tf.transpose(tf.nn.embedding_lookup(tf.transpose(input_layer),
config['cluster'][c]))))
# append the results and perform 1 dense layer with last_channel dimension and the output layer
x = tf.keras.layers.Concatenate()(output)
dense = tf.keras.layers.Dense(32, activation='relu')(x)
output_layer = tf.keras.layers.Dense(1, activation='sigmoid')(dense)
model = tf.keras.models.Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
return model
def fit(self, inception_x, y):
csv_logger = CSVLogger(config['batches_log'], append=True, separator=';')
hist = self.model.fit(inception_x, y, verbose=1, validation_split=0.2, epochs=1, callbacks=[csv_logger])
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
hist = self.model.fit(inception_x, y, verbose=1, validation_split=0.2,
epochs=self.nb_epochs, callbacks=[csv_logger, early_stop, ckpt])
return hist
......@@ -16,10 +16,6 @@ import numpy as np
import logging
def run(trainX, trainY):
if config['split']:
config['model']=config['model']+'_cluster'
classifier = Classifier_EEGNet(output_directory=config['root_dir'])
hist = classifier.fit(trainX, trainY)
......@@ -27,7 +23,7 @@ def run(trainX, trainY):
plot_acc(hist, config['model_dir'], config['model'], True)
# Newly added lines below
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
#save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
......@@ -112,7 +108,7 @@ class Classifier_EEGNet:
block2 = AveragePooling2D((1, 6))(block2)
block2 = dropoutType(self.dropoutRate)(block2)
flatten = Flatten(name='flatten_'+c)(block2)
flatten = Flatten()(block2)
if config['split']:
return flatten
else:
......@@ -125,5 +121,8 @@ class Classifier_EEGNet:
def fit(self, eegnet_x, y):
self.model.compile(loss='binary_crossentropy', optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
csv_logger = CSVLogger(config['batches_log'], append=True, separator=';')
hist = self.model.fit(eegnet_x, y, verbose=1, validation_split=0.2, epochs=2, callbacks=[csv_logger])
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
hist = self.model.fit(eegnet_x, y, verbose=1, validation_split=0.2, epochs=50, callbacks=[csv_logger, ckpt, early_stop])
return hist
......@@ -8,16 +8,13 @@ from keras.callbacks import CSVLogger
def run(trainX, trainY):
logging.info("Starting InceptionTime.")
if config['split']:
config['model']=config['model']+'_cluster'
classifier = Classifier_INCEPTION(output_directory=config['root_dir'], input_shape=config['inception']['input_shape'])
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
#save_model_param(classifier.model, config['model_dir'], config['model'], pytorch=False)
......@@ -137,5 +134,9 @@ class Classifier_INCEPTION:
def fit(self, inception_x, y):
csv_logger = CSVLogger(config['batches_log'], append=True, separator=';')
hist = self.model.fit(inception_x, y, verbose=1, validation_split=0.2, epochs=1, callbacks=[csv_logger])
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
hist = self.model.fit(xception_x, y, verbose=1, validation_split=0.2, epochs=35,
callbacks=[csv_logger, ckpt, early_stop])
return hist
......@@ -39,6 +39,9 @@ config['model'] = 'deepeye3'
config['downsampled'] = False
config['split'] = False
config['cluster'] = clustering()
if config['split']:
config['model'] = config['model'] + '_cluster'
config['trainX_file'] = 'noweEEG.mat' if config['downsampled'] else 'all_EEGprocuesan.mat'
config['trainY_file'] = 'all_trialinfoprosan.mat'
......
......@@ -92,11 +92,11 @@ def save_logs(hist, output_directory, model, pytorch=False):
# Save the model parameters (newly added without debugging)
def save_model_param(classifier, output_directory, model, pytorch=False):
try:
if pytorch:
torch.save(classifier.state_dict(), output_directory + '/' + model + '_' + 'model.pth')
else:
classifier.save(output_directory + '/' + model + '_' + 'model.h5')
except:
return
\ No newline at end of file
# def save_model_param(classifier, output_directory, model, pytorch=False):
# try:
# if pytorch:
# torch.save(classifier.state_dict(), output_directory + '/' + model + '_' + 'model.pth')
# else:
# classifier.save(output_directory + '/' + model + '_' + 'model.h5')
# except:
# return
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment