Commit ec019c53 authored by Lukas Wolf's avatar Lukas Wolf
Browse files

added optional regularization to models

parent 45e9de9f
......@@ -67,11 +67,13 @@ class Regression_DEEPEYE(Regression_ConvNet):
def _module(self, input_tensor, current_depth):
"""
The module of DeepEye. It starts with a bottleneck of InceptionTime which is followed by different filters with a different kernel size.
The default values are [40,20,10]. In parallel it uses a simple convolution and a separable convolution to make use of 'extrene'
The default values are [40,20,10]. In parallel it uses a simple convolution and a separable convolution to make use of 'extreme'
convolutions as explained in Xception paper.
"""
if int(input_tensor.shape[-1]) > 1:
input_inception = tf.keras.layers.Conv1D(filters=self.bottleneck_size, kernel_size=1, padding='same', use_bias=False)(input_tensor)
input_inception = tf.keras.layers.Conv1D(filters=self.bottleneck_size, kernel_size=1, padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(input_tensor)
else:
input_inception = input_tensor
......@@ -80,19 +82,27 @@ class Regression_DEEPEYE(Regression_ConvNet):
for i in range(len(kernel_size_s)):
conv_list.append(
tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=kernel_size_s[i], padding='same', use_bias=False)(input_inception))
tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=kernel_size_s[i], padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(input_inception))
max_pool_1 = tf.keras.layers.MaxPool1D(pool_size=10, strides=1, padding='same')(input_tensor)
conv_6 = tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=1, padding='same', use_bias=False)(max_pool_1)
conv_6 = tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=1, padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(max_pool_1)
conv_list.append(conv_6)
if self.use_simple_convolution:
max_pool_2 = tf.keras.layers.MaxPool1D(pool_size=10, strides=1, padding='same')(input_tensor)
conv_7 = tf.keras.layers.Conv1D(filters=self.nb_filters / 8, kernel_size=16, padding='same', use_bias=False)(max_pool_2)
conv_7 = tf.keras.layers.Conv1D(filters=self.nb_filters / 8, kernel_size=16, padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(max_pool_2)
conv_list.append(conv_7)
if self.use_separable_convolution:
conv_8 = tf.keras.layers.SeparableConv1D(filters=self.nb_filters, kernel_size=32, padding='same', use_bias=False, depth_multiplier=1)(input_tensor)
conv_8 = tf.keras.layers.SeparableConv1D(filters=self.nb_filters, kernel_size=32, padding='same', use_bias=False, depth_multiplier=1,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(input_tensor)
conv_list.append(conv_8)
x = tf.keras.layers.Concatenate(axis=2)(conv_list)
......
......@@ -14,9 +14,11 @@ class Regression_INCEPTION(Regression_ConvNet):
"""
def __init__(self, input_shape, kernel_size=64, epochs = 50, nb_filters=16, verbose=True, batch_size=64, use_residual=True, depth=12, bottleneck_size=16,
learning_rate=0.01):
learning_rate=0.01, regularization=0.03):
self.bottleneck_size = bottleneck_size
self.regularization = regularization
super(Regression_INCEPTION, self).__init__(input_shape, kernel_size=kernel_size, epochs=epochs, nb_filters=nb_filters,
verbose=verbose, batch_size=batch_size, use_residual=use_residual, depth=depth, learning_rate=learning_rate)
......@@ -31,7 +33,9 @@ class Regression_INCEPTION(Regression_ConvNet):
The outputs of each convolution are concatenated, followed by batch normalization and a ReLu activation.
"""
if int(input_tensor.shape[-1]) > 1:
input_inception = tf.keras.layers.Conv1D(filters=self.bottleneck_size, kernel_size=1, padding='same', use_bias=False)(input_tensor)
input_inception = tf.keras.layers.Conv1D(filters=self.bottleneck_size, kernel_size=1, padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(self.regularization),
activity_regularizer=tf.keras.regularizers.l2(self.regularization))(input_tensor)
else:
input_inception = input_tensor
......@@ -41,10 +45,14 @@ class Regression_INCEPTION(Regression_ConvNet):
for i in range(len(kernel_size_s)):
conv_list.append(
tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=kernel_size_s[i], padding='same', use_bias=False)(input_inception))
tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=kernel_size_s[i], padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(self.regularization),
activity_regularizer=tf.keras.regularizers.l2(self.regularization))(input_inception))
max_pool_1 = tf.keras.layers.MaxPool1D(pool_size=3, strides=1, padding='same')(input_tensor)
conv_6 = tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=1, padding='same', use_bias=False)(max_pool_1)
conv_6 = tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=1, padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(self.regularization),
activity_regularizer=tf.keras.regularizers.l2(self.regularization))(max_pool_1)
conv_list.append(conv_6)
x = tf.keras.layers.Concatenate(axis=2)(conv_list)
......
......@@ -21,7 +21,9 @@ class Regression_PyramidalCNN(Regression_ConvNet):
"""
The module of CNN is made of a simple convolution with batch normalization and ReLu activation. Finally, MaxPooling is also used.
"""
x = tf.keras.layers.Conv1D(filters=self.nb_filters*(current_depth + 1), kernel_size=self.kernel_size, padding='same', use_bias=False)(input_tensor)
x = tf.keras.layers.Conv1D(filters=self.nb_filters*(current_depth + 1), kernel_size=self.kernel_size, padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(input_tensor)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(activation='relu')(x)
x = tf.keras.layers.MaxPool1D(pool_size=2, strides=2)(x)
......
......@@ -79,7 +79,9 @@ class Regression_ConvNet(ABC):
pass
def _shortcut_layer(self, input_tensor, out_tensor):
shortcut_y = tf.keras.layers.Conv1D(filters=int(out_tensor.shape[-1]), kernel_size=1, padding='same', use_bias=False)(input_tensor)
shortcut_y = tf.keras.layers.Conv1D(filters=int(out_tensor.shape[-1]), kernel_size=1, padding='same', use_bias=False,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(input_tensor)
shortcut_y = tf.keras.layers.BatchNormalization()(shortcut_y)
x = keras.layers.Add()([shortcut_y, out_tensor])
x = keras.layers.Activation('relu')(x)
......
......@@ -28,7 +28,9 @@ class Regression_XCEPTION(Regression_ConvNet):
"""
The module of Xception. Consists of a separable convolution followed by batch normalization and a ReLu activation function.
"""
x = tf.keras.layers.SeparableConv1D(filters=self.nb_filters, kernel_size=self.kernel_size, padding='same', use_bias=False, depth_multiplier=1)(input_tensor)
x = tf.keras.layers.SeparableConv1D(filters=self.nb_filters, kernel_size=self.kernel_size, padding='same', use_bias=False, depth_multiplier=1,
kernel_regularizer=tf.keras.regularizers.l1(0.01),
activity_regularizer=tf.keras.regularizers.l2(0.01))(input_tensor)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(activation='relu')(x)
return x
......@@ -56,10 +56,10 @@ config['ensemble'] = 1 #number of models in the ensemble method
# Choosing model
#config['model'] = 'cnn'
#config['model'] ='inception'
config['model'] = 'inception'
#config['model'] = 'eegnet'
#config['model'] = 'deepeye'
config['model'] = 'xception'
#config['model'] = 'xception'
#config['model'] = 'pyramidal_cnn'
# Options for classification task, currently not used for regression
......
......@@ -38,26 +38,28 @@ def run(trainX, trainY):
if config['model'] == 'cnn':
reg = Regression_CNN(input_shape=config['cnn']['input_shape'], kernel_size=64, epochs = 50,
nb_filters=16, verbose=True, batch_size=64, use_residual=True, depth=12,
learning_rate=0.001)
learning_rate=0.001, regularization=0.03)
elif config['model'] == 'inception':
reg = Regression_INCEPTION(input_shape=config['inception']['input_shape'], use_residual=True,
kernel_size=64, nb_filters=16, depth=12, bottleneck_size=16, epochs=50,
learning_rate=0.001)
kernel_size=64, nb_filters=32, depth=16, bottleneck_size=32, epochs=50,
learning_rate=0.001, regularization=0.03)
elif config['model'] == 'xception':
reg = Regression_XCEPTION(input_shape=config['inception']['input_shape'], use_residual=True,
kernel_size=64, nb_filters=16, depth=8, epochs=50,
learning_rate=0.001)
kernel_size=64, nb_filters=32, depth=24, epochs=50,
learning_rate=0.001, regularization=0.03)
elif config['model'] == 'deepeye':
reg = Regression_DEEPEYE(input_shape=config['deepeye']['input_shape'], use_residual=True,
kernel_size=64, nb_filters=32, depth=5, epochs=50, preprocessing=False,
kernel_size=64, nb_filters=32, depth=10, epochs=50, preprocessing=False,
use_separable_convolution=True, use_simple_convolution=True,
bottleneck_size=32, learning_rate=0.001)
bottleneck_size=32,
learning_rate=0.001, regularization=0.03)
elif config['model'] == 'pyramidal_cnn':
reg = Regression_PyramidalCNN(input_shape=config['cnn']['input_shape'], epochs=50, learning_rate=0.001)
reg = Regression_PyramidalCNN(input_shape=config['cnn']['input_shape'], epochs=50,
learning_rate=0.001, regularization=0.03)
else:
logging.info('Cannot start the program. Please choose one model in the config.py file')
......
......@@ -3,7 +3,7 @@
#SBATCH --mail-type=ALL # mail configuration: NONE, BEGIN, END, FAIL, REQUEUE, ALL
#SBATCH --output=log/%j.out # where to store the output (%j is the JOBID), subdirectory must exist
#SBATCH --error=log/%j.err # where to store error messages
#SBATCH --gres=gpu:1
#SBATCH --gres=gpu:2
#SBATCH --mem=60G
echo "Running on host: $(hostname)"
......
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
path = '../runs/1615920751_xception_gaze-reg/batches.log'
......@@ -9,8 +10,8 @@ def plot_batches_log_loss(model_name):
Create loss and validation loss plots from the batches.log file
"""
dir = '../runs/'
path = dir + model_name
dir = './runs/' # must be correct relative to caller
path = dir + model_name + 'batches.log'
df = pd.read_csv(path, sep=';')
nparr = df.to_numpy()
epochs = nparr[:, 0]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment