Commit 6298acfe authored by Lukas Wolf's avatar Lukas Wolf
Browse files

added torch models

parent d2ec40d2
......@@ -34,4 +34,5 @@ class Trainer:
else:
raise Exception("Choose a valid deep learning framework")
logging.info("Trainer: finished training")
\ No newline at end of file
logging.info("Trainer: finished training")
logging.info("------------------------------------------------------------------------------------")
......@@ -41,8 +41,8 @@ config['root_dir'] = '.'
##################################################################
#config['task'] = 'prosaccade-clf'
#config['task'] = 'gaze-reg'
config['task'] = 'angle-reg'
config['task'] = 'gaze-reg'
#config['task'] = 'angle-reg'
if config['task'] != 'prosaccade-clf':
#config['dataset'] = 'processing_speed_task'
......@@ -59,7 +59,7 @@ config['framework'] = 'torch'
# Choose model
##################################################################
config['ensemble'] = 3 #number of models in the ensemble
config['ensemble'] = 2 #number of models in the ensemble
config['pretrained'] = False # We can use a model pretrained on processing speed task
config['model'] = 'cnn'
......@@ -76,7 +76,7 @@ config['model'] = 'cnn'
##################################################################
config['learning_rate'] = 1e-3 # fix only: 1e-2, sac only: 1e-3, sac_fix: 1e-3 , fix_sac_fix: 1e-4, for inception on angle 1e-5
config['regularization'] = 0 # fix only: 1e-3, sac only: 1e-2, sac_fix: 1, fix_sac_fix: 5, for inception on angle 0
config['epochs'] = 10
config['epochs'] = 2
config['batch_size'] = 64
##################################################################
......
......@@ -105,7 +105,7 @@ class Ensemble_tf:
config['model'] = config['model'] + '_cluster'
hist.history['val_loss'] = loss
np.savetxt('ensemble_loss.csv', loss, delimiter=',')
np.savetxt(X=loss, fname=config['model_dir']+'/'+'ensemble_loss.csv', delimiter=',')
plot_loss(hist, config['model_dir'], config['model'], val = True)
if config['task'] == 'prosaccade-clf':
hist.history['val_accuracy'] = accuracy
......
......@@ -9,15 +9,13 @@ from torch_models.torch_utils.training import train_loop, test_loop
class Prediction_history:
"""
Collect predictions of the given validation set after each epoch
predhis is a list of lists (one for each epoch) of tensors (one for each batch)
"""
def __init__(self, dataloader) -> None:
self.dataloader = dataloader
self.predhis = []
def on_epoch_end(self, model):
"""
When epoch ends predict the validation set and store it in predhis=[ypred_epoch_1, ypred_epoch_2,...]
"""
y_pred = []
for x, y in self.dataloader:
y_pred.append(model(x.float()))
......@@ -92,7 +90,7 @@ class BaseNet(nn.Module):
# Train the model
epochs = config['epochs']
for t in range(epochs):
logging.info(f"Epoch {t+1}\n-------------------------------")
print(f"Epoch {t+1}\n-------------------------------")
train_loop(train_dataloader, self.float(), self.loss_fn, optimizer)
test_loop(test_dataloader, self.float(), self.loss_fn)
prediction_ensemble.on_epoch_end(model=self)
......
......@@ -28,7 +28,6 @@ class ConvNet(ABC, BaseNet):
self.kernel_size = kernel_size
self.nb_filters = nb_filters
self.preprocessing = preprocessing
#self.nb_features = nb_features
# Define all the convolutional and shortcut modules that we will need in the model
self.conv_blocks = nn.ModuleList([self._module(d) for d in range(self.depth)])
......@@ -36,7 +35,7 @@ class ConvNet(ABC, BaseNet):
self.gap_layer = nn.AvgPool1d(kernel_size=2, stride=1)
self.gap_layer_pad = Pad_Pool(left=0, right=1, value=0)
#logging.info('Parameters of {}, model number {}: '.format(self, model_number))
logging.info('Parameters of model number {}: '.format(self.model_number))
logging.info('--------------- use residual : ' + str(self.use_residual))
logging.info('--------------- depth : ' + str(self.depth))
logging.info('--------------- batch size : ' + str(self.batch_size))
......@@ -98,6 +97,7 @@ class ConvNet(ABC, BaseNet):
def get_nb_features_output_layer(self):
"""
Return number of features passed into the output layer of the network
nb.features has to be defined in a model implementing ConvNet
"""
return self.nb_features * self.timesamples
......
import numpy as np
from sklearn.metrics import roc_auc_score, precision_score, recall_score, accuracy_score
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import torch.nn.functional as F
import torch.optim as optim
from torch_models import BaseNetTorch
from torch_models.BaseNetTorch import BaseNet
from config import config
class EEGNet(BaseNetTorch):
class EEGNet(BaseNet):
def __init__(self, input_shape, epochs=50, model_number=0):
super().__init__(input_shape=input_shape, epochs=epochs, model_number=model_number)
......@@ -33,11 +28,6 @@ class EEGNet(BaseNetTorch):
self.conv3 = nn.Conv2d(4, 4, (8, 4))
self.batchnorm3 = nn.BatchNorm2d(4, False)
self.pooling3 = nn.MaxPool2d((2, 4))
# FC Layer
#self.fc1 = nn.Linear(4*2*7, 1)
# NOTE: above output layer is defined in BaseNet as output_layer
def forward(self, x):
# Layer 1
......@@ -62,43 +52,11 @@ class EEGNet(BaseNetTorch):
# FC Layer
x = x.view(-1, 4*2*7)
#x = F.sigmoid(self.fc1(x))
x = F.sigmoid(self.output_layer(x))
x = self.output_layer(x)
return x
def evaluate(model, X, Y, params = ["acc"]):
results = []
batch_size = 100
predicted = []
for i in range(len(X)/batch_size):
s = i*batch_size
e = i*batch_size+batch_size
inputs = Variable(torch.from_numpy(X[s:e]).cuda(0))
pred = model(inputs)
predicted.append(pred.data.cpu().numpy())
inputs = Variable(torch.from_numpy(X).cuda(0))
predicted = model(inputs)
predicted = predicted.data.cpu().numpy()
for param in params:
if param == 'acc':
results.append(accuracy_score(Y, np.round(predicted)))
if param == "auc":
results.append(roc_auc_score(Y, predicted))
if param == "recall":
results.append(recall_score(Y, np.round(predicted)))
if param == "precision":
results.append(precision_score(Y, np.round(predicted)))
if param == "fmeasure":
precision = precision_score(Y, np.round(predicted))
recall = recall_score(Y, np.round(predicted))
results.append(2*precision*recall/ (precision+recall))
return results
\ No newline at end of file
def get_nb_features_output_layer(self):
"""
Return number of features passed into the output layer of the network
"""
return 4 * 2 * 7
\ No newline at end of file
......@@ -14,7 +14,9 @@ from torch_models.torch_utils.dataloader import create_dataloader
from torch_models.CNN.CNN import CNN
from torch_models.InceptionTime.InceptionTime import Inception
#from torch_models.EEGNet.eegNet import EEGNet
from torch_models.Xception.Xception import XCEPTION
from torch_models.EEGNet.eegNet import EEGNet
from torch_models.PyramidalCNN.PyramidalCNN import PyramidalCNN
class Ensemble_torch:
"""
......@@ -70,25 +72,12 @@ class Ensemble_torch:
pred[j][batch] = pred[j][batch] + predictions
#pred[j] = pred[j] + pred_epoch
logging.info('Finished training model number {}/{} ...'.format(i+1, self.nb_models))
logging.info("------------------------------------------------------------------------------------")
logging.info("Finished ensemble training")
print(f"pred size: {len(pred)}")
print(f"pred type: {type(pred)}")
print(f"pred[0] size: {len(pred[0])}")
print(f"pred[0] type: {type(pred[0])}")
print(f"pred[0]0 size: {len(pred[0][0])}")
print(f"pred[0]0 type: {type(pred[0][0])}")
for j, pred_epoch in enumerate(pred_ensemble.predhis):
print(f"predensemble predhis size: {len(pred_ensemble.predhis)}")
for j, pred_epoch in enumerate(pred):
print(f"compute loss epoch number {j}")
#pred_epoch = (pred_epoch/config['ensemble']).tolist()
#pred_epoch = (pred_epoch/config['ensemble']).tolist() this is now done in compute_loss with the tensor
loss.append(compute_loss(loss_fn=self.loss_fn, dataloader=test_dataloader, pred_list=pred_epoch, nb_models=config['ensemble']))
pred_epoch = np.round(pred_epoch,0)
#pred_epoch = np.round(pred_epoch,0) this is now done in compute_accuracy with the tensor
if config['task'] == 'prosaccade-clf':
#accuracy.append(np.mean((np.array(pred_epoch).reshape(-1)+np.array(pred_ensemble.targets).reshape(-1)-1)**2))
accuracy.append(compute_accuracy(dataloader=test_dataloader, pred_list=pred_epoch, nb_models=config['ensemble']))
......@@ -100,7 +89,7 @@ class Ensemble_torch:
#TODO: create plots for torch models
#hist.history['val_loss'] = loss
np.savetxt('ensemble_loss.csv', loss, delimiter=',')
np.savetxt(X=loss, fname=config['model_dir']+'/'+'ensemble_loss.csv', delimiter=',')
#plot_loss(hist, config['model_dir'], config['model'], val = True)
#if config['task'] == 'prosaccade-clf':
## plot_acc(hist, config['model_dir'], config['model'], val = True)
......@@ -154,20 +143,20 @@ def create_model(model_type, model_number):
elif model_type == 'inception':
model = Inception(input_shape=config['inception']['input_shape'], use_residual=True, model_number=model_number,
kernel_size=64, nb_filters=16, depth=12, bottleneck_size=16, epochs=config['epochs'])
#elif model_type == 'eegnet':
# model = EEGNet(input_shape=(config['eegnet']['samples'], config['eegnet']['channels']),
# model_number=model_number, epochs=config['epochs'])
elif model_type == 'xception':
model = XCEPTION(input_shape=config['inception']['input_shape'], use_residual=True, model_number=model_number,
kernel_size=40, nb_filters=64, depth=18, epochs=config['epochs'])
elif model_type == 'eegnet':
model = EEGNet(input_shape=(config['eegnet']['samples'], config['eegnet']['channels']),
model_number=model_number, epochs=config['epochs'])
elif model_type == 'pyramidal_cnn':
model = PyramidalCNN(input_shape=config['cnn']['input_shape'], epochs=config['epochs'],
model_number=model_number)
"""
elif model_type == 'deepeye':
model = DEEPEYE(input_shape=config['deepeye']['input_shape'], model_number=model_number,
epochs=config['epochs'])
elif model_type == 'pyramidal_cnn':
model = PyramidalCNN(input_shape=config['cnn']['input_shape'], epochs=config['epochs'],
model_number=model_number)
elif model_type == 'xception':
model = XCEPTION(input_shape=config['inception']['input_shape'], use_residual=True, model_number=model_number,
kernel_size=40, nb_filters=64, depth=18, epochs=config['epochs'])
elif model_type == 'deepeye-rnn':
model = DEEPEYE_RNN(input_shape=config['deepeye-rnn']['input_shape'], model_number=model_number)
"""
......
from torch.nn.modules import batchnorm
from torch.nn.modules.batchnorm import BatchNorm1d
from config import config
from torch_models.ConvNetTorch import ConvNet
import torch
import torch.nn as nn
import math
from torch_models.Modules import Shortcut_layer
import logging
from torch_models.torch_utils.padding import pad_conv1d, pad_pool1d
from torch_models.Modules import Pad_Conv, Pad_Pool
class Inception(ConvNet):
"""
......@@ -19,13 +18,13 @@ class Inception(ConvNet):
def __init__(self, input_shape, kernel_size=64, epochs = 50, nb_filters=16, verbose=True,
batch_size=64, use_residual=True, depth=12, bottleneck_size=16, model_number=0):
self.bottleneck_size = bottleneck_size
super().__init__(input_shape, kernel_size=kernel_size, epochs=epochs,
nb_features = 4 * self.nb_filters # these are the 4 concatenated parallel convolutions, width of the inner tensort passed through network
logging.info('--------------- bottleneck_size : ' + str(self.bottleneck_size))
super().__init__(input_shape, kernel_size=kernel_size, epochs=epochs, nb_features=nb_features,
nb_filters=nb_filters, verbose=verbose, batch_size=batch_size,
use_residual=use_residual, depth=depth, model_number=model_number)
logging.info('--------------- bottleneck_size : ' + str(self.bottleneck_size))
def _module(self, input_tensor, current_depth):
def _module(self, depth):
"""
The module of InceptionTime (Taken from the implementation of InceptionTime paper).
It is made of a bottleneck convolution that reduces the number of channels from 128 -> 32.
......@@ -33,11 +32,9 @@ class Inception(ConvNet):
In parallel it uses a simple convolution with kernel size 1 with max pooling for stability during training.
The outputs of each convolution are concatenated, followed by batch normalization and a ReLu activation.
"""
pass
return Inception_module(mother=self, depth=depth)
"""
Tensorflow:
Tensorflow code:
if int(input_tensor.shape[-1]) > 1:
input_inception = tf.keras.layers.Conv1D(filters=self.bottleneck_size, kernel_size=1, padding='same', use_bias=False)(input_tensor)
else:
......@@ -59,4 +56,52 @@ class Inception(ConvNet):
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(activation='relu')(x)
return x
"""
\ No newline at end of file
"""
class Inception_module(nn.Module):
def __init__(self, mother, depth):
super().__init__()
self.mother = mother
kernel_size_s = [mother.kernel_size // (2 ** i) for i in range(3)]
# Define all the layers and modules we need in the forward pass: first the initial convolution and the parallel maxpooling
self.pad_conv_in = Pad_Conv(kernel_size=mother.kernel_size)
self.conv_in = nn.Conv1d(in_channels=mother.nb_channels if depth==0 else mother.nb_features,
out_channels=mother.nb_filters, kernel_size=mother.kernel_size, bias=False)
self.pad_pool_in = Pad_Pool(left=1, right=1)
self.maxpool_in = nn.MaxPool1d(kernel_size=3, stride=1)
# 3 parallel convolutions with their paddings
self.conv1 = nn.Conv1d(in_channels=mother.nb_filters, out_channels=mother.nb_filters, kernel_size=kernel_size_s[0], bias=False)
self.pad1 = Pad_Conv(kernel_size=kernel_size_s[0])
self.conv2 = nn.Conv1d(in_channels=mother.nb_filters, out_channels=mother.nb_filters, kernel_size=kernel_size_s[1], bias=False)
self.pad2 = Pad_Conv(kernel_size=kernel_size_s[1])
self.conv3 = nn.Conv1d(in_channels=mother.nb_filters, out_channels=mother.nb_filters, kernel_size=kernel_size_s[2], bias=False)
self.pad3 = Pad_Conv(kernel_size=kernel_size_s[2])
# and the 4th parallel convolution following the maxpooling, no padding needed since 1x1 convolution
self.conv4 = nn.Conv1d(in_channels=mother.nb_channels if depth==0 else mother.nb_features,
out_channels=mother.nb_channels if depth==0 else mother.nb_filters,
kernel_size=1, bias=False)
self.batchnorm = nn.BatchNorm1d(num_features=mother.nb_features)
self.activation = nn.ReLU()
def forward(self, x):
"""
Implements a forward pass through one inception module
"""
# Implement the left convolution
x_left = self.pad_conv_in(x)
x_left = self.conv_in(x_left)
# Implement the 3 parallel convolutions afterwards
x_left1 = self.pad1(x_left)
x_left1 = self.conv1(x_left1)
x_left2 = self.pad2(x_left)
x_left2 = self.conv2(x_left2)
x_left3 = self.pad1(x_left)
x_left3 = self.conv1(x_left3)
# Implement the right maxpooling followed by a conv
x_right = self.pad_pool_in(x)
x_right = self.maxpool_in(x_right)
x_right = self.conv4(x_right)
# Concatenate the 4 outputs
x = torch.cat(tensors=(x_left1, x_left2, x_left3, x_right), dim=2) # concatenate along the feature dimension
x = self.batchnorm(x)
return self.activation(x)
\ No newline at end of file
from torch_models.ConvNetTorch import ConvNet
import torch.nn as nn
from torch_models.Modules import Pad_Conv, Pad_Pool
class PyramidalCNN(ConvNet):
"""
The Classifier_PyramidalCNN is one of the simplest classifiers. It implements the class ConvNet, which is made of modules with a
specific depth, where for each depth the number of filters is increased.
nb_features: specifies number of channels before the output layer
"""
def __init__(self, input_shape, kernel_size=16, epochs = 50, nb_filters=16, verbose=True, batch_size=64,
use_residual=False, depth=6, model_number=0, regularization=0):
self.regularization = regularization
self.nb_features = depth * nb_filters # For pyramidal we increase the nbfilters each depth layer
super().__init__(input_shape, kernel_size=kernel_size, epochs=epochs, nb_filters=nb_filters,
verbose=verbose, batch_size=batch_size, use_residual=use_residual, depth=depth,
model_number=model_number)
def _module(self, depth):
"""
The module of CNN is made of a simple convolution with batch normalization and ReLu activation. Finally, MaxPooling is also used.
"""
return nn.Sequential(
Pad_Conv(kernel_size=self.kernel_size),
nn.Conv1d(in_channels=129 if depth==0 else depth*self.nb_filters,
out_channels=(depth+1)*self.nb_filters, kernel_size=self.kernel_size, bias=False),
nn.BatchNorm1d(num_features=(depth+1)*self.nb_filters),
nn.ReLU(),
Pad_Pool(left=0, right=1),
nn.MaxPool1d(kernel_size=2, stride=1)
)
"""
Tensorflow code:
x = tf.keras.layers.Conv1D(filters=self.nb_filters*(current_depth + 1), kernel_size=self.kernel_size, padding='same', use_bias=False)(input_tensor)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(activation='relu')(x)
x = tf.keras.layers.MaxPool1D(pool_size=2, strides=2)(x)
"""
import math
from torch import nn
def pad_conv1d(x, kernel_size, value=0):
"""
Pads the input tensor on the left and right side
Assumes a stride of 1
"""
left_pad = math.floor(kernel_size/2)-1
#right_pad = math.ceil(kernel_size/2)
right_pad = math.floor(kernel_size/2)
pad = nn.ConstantPad1d(padding=(left_pad,right_pad), value=value)
return pad(x)
def pad_pool1d(x, value=0):
"""
Pads the input tensor on the left and right side
Assumes a stride of 1, kernel size of 2
"""
pad = nn.ConstantPad1d(padding=(0,1), value=value)
return pad(x)
\ No newline at end of file
......@@ -34,5 +34,6 @@ def compute_accuracy(dataloader, pred_list, nb_models):
size = len(dataloader.dataset)
for batch, (X,y) in dataloader:
pred = pred_list[batch]
pred = torch.round(pred)
correct += (pred - y).type(torch.float).sum().item()
return correct / size
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment