Commit fd57ab66 authored by Lukas Wolf's avatar Lukas Wolf
Browse files

implemented cnn in torch

parent 90417504
......@@ -62,11 +62,11 @@ config['framework'] = 'torch'
config['ensemble'] = 5 #number of models in the ensemble
config['pretrained'] = False # We can use a model pretrained on processing speed task
#config['model'] = 'cnn'
config['model'] = 'cnn'
#config['model'] = 'inception'
#config['model'] = 'eegnet'
#config['model'] = 'deepeye'
config['model'] = 'deepeye-rnn'
#config['model'] = 'deepeye-rnn'
#config['model'] = 'xception'
#config['model'] = 'pyramidal_cnn'
#config['model'] = 'siamese' # Note that you have to set data_mode to sacc_fix for this model
......
......@@ -19,9 +19,9 @@ def main():
# Load the data
try:
#trainX, trainY = IOHelper.get_mat_data(config['data_dir'], verbose=True)
trainX = np.load("./data/precomputed/processing_speed_task/sacc_fix_X.npy")
trainY = np.load("./data/precomputed/processing_speed_task/sacc_fix_y.npy")
trainX, trainY = IOHelper.get_mat_data(config['data_dir'], verbose=True)
#trainX = np.load("./data/precomputed/processing_speed_task/sacc_fix_X.npy")
#trainY = np.load("./data/precomputed/processing_speed_task/sacc_fix_y.npy")
except:
raise Exception("Could not load mat data")
......
......@@ -34,7 +34,7 @@ class BaseNet:
elif config['task'] == 'gaze-reg':
self.model.compile(loss='mean_squared_error', optimizer=keras.optimizers.Adam(), metrics=['mean_squared_error'])
elif config['task'] == 'angle-reg':
from utils.losses import angle_loss
from tf_models.utils.losses import angle_loss
self.model.compile(loss=angle_loss, optimizer=keras.optimizers.Adam())
if self.verbose:
......
......@@ -54,7 +54,6 @@ class Ensemble_tf:
self.models.append(create_model(self.model_type, model_number=i))
logging.info("Built ensemble model with {} {} model(s)".format(self.nb_models, self.model_type))
def run(self, X, y):
"""
Fit all the models in the ensemble and save them to the run directory
......@@ -126,7 +125,7 @@ class Ensemble_tf:
def create_model(model_type, model_number):
"""
Returns a compiled model
Returns a compiled tensorflow model
"""
if model_type == 'deepeye':
model = DEEPEYE(input_shape=config['deepeye']['input_shape'], model_number=model_number,
......
......@@ -3,12 +3,16 @@ from torch import nn
import numpy as np
from config import config
import logging
from sklearn.model_selection import train_test_split
from torch_models.utils.dataloader import create_dataloader
from torch_models.utils.training import train_loop, test_loop
class BaseNet(nn.Module):
"""
BaseNet class for ConvNet and EEGnet to inherit common functionality
"""
def __init__(self, epochs=50, verbose=True, model_number=0):
"""
BaseNet class for ConvNet and EEGnet to inherit common functionality
"""
super().__init__()
self.epochs = epochs
self.verbose = verbose
......@@ -25,10 +29,45 @@ class BaseNet(nn.Module):
def forward(self, x):
pass
def get_model(self):
return self
# abstract method
def _split_model(self):
pass
# abstract method
def _build_model(self):
pass
\ No newline at end of file
pass
def fit(self, x, y, subjectID=None):
logging.info(f"Fiting model {self.__name__}, model number {self.model_number}")
# Create a split
X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=42)
# Create dataloaders
train_dataloader = create_dataloader(X_train, y_train, batch_size=config['batch_size'])
test_dataloader = create_dataloader(X_val, y_val, batch_size=config['batch_size'])
logging.info(f"Created train dataloader with x shape{x[0].shape} and y shape {y[0].shape}")
# Create the training depending on the task
if config['task'] == 'prosaccade-clf':
loss_fn = nn.BCELoss()
elif config['task'] == 'gaze-reg':
loss_fn = nn.MSELoss()
elif config['task'] == 'angle-reg':
from torch_models.utils.custom_losses import angle_loss
# Create the optimizer
optimizer = torch.optim.Adam(list(self.parameters()), lr=config['learning_rate'])
# Train the model
epochs = config['epochs']
for t in range(epochs):
logging.info(f"Epoch {t+1}\n-------------------------------")
train_loop(train_dataloader, self, loss_fn, optimizer)
test_loop(test_dataloader, self, loss_fn)
logging.info(f"Finished model {self.__name__}, model number {self.model_number}")
# Save model
ckpt_dir = config['model_dir'] + '/best_models/' + self.__str__ + '_nb_{}_'.format(self.model_number) + 'best_model.h5'
torch.save(self, ckpt_dir)
import torch
from config import config
import logging
from torch_models.ConvNetTorch import ConvNet
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch_models.Modules import conv_block
class CNN(ConvNet):
"""
......@@ -27,14 +23,15 @@ class CNN(ConvNet):
"""
The module of CNN is made of a simple convolution with batch normalization and ReLu activation. Finally, MaxPooling is also used.
"""
return conv_block(
in_channels=1, # we only have a 2D input on 1 channel
out_channels=self.nb_filters,
kernel_size=self.kernel_size,
stride=1,
padding=3,
)(input_tensor)
return nn.Sequential(
nn.Conv1d(in_channels=self.nb_filters, out_channels=self.nb_filters, kernel_size=self.kernel_size, groups=self.nb_filters),
nn.BatchNorm1d(),
nn.ReLU(),
nn.MaxPool1d(kernel_size=2, stride=1, padding=1)
)
"""
Tensorflow code:
x = tf.keras.layers.Conv1D(filters=self.nb_filters, kernel_size=self.kernel_size,
padding='same', use_bias=False)(input_tensor)
x = tf.keras.layers.BatchNormalization()(x)
......
from abc import ABC, abstractmethod
from re import X
from torch_models.BaseNetTorch import BaseNet
from torch_models.Modules import Shortcut_layer
import torch
import torch.nn as nn
import torch.nn.functional as F
......@@ -28,7 +29,32 @@ class ConvNet(ABC, BaseNet):
self.preprocessing = preprocessing
self.input_shape = input_shape
# Define the layers that we need
# Define the layers that we need in the forward pass
#self.module = self._module() #returns a nn.Sequential that can be used
self.input_layer = nn.Sequential(
nn.Conv1d(in_channels=self.input_shape[0], out_channels=16, kernel_size=3),
nn.Sigmoid(),
nn.MaxPool1d(kernel_size=1, stride=1) # do not reduce size
)
self.gap_layer = nn.AvgPool1d(kernel_size=1, stride=1)
# Create output layer depending on task
if config['task'] == 'prosaccade_clf':
self.output_layer = nn.Sequential(
nn.Linear(16, 1),
nn.Sigmoid()
)
elif config['task'] == 'gaze-reg':
self.output_layer = nn.Sequential(
nn.Linear(16, 2)
)
else: #elif config['task'] == 'angle-reg':
self.output_layer = nn.Sequential(
nn.Linear(16, 1)
)
"""
Working example network
self.conlayer1 = nn.Sequential(
nn.Conv1d(in_channels=180, out_channels=16, kernel_size=3, groups=4),
nn.Sigmoid(),
......@@ -41,6 +67,7 @@ class ConvNet(ABC, BaseNet):
nn.Linear(480,120),
nn.Linear(120,84),
nn.Linear(84,2))
"""
logging.info('Parameters of {}, model number {}: '.format(self, model_number))
logging.info('--------------- use residual : ' + str(self.use_residual))
......@@ -51,58 +78,37 @@ class ConvNet(ABC, BaseNet):
logging.info('--------------- preprocessing: ' + str(self.preprocessing))
def forward(self, x):
#return self.model(x)
out = self.conlayer1(x)
out = self.conlayer2(out)
out = out.view(out.size(0),-1)
out = self.fc(out)
return out
def fit(self, dataloader):
#TODO: use the train and test loop and fit the model here instead of in ensemble
pass
def _build_model(self, x):
"""
Build the model architecure
"""
# Build the model architecture here in torch
x = self.input_layer(x)
for d in range(self.depth):
x = self._module(x, d)
if self.use_residual and d % 3 == 2:
x = self._shortcut_layer(input_res, x)
x = self.Shortcut_layer(input_res, x)
input_res = x
x = nn.AvgPool1d()(x)
x = self.gap_layer(x)
if config['task'] == 'prosaccade_clf':
output_layer = nn.Linear(out_features=1)(x)
output_layer = nn.Sigmoid(output_layer)
elif config['task'] == 'gaze-reg':
output_layer = nn.Linear(out_features=2)(x)
else: #elif config['task'] == 'angle-reg':
output_layer = nn.Linear(out_features=1)(x)
return output_layer
if config['split']:
return x
def _shortcut_layer(self, input_tensor, out_tensor):
output = self.output_layer(x)
return output
"""
Equivalent to following tensorflow code:
shortcut_y = tf.keras.layers.Conv1D(filters=int(out_tensor.shape[-1]), kernel_size=1, padding='same', use_bias=False)(input_tensor)
shortcut_y = tf.keras.layers.BatchNormalization()(shortcut_y)
x = keras.layers.Add()([shortcut_y, out_tensor])
x = keras.layers.Activation('relu')(x)
Working example network:
out = self.conlayer1(x)
out = self.conlayer2(out)
out = out.view(out.size(0),-1)
out = self.fc(out)
return out
"""
shortcut_y = nn.Conv1d(
in_channels=1,
out_channels=self.nb_filters,
kernel_size=self.kernel_size
)(input_tensor)
shortcut_y = nn.BatchNorm1d()(shortcut_y)
x = torch.cat((shortcut_y, out_tensor), 1)
x = nn.ReLU(x)
return x
def _split_model(self):
def _build_model(self, x):
# Redundant for torch models since model architecture is defined in the forward() pass
pass
def _shortcut_layer(self, input_tensor, out_tensor):
# Redundant as this is implemented in a class now
pass
# abstract method
......@@ -111,4 +117,7 @@ class ConvNet(ABC, BaseNet):
# abstract method
def _module(self, input_tensor, current_depth):
pass
def _split_model(self):
pass
\ No newline at end of file
......@@ -2,9 +2,7 @@ from config import config
import logging
import torch
import torch.nn as nn
from sklearn.model_selection import train_test_split
from torch_models.utils.dataloader import create_dataloader
from torch_models.utils.training import train_loop, test_loop
import os
from torch_models.CNN.CNN import CNN
......@@ -14,10 +12,7 @@ class Ensemble_torch:
Default: nb_models of the model_type model
Optional: Initialize with a model list, create a more versatile Ensemble model
"""
def __init__(self):
pass
def __init__(self, nb_models=1, model_type='cnn', model_list=[]):
"""
nb_models: Number of models to run in the ensemble
......@@ -34,43 +29,38 @@ class Ensemble_torch:
return self.__class__.__name__
def load_models(self, path_to_models):
#TODO: implement
# load all models into the model_list to predict with them
pass
def run(self, X, y):
for model_name in os.listdir(path_to_models):
# load models and save them in self.models
# Adapt nb_models
# Create model_list as string list for completeness
pass
def run(self, x, y):
"""
Fit all the models in the ensemble and save them to the run directory
"""
# Create a split
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)
# create dataloaders
train_dataloader = create_dataloader(X_train, y_train, batch_size=64)
test_dataloader = create_dataloader(X_val, y_val, batch_size=64)
logging.info("Created dataloader")
# Create model
model = CNN(input_shape=config['cnn']['input_shape'])
logging.info("Created model")
model.fit(x, y)
# Hyperparams
learning_rate = 1e-3
batch_size = 64
# Define loss and optimizer
loss_fn = nn.MSELoss()
optimizer = torch.optim.Adam(list(model.parameters()), lr=learning_rate)
# Train the model
epochs = 30
for t in range(epochs):
print(f"Epoch {t+1}\n-------------------------------")
train_loop(train_dataloader, model, loss_fn, optimizer)
test_loop(test_dataloader, model, loss_fn)
print("Done!")
#TODO: create logs, create plots, etc.
def _build_ensemble_model(self, model_list):
"""
Create a list of compiled models
Default: create a list of self.nb_models many models of type self.model_type
Optional: create a list of the models described in self.model_list
"""
if len(model_list) > 0:
#TODO: implement loading a list of different models
logging.info("Built ensemble model with model(s): {}".format(self.model_list))
pass
else:
for i in range(self.nb_models):
self.models.append(create_model(self.model_type, model_number=i))
logging.info("Built ensemble model with {} {} model(s)".format(self.nb_models, self.model_type))
def predict(self, X):
"""
......@@ -82,4 +72,34 @@ class Ensemble_torch:
pred = self.models[i].model.predict(X)
else:
pred += self.models[i].model.predict(X)
return pred / len(self.models)
\ No newline at end of file
return pred / len(self.models)
def create_model(model_type, model_number):
"""
Returns a torch model
"""
if model_type == 'cnn':
model = CNN(input_shape=config['cnn']['input_shape'], kernel_size=64, epochs = config['epochs'],
nb_filters=16, verbose=True, batch_size=64, use_residual=True, depth=12,
model_number=model_number)
"""
elif model_type == 'deepeye':
model = DEEPEYE(input_shape=config['deepeye']['input_shape'], model_number=model_number,
epochs=config['epochs'])
elif model_type == 'pyramidal_cnn':
model = PyramidalCNN(input_shape=config['cnn']['input_shape'], epochs=config['epochs'],
model_number=model_number)
elif model_type == 'eegnet':
model = EEGNet(dropoutRate = 0.5, kernLength = 64, F1 = 32, D = 8, F2 = 512, norm_rate = 0.5,
dropoutType = 'Dropout', model_number=model_number, epochs=config['epochs'])
elif model_type == 'inception':
model = INCEPTION(input_shape=config['inception']['input_shape'], use_residual=True, model_number=model_number,
kernel_size=64, nb_filters=16, depth=12, bottleneck_size=16, epochs=config['epochs'])
elif model_type == 'xception':
model = XCEPTION(input_shape=config['inception']['input_shape'], use_residual=True, model_number=model_number,
kernel_size=40, nb_filters=64, depth=18, epochs=config['epochs'])
elif model_type == 'deepeye-rnn':
model = DEEPEYE_RNN(input_shape=config['deepeye-rnn']['input_shape'], model_number=model_number)
"""
return model
\ No newline at end of file
import torch
import torch.nn as nn
class conv_block(nn.Module):
class Conv_block(nn.Module):
"""
Basic convolutional module for use in our modules
"""
def __init__(self, in_channels, out_channels, **kwargs):
super(conv_block, self).__init__()
super(Conv_block, self).__init__()
self.relu = nn.ReLU()
self.conv = nn.Conv1d(in_channels, out_channels, **kwargs)
self.batchnorm = nn.BatchNorm1d(out_channels)
self.maxpool = nn.MaxPool1d(**kwargs)
def forward(self, x):
return self.maxpool(self.relu(self.batchnorm(self.conv(x))))
\ No newline at end of file
return self.maxpool(self.relu(self.batchnorm(self.conv(x))))
class Shortcut_layer(nn.Module):
"""
Implements the functionality of a shortcut with a conv1d module in between
"""
def __init__(self, input_tensor, output_tensor) -> None:
super().__init__()
self.input_tensor = input_tensor
self.output_tensor = output_tensor
self.conv = nn.Conv1d(input_tensor.shape, output_tensor.shape, kernel_size=1, use_bias=False)
self.batch_norm = nn.BatchNorm1d()
self.sigmoid = nn.Sigmoid()
self.maxpool = nn.MaxPool1d(kernel_size=1, stride=1)
self.relu = nn.ReLU()
def forward(self, x):
shortcut_y = self.conv(x)
shortcut_y = self.batch_norm(shortcut_y)
concat = torch.cat((shortcut_y, self.output_tensor), dim=1)
out = self.relu(concat)
return out
\ No newline at end of file
"""
Definition of custum loss functions that can be used in our models
"""
import torch
def angle_loss(a, b):
"""
Custom loss function for models that predict the angle on the fix-sacc-fix dataset
Angles -pi and pi should lead to 0 loss, since this is actually the same angle on the unit circle
Angles pi/2 and -pi/2 should lead to a large loss, since this is a difference by pi on the unit circle
Therefore we compute the absolute error of the "shorter" direction on the unit circle
"""
return torch.reduce_mean(torch.abs(torch.atan2(torch.sin(a - b), torch.cos(a - b))))
\ No newline at end of file
import torch
import logging
from config import config
import torch
def train_loop(dataloader, model, loss_fn, optimizer):
"""
Performs one epoch of training the model through the dataset stored in dataloader
Using the given loss_fn and optimizer
"""
size = len(dataloader.dataset)
for batch, (X, y) in enumerate(dataloader):
# Compute prediction and loss
#logging.info(f"batch shape X {X.shape}, y {y.shape}")
pred = model(X)
loss = loss_fn(pred.float(), y.float())
# Backpropagation
optimizer.zero_grad()
loss.backward()
optimizer.step()
if batch % 100 == 0:
loss, current = loss.item(), batch * len(X)
print(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]")
def test_loop(dataloader, model, loss_fn):
"""
Performs one prediction run through the test set stored in the dataloader
Prints the loss function computed with the prediction pred and the labels y
"""
size = len(dataloader.dataset)
test_loss, correct = 0, 0
......@@ -29,8 +33,12 @@ def test_loop(dataloader, model, loss_fn):
for X, y in dataloader:
pred = model(X)
test_loss += loss_fn(pred, y).item()
#correct += (pred - y).type(torch.float).sum().item()
if config['task'] == 'prosaccade_clf':
correct += (pred - y).type(torch.float).sum().item()
test_loss /= size
#correct /= size
print(f"Avg test loss: {test_loss:>8f} \n")
\ No newline at end of file
print(f"Avg test loss: {test_loss:>8f} \n")
if config['task'] == 'prosaccade_clf':
correct /= size
print(f"Avg accuracy {correct:>8f} \n")
\ No newline at end of file
......@@ -7,7 +7,7 @@ def log_config():
else:
logging.info("Running the ensemble with {} {} models".format(config['ensemble'], config['model']))
if config['task'] == 'gaze-reg':
logging.info("Training gaze regression task")
logging.info("Training on the gaze regression task")
logging.info("Using data from {}".format(config['dataset']))
logging.info("Using {} padding".format(config['padding']))
if config["data_mode"] != "sacc_only":
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment