To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 5cdc8474 authored by Lukas Wolf's avatar Lukas Wolf
Browse files

prepare for computing metrics on test set after training, plots are done now...

prepare for computing metrics on test set after training, plots are done now only for each model that is part of the ensemble (and potentially early stopped). After a model is done it once predicts the test set, predictions are accumulated and then averaged (majority decision for classification)
parent 4cd00f24
......@@ -40,11 +40,17 @@ config['root_dir'] = '.'
# Choose task and dataset
##################################################################
config['preprocessing'] = 'max' # options: min and max
config['preprocessing'] = 'min' # options: min and max
#config['task'] = 'prosaccade-clf'
config['task'] = 'gaze-reg'
config['task'] = 'prosaccade-clf'
#config['task'] = 'gaze-reg'
#config['task'] = 'angle-reg'
#config['task'] = 'amplitude-reg'
# For angle and direction reg we can choose 2 datasets
config['dataset'] = 'calibration_task'
#config['dataset'] = 'processing_speed_task'
config['input_shape'] = (500,129)
......@@ -63,11 +69,11 @@ config['ensemble'] = 5 #number of models in the ensemble
config['pretrained'] = False # We can use a model pretrained on processing speed task
# MODELS FOR BENCHMARK
config['model'] = 'cnn'
#config['model'] = 'cnn'
#config['model'] = 'inception'
#config['model'] = 'eegnet'
#config['model'] = 'xception'
#config['model'] = 'pyramidal_cnn'
config['model'] = 'pyramidal_cnn'
# EXPERIMENTAL MODELS
#config['model'] = 'deepeye'
......@@ -82,10 +88,10 @@ with open('hyperparams.json', 'r') as file:
params = json.load(file)
config['learning_rate'] = params[config['model']][config['task']]['learning_rate']
config['regularization'] = params[config['model']][config['task']]['regularization']
config['epochs'] = 75
config['epochs'] = 50
config['batch_size'] = 64
config['early_stopping'] = True
config['patience'] = 10
config['early_stopping'] = True
config['patience'] = 20
##################################################################
# Choose between ensemble and kerasTuner
......@@ -120,6 +126,7 @@ model_folder_name += "_pretrained_" + config['model'] if config['pretrained'] el
# Modify the model folder name depending on which task tuns
model_folder_name += "_" + config['task']
model_folder_name += "_prep" + config['preprocessing']
model_folder_name += f"_lr_{str(config['learning_rate'])}"
if config['split']:
model_folder_name += '_cluster'
......
......@@ -12,6 +12,8 @@ from torch_models.torch_utils.utils import get_gpu_memory
import psutil
from torch_models.torch_utils.utils import timing_decorator
from memory_profiler import profile
class Prediction_history:
"""
Collect predictions of the given validation set after each epoch
......@@ -36,16 +38,6 @@ class Prediction_history:
y = y.cuda()
pred = self.model(x)
y_pred.append(pred)
if batch==0:
loss = nn.MSELoss()
cat = torch.cat((y, pred), dim=1)
batch_loss = loss(y, pred)
print(f"true label vs. pred")
print(cat)
print(f"batch loss")
print(batch_loss)
# Remove batch from GPU
del x
del y
......@@ -69,9 +61,9 @@ class BaseNet(nn.Module):
self.batch_size = batch_size
self.timesamples = self.input_shape[0]
self.nb_channels = self.input_shape[1]
self.early_stopped = False # Set to true and skip training if this is false
self.early_stopped = False
# Create output layer depending on task and
# Create output layer depending on task
if config['task'] == 'prosaccade-clf':
self.loss_fn = nn.BCELoss()
self.output_layer = nn.Sequential(
......@@ -83,12 +75,17 @@ class BaseNet(nn.Module):
self.output_layer = nn.Sequential(
nn.Linear(in_features=self.get_nb_features_output_layer(), out_features=2)
)
else: #elif config['task'] == 'angle-reg':
elif config['task'] == 'angle-reg':
from torch_models.torch_utils.custom_losses import angle_loss
self.loss_fn = angle_loss
self.output_layer = nn.Sequential(
nn.Linear(in_features=self.get_nb_features_output_layer(), out_features=1)
)
else: # elif config['task'] == 'amplitude-reg':
self.loss_fn = nn.L1Loss()
self.output_layer = nn.Sequential(
nn.Linear(in_features=self.get_nb_features_output_layer(), out_features=1)
)
if verbose and self.model_number == 0:
logging.info(f"Using loss fct: {self.loss_fn}")
......@@ -113,7 +110,7 @@ class BaseNet(nn.Module):
#@profile
#@timing_decorator
def fit(self, train_dataloader, validation_dataloader, test_dataloader, subjectID=None):
def fit(self, train_dataloader, validation_dataloader, test_dataloader=None, subjectID=None):
"""
Fit the model on the dataset defined by data x and labels y
"""
......@@ -124,11 +121,11 @@ class BaseNet(nn.Module):
# Create the optimizer
optimizer = torch.optim.Adam(list(self.parameters()), lr=config['learning_rate'])
# Create a history to track ensemble performance
prediction_ensemble = Prediction_history(dataloader=test_dataloader, model=self)
#prediction_ensemble = Prediction_history(dataloader=test_dataloader, model=self)
# Train the model
epochs = config['epochs']
metrics = {'train_loss':[], 'val_loss':[], 'train_acc':[], 'val_acc':[]} if config['task'] == 'prosaccade-clf' else {'train_loss':[], 'val_loss':[]}
curr_val_loss = sys.maxsize # For early stopping
best_val_loss = sys.maxsize # For early stopping
patience = 0
for t in range(epochs):
logging.info("-------------------------------")
......@@ -150,19 +147,23 @@ class BaseNet(nn.Module):
# print("Free GPU mem after test loop:")
# print(f"memory {psutil.virtual_memory()}")
# Add the predictions on the validation set, even if model was early stopped
prediction_ensemble.on_epoch_end()
#prediction_ensemble.on_epoch_end()
# print("Free GPU mem after prediction hist:")
# print(f"memory {psutil.virtual_memory()}")
# Impementation of early stopping
if config['early_stopping'] and not self.early_stopped:
if config['early_stopping']:
if patience > config['patience']:
logging.info(f"Early stopping the model after {t} epochs")
self.early_stopped = True
if val_loss_epoch > curr_val_loss:
break
if val_loss_epoch >= best_val_loss:
logging.info(f"Validation loss did not improve, best was {best_val_loss}")
patience +=1
else:
curr_val_loss = val_acc_epoch
best_val_loss = val_acc_epoch
logging.info(f"Improved validation loss to: {best_val_loss}")
patience = 0
# Plot and save metrics
plot_metrics(metrics['train_loss'], metrics['val_loss'], output_dir=config['model_dir'], metric='loss', model_number=self.model_number)
if config['task'] == 'prosaccade-clf':
......@@ -173,4 +174,4 @@ class BaseNet(nn.Module):
if config['save_models']:
ckpt_dir = config['model_dir'] + '/best_models/' + config['model'] + '_nb_{}_'.format(self.model_number) + 'best_model.pth'
torch.save(self.state_dict(), ckpt_dir)
return prediction_ensemble
\ No newline at end of file
#return prediction_ensemble
\ No newline at end of file
from sklearn.utils import validation
from torch.utils import data
from config import config
import logging
import torch.nn as nn
import numpy as np
from sklearn.model_selection import train_test_split
from torch_models.torch_utils.custom_losses import angle_loss
from torch_models.torch_utils.utils import compute_loss, compute_accuracy
from torch_models.torch_utils.utils import compute_loss, compute_accuracy, sum_predictions
from torch_models.torch_utils.plot import plot_array
from torch_models.torch_utils.dataloader import create_dataloader
from utils.data_split import generate_split
......@@ -41,6 +41,9 @@ class Ensemble_torch:
self.loss_fn = angle_loss
elif config['task'] == 'gaze-reg':
self.loss_fn = nn.MSELoss()
else: # amplitude-task
self.loss_fn = nn.L1Loss()
def run(self, x, y):
"""
......@@ -52,10 +55,16 @@ class Ensemble_torch:
# Create a split
x = np.transpose(x, (0, 2, 1)) # (batch_size, samples, channels) to (bs, ch, samples) as torch conv layers want it
X_train, y_train, X_val, y_val, X_test, y_test = generate_split(x, y)
# Remove the subject counter from the labels
y_train = y_train[:, 1:]
y_val = y_val[:, 1:]
y_test = y_test[:, 1:]
# Remove the subject counter from the labels and extract only angle for direction task
if config['task'] == 'amplitude-task':
y_train = y_train[:, 1:2]
y_val = y_val[:, 1:2] # only extract the amplitude, not the subjectid or angle
y_test = y_test[:, 1:2]
else:
offset = 2 if config['task'] == 'angle-reg' else 1
y_train = y_train[:, offset:]
y_val = y_val[:, offset:]
y_test = y_test[:, offset:]
# Log shapes
logging.info(f"Training data shapes X, y: {X_train.shape, y_train.shape}")
logging.info(f"Test data shapes X, y: {X_test.shape, y_test.shape}")
......@@ -67,26 +76,43 @@ class Ensemble_torch:
# Metrics to save across the ensemble
loss=[]
accuracy=[]
prediction_list = [] # saves the predicted batches in the test dataloader across models
# Fit the models
for i in range(config['ensemble']):
logging.info("------------------------------------------------------------------------------------")
logging.info('Start training model number {}/{} ...'.format(i+1, self.nb_models))
model = create_model(model_type=self.model_type, model_number=i)
pred_ensemble = model.fit(train_dataloader, validation_dataloader, test_dataloader)
# Collect the predictions on the validation sets
model.fit(train_dataloader, validation_dataloader, test_dataloader)
# Collect the prediction on the test set
prediction_list = sum_predictions(test_dataloader, model, model_number=i, prediction_list=prediction_list)
"""
Compute ensemble metrics
if i == 0:
pred = pred_ensemble.predhis
else:
for j, pred_epoch in enumerate(pred_ensemble.predhis):
for batch, predictions in enumerate(pred_epoch):
pred[j][batch] = pred[j][batch] + predictions
"""
logging.info('Finished training model number {}/{} ...'.format(i+1, self.nb_models))
logging.info("------------------------------------------------------------------------------------")
#Divide prediction list by number of models and compute the final loss and accuracy of the ensemble
ensemble_loss = compute_loss(loss_fn=self.loss_fn, dataloader=test_dataloader, pred_list=prediction_list, nb_models=self.nb_models)
if config['task'] == 'prosaccade-clf':
ensemble_acc = compute_accuracy(dataloader=test_dataloader, pred_list=prediction_list, nb_models=self.nb_models)
logging.info(f"ENSEMBLE ACCURACY ON TEST SET: {ensemble_acc}")
logging.info(f"ENSEMBLE LOSS ON TEST SET: {ensemble_loss}")
"""
# Create the ensemble metrics
for j, pred_epoch in enumerate(pred):
loss.append(compute_loss(loss_fn=self.loss_fn, dataloader=test_dataloader, pred_list=pred_epoch, nb_models=config['ensemble']))
if config['task'] == 'prosaccade-clf':
accuracy.append(compute_accuracy(dataloader=test_dataloader, pred_list=pred_epoch, nb_models=config['ensemble']))
"""
# Adapt model name if necessary
if config['ensemble']>1:
......@@ -95,11 +121,11 @@ class Ensemble_torch:
config['model'] = config['model'] + '_cluster'
# Save metrics and plot them
np.savetxt(X=loss, fname=config['model_dir']+'/metrics/'+'ensemble_test_loss_val.csv', delimiter=',')
plot_array(loss, config['model_dir'], 'test loss')
if config['task'] == 'prosaccade-clf':
np.savetxt(X=accuracy, fname=config['model_dir']+'/metrics/'+'ensemble_test_acc_val.csv', delimiter=',')
plot_array(accuracy, config['model_dir'], 'test accuracy')
# np.savetxt(X=loss, fname=config['model_dir']+'/metrics/'+'ensemble_test_loss_val.csv', delimiter=',')
# plot_array(loss, config['model_dir'], 'test loss')
# if config['task'] == 'prosaccade-clf':
# np.savetxt(X=accuracy, fname=config['model_dir']+'/metrics/'+'ensemble_test_acc_val.csv', delimiter=',')
# plot_array(accuracy, config['model_dir'], 'test accuracy')
def create_model(model_type, model_number):
"""
......@@ -115,7 +141,7 @@ def create_model(model_type, model_number):
model = XCEPTION(input_shape=config['input_shape'], use_residual=True, model_number=model_number,
kernel_size=40, nb_filters=64, depth=18, epochs=config['epochs'], batch_size=config['batch_size'])
elif model_type == 'eegnet':
model = EEGNet(input_shape=(config['input_shape'], config['eegnet']['channels']), batch_size=config['batch_size'],
model = EEGNet(input_shape=config['input_shape'], batch_size=config['batch_size'],
model_number=model_number, epochs=config['epochs'])
elif model_type == 'pyramidal_cnn':
model = PyramidalCNN(input_shape=config['input_shape'], epochs=config['epochs'],
......
......@@ -51,12 +51,6 @@ def compute_loss(loss_fn, dataloader, pred_list, nb_models):
# Average the predictions from the ensemble
pred = pred_list[batch]
pred = torch.div(pred, nb_models).float() # is already on gpu
"""
if batch==0:
cat = torch.cat((y, pred), dim=1)
print("Ensemble labels labels vs predictions")
print(cat)
"""
loss.append(loss_fn(y, pred) / config['batch_size'])
return sum(loss) / len(loss)
......@@ -80,13 +74,24 @@ def compute_accuracy(dataloader, pred_list, nb_models):
y = y.cuda()
# Average the predictions from the ensemble
pred = pred_list[batch]
pred = torch.div(pred, nb_models).float() # is already on gpu pred = torch.round(pred)
"""
if batch==0:
cat = torch.cat((y, pred), dim=1)
print("Ensemble labels labels vs predictions")
print(cat)
"""
pred = torch.div(pred, nb_models).float() # tensor is already on gpu
pred = torch.round(pred) # majority decision for classification
pred = (pred > 0.5).float()
correct += (pred == y).float().sum()
return correct / size
def sum_predictions(dataloader, model, model_number, prediction_list):
"""
Predict with the given model and add up the predictions in prediction_list to compute ensemble metrics
"""
for batch, (X, y) in enumerate(dataloader):
if torch.cuda.is_available():
X.cuda()
pred = model(X)
if model_number == 0:
prediction_list.append(pred) # append the predicted tensor for each batch
else:
prediction_list[batch] += pred
# Maybe remove X from GPU (but this is called after training, there should be sufficient space for the data)
return prediction_list
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment