To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit a373657b authored by Lukas Wolf's avatar Lukas Wolf
Browse files

use train test val split and plot the metrics of each model and as ensemble

parent ea970816
......@@ -67,7 +67,7 @@ class Trainer:
X = data['EEG'] # has shape (samples, timesamples, channels)
y = data['labels']
y = y[:,1:] # Remove the subject counter from the labels
#y = y[:,1:] # Remove the subject counter from the labels, do not do this for split
return X, y
from os import path
from torch_models.torch_utils.plot import plot_metrics
import torch
import sys
from torch import nn
import numpy as np
import pandas as pd
from config import config
import logging
from torch_models.torch_utils.training import train_loop, test_loop
from torch_models.torch_utils.training import train_loop, validation_loop
from torch_models.torch_utils.utils import get_gpu_memory
import psutil
from torch_models.torch_utils.utils import timing_decorator
......@@ -96,7 +100,7 @@ class BaseNet(nn.Module):
#@profile
#@timing_decorator
def fit(self, train_dataloader, test_dataloader, subjectID=None):
def fit(self, train_dataloader, validation_dataloader, test_dataloader, subjectID=None):
"""
Fit the model on the dataset defined by data x and labels y
"""
......@@ -109,23 +113,48 @@ class BaseNet(nn.Module):
# Create a history to track ensemble performance
prediction_ensemble = Prediction_history(dataloader=test_dataloader, model=self)
# Train the model
epochs = config['epochs']
epochs = config['epochs']
metrics = {'train_loss':[], 'val_loss':[], 'train_acc':[], 'val_acc':[]}
curr_val_loss = sys.maxsize # For early stopping
patience = 0
for t in range(epochs):
logging.info(f"Epoch {t+1}\n-------------------------------")
# print(f"Start EPOCH: Free GPU memory: {get_gpu_memory()}")
# print(f"memory {psutil.virtual_memory()}")
# Run through training and test set
train_loop(train_dataloader, self.float(), self.loss_fn, optimizer)
train_loss_epoch, train_acc_epoch = train_loop(train_dataloader, self.float(), self.loss_fn, optimizer)
# print(f"Free GPU mem after train loop: {get_gpu_memory()}")
# print(f"memory {psutil.virtual_memory()}")
test_loop(test_dataloader, self.float(), self.loss_fn)
val_loss_epoch, val_acc_epoch = validation_loop(validation_dataloader, self.float(), self.loss_fn)
# Add them for later printout
metrics['train_loss'].append(train_loss_epoch)
metrics['val_loss'].append(val_loss_epoch)
if config['task'] == 'prosaccade-clf':
metrics['train_acc'].append(train_acc_epoch)
metrics['val_acc'].append(val_acc_epoch)
# print("Free GPU mem after test loop:")
# print(f"memory {psutil.virtual_memory()}")
# Add the predictions on the validation set
prediction_ensemble.on_epoch_end()
# print("Free GPU mem after prediction hist:")
# print(f"memory {psutil.virtual_memory()}")
# Done with training this model
# Impementation of early stopping
if config['early_stopping']:
if patience > config['patience']:
logging.info(f"Early stopping the model after {t} epochs")
break
if val_loss_epoch > curr_val_loss:
patience +=1
else:
patience = 0
# Plot metrics and save metrics
plot_metrics(metrics['train_loss'], metrics['val_loss'], output_dir=config['model_dir'], metric='loss', model_number=self.model_number)
if config['task'] == 'prosaccade-clf':
plot_metrics(metrics['train_acc'], metrics['val_acc'], output_dir=config['model_dir'], metric='accuracy', model_number=self.model_number)
# Save metrics
df = pd.DataFrame.from_dict(metrics)
df.to_csv(path_or_buf=config['model_dir'] + '/metrics.csv')
if config['save_models']:
ckpt_dir = config['model_dir'] + '/best_models/' + config['model'] + '_nb_{}_'.format(self.model_number) + 'best_model.pth'
torch.save(self.state_dict(), ckpt_dir)
......
from sklearn.utils import validation
from config import config
import logging
import torch.nn as nn
......@@ -8,6 +9,7 @@ from torch_models.torch_utils.custom_losses import angle_loss
from torch_models.torch_utils.utils import compute_loss, compute_accuracy
from torch_models.torch_utils.plot import plot_array
from torch_models.torch_utils.dataloader import create_dataloader
from utils.data_split import generate_split
from torch_models.CNN.CNN import CNN
from torch_models.InceptionTime.InceptionTime import Inception
......@@ -49,10 +51,19 @@ class Ensemble_torch:
"""
# Create a split
x = np.transpose(x, (0, 2, 1)) # (batch_size, samples, channels) to (bs, ch, samples) as torch conv layers want it
X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=42)
X_train, y_train, X_val, y_val, X_test, y_test = generate_split(x, y)
# Remove the subject counter from the labels
y_train = y_train[:, 1:]
y_val = y_val[:, 1:]
y_test = y_test[:, 1:]
# Create dataloaders
train_dataloader = create_dataloader(X_train, y_train, config['batch_size'], 'train')
test_dataloader = create_dataloader(X_val, y_val, config['batch_size'], 'test')
validation_dataloader = create_dataloader(X_val, y_val, config['batch_size'], 'val')
test_dataloader = create_dataloader(X_test, y_test, config['batch_size'], 'test')
print(f"len train loader {len(train_dataloader.dataset)}")
print(f"len validation loader {len(validation_dataloader.dataset)}")
print(f"len test loader {len(test_dataloader.dataset)}")
# Metrics to save across the ensemble
loss=[]
......@@ -62,7 +73,7 @@ class Ensemble_torch:
logging.info("------------------------------------------------------------------------------------")
logging.info('Start training model number {}/{} ...'.format(i+1, self.nb_models))
model = create_model(model_type=self.model_type, model_number=i)
pred_ensemble = model.fit(train_dataloader, test_dataloader)
pred_ensemble = model.fit(train_dataloader, validation_dataloader, test_dataloader)
# Collect the predictions on the validation sets
if i == 0:
pred = pred_ensemble.predhis
......@@ -85,11 +96,11 @@ class Ensemble_torch:
config['model'] = config['model'] + '_cluster'
# Save metrics and plot them
np.savetxt(X=loss, fname=config['model_dir']+'/'+'ensemble_loss_val.csv', delimiter=',')
plot_array(loss, config['model_dir'], 'loss')
np.savetxt(X=loss, fname=config['model_dir']+'/'+'ensemble_test_loss_val.csv', delimiter=',')
plot_array(loss, config['model_dir'], 'test loss')
if config['task'] == 'prosaccade-clf':
np.savetxt(X=accuracy, fname=config['model_dir']+'/'+'ensemble_acc_val.csv', delimiter=',')
plot_array(accuracy, config['model_dir'], 'accuracy')
np.savetxt(X=accuracy, fname=config['model_dir']+'/'+'ensemble_test_acc_val.csv', delimiter=',')
plot_array(accuracy, config['model_dir'], 'test accuracy')
def create_model(model_type, model_number):
"""
......
......@@ -52,4 +52,24 @@ def plot_array(x, output_dir, metric, savefig=True):
else:
plt.ylabel('Binary Cross Entropy Loss')
if savefig:
plt.savefig(output_dir + '/' + config['model'] + '_val_' + metric + '.png')
\ No newline at end of file
plt.savefig(output_dir + '/' + config['model'] + '_val_' + metric + '.png')
def plot_metrics(train, val, output_dir, metric, model_number=0 ,savefig=True):
"""
Plot the training and validation metric together in one image
"""
logging.info("Plotting training and validation metrics ...")
epochs = np.arange(len(train))
plt.figure()
if config['pretrained']:
plt.title("Pretrained " + config['model'] + " " + metric)
else:
plt.title(config['model'] + " " + str(model_number) + " " + metric)
plt.plot(epochs, np.array(train), 'b-', label='train')
plt.plot(epochs, np.array(val), 'g-', label='validation')
plt.legend()
plt.xlabel('epochs')
plt.ylabel(metric)
if savefig:
plt.savefig(output_dir + '/' + config['model'] + '_' +str(model_number) + "_" + metric + '.png')
\ No newline at end of file
......@@ -16,6 +16,7 @@ def train_loop(dataloader, model, loss_fn, optimizer):
Returns training loss of the epoch to be tracked by the caller
"""
size = len(dataloader.dataset)
print(f"size {size}")
training_loss, correct = 0, 0
for batch, (X, y) in enumerate(dataloader):
# Move tensors to GPU
......@@ -34,19 +35,23 @@ def train_loop(dataloader, model, loss_fn, optimizer):
pred = (pred > 0.5).float()
correct += (pred == y).float().sum()
logging.info(f"Avg training loss: {training_loss/size:>7f}")
if config['task'] == 'prosaccade-clf':
logging.info(f"Avg training accuracy {correct/size:>8f}")
loss = training_loss / size
logging.info(f"Avg training loss: {loss:>7f}")
if config['task'] == 'prosaccade-clf':
accuracy = correct / size
logging.info(f"Avg training accuracy {accuracy:>8f}")
return loss.item(), accuracy.item()
return loss.item(), -1
#@timing_decorator
@profile
def test_loop(dataloader, model, loss_fn):
def validation_loop(dataloader, model, loss_fn):
"""
Performs one prediction run through the test set stored in the dataloader
Prints the loss function computed with the prediction pred and the labels y
"""
size = len(dataloader.dataset)
test_loss, correct = 0, 0
val_loss, correct = 0, 0
with torch.no_grad():
for batch, (X, y) in enumerate(dataloader):
# Move tensors to GPU
......@@ -56,15 +61,16 @@ def test_loop(dataloader, model, loss_fn):
# Predict
pred = model(X)
# Compute metrics
test_loss += loss_fn(pred, y).item()
val_loss += loss_fn(pred, y).item()
if config['task'] == 'prosaccade-clf':
pred = (pred > 0.5).float()
correct += (pred == y).float().sum()
test_loss /= size
logging.info(f"Avg test loss: {test_loss:>8f}")
loss = val_loss / size
logging.info(f"Avg validation loss: {loss:>8f}")
if config['task'] == 'prosaccade-clf':
correct /= size
logging.info(f"Avg test accuracy {correct:>8f}")
accuracy = correct / size
logging.info(f"Avg validation accuracy {accuracy:>8f}")
return loss.item(), accuracy.item()
return loss, -1 # Can be used for early stopping
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment