To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 4214a425 authored by Lukas Wolf's avatar Lukas Wolf
Browse files

push for merge

parent fe5ce3df
......@@ -88,7 +88,7 @@ with open('hyperparams.json', 'r') as file:
params = json.load(file)
config['learning_rate'] = params[config['model']][config['task']]['learning_rate']
config['regularization'] = params[config['model']][config['task']]['regularization']
config['epochs'] = 2
config['epochs'] = 50
config['batch_size'] = 64
config['early_stopping'] = True
config['patience'] = 10
......@@ -127,6 +127,8 @@ def create_folder(config):
model_folder_name += "_pretrained_" + config['model'] if config['pretrained'] else "_" + config['model']
# Modify the model folder name depending on which task tuns
model_folder_name += "_" + config['task']
if config['task'] == 'angle-reg' or config['task'] == 'amplitude-reg':
model_folder_name += "_" + config['dataset']
model_folder_name += "_prep" + config['preprocessing']
model_folder_name += f"_lr_{str(config['learning_rate'])}"
......
......@@ -32,10 +32,10 @@ def main():
"""
# Create trainer that runs ensemble of models
benchmark_task('prosaccade-clf')
#benchmark_task('prosaccade-clf')
benchmark_task('gaze-reg')
benchmark_task('angle-reg')
benchmark_task('amplitude-reg')
#benchmark_task('angle-reg')
#benchmark_task('amplitude-reg')
# select_best_model()
# comparison_plot_loss()
......@@ -60,6 +60,7 @@ def benchmark_task(task):
#config['dataset'] = 'processing_speed_task'
# MODELS FOR BENCHMARK
config['model'] = 'cnn'
create_folder(config)
logging.basicConfig(filename=config['info_log'], level=logging.INFO)
......@@ -100,6 +101,7 @@ def benchmark_task(task):
logging.info("--- Runtime: %s seconds ---" % (time.time() - start_time))
start_time = time.time()
config['model'] = 'pyramidal_cnn'
create_folder(config)
logging.basicConfig(filename=config['info_log'], level=logging.INFO)
......
......@@ -4,7 +4,7 @@
#SBATCH --output=log/%j.out # where to store the output (%j is the JOBID), subdirectory must exist
#SBATCH --error=log/%j.err # where to store error messages
#SBATCH --gres=gpu:1
#SBATCH --mem=50G
#SBATCH --mem=60G
echo "Running on host: $(hostname)"
echo "In directory: $(pwd)"
......
......@@ -151,7 +151,7 @@ class BaseNet(nn.Module):
# print("Free GPU mem after prediction hist:")
# print(f"memory {psutil.virtual_memory()}")
# Impementation of early stopping
if config['early_stopping']:
if config['early_stopping'] and not self.early_stopped:
if patience > config['patience']:
logging.info(f"Early stopping the model after {t} epochs")
self.early_stopped = True
......
......@@ -18,7 +18,7 @@ def train_loop(dataloader, model, loss_fn, optimizer):
"""
#get_gpu_memory()
#print(torch.cuda.memory_summary())
size = len(dataloader.dataset)
size = len(dataloader)
training_loss, correct = 0, 0
for batch, (X, y) in enumerate(dataloader):
# Move tensors to GPU
......@@ -28,14 +28,6 @@ def train_loop(dataloader, model, loss_fn, optimizer):
# Compute prediction and loss
pred = model(X)
loss = loss_fn(pred, y)
print(f"y vs. pred")
cat = torch.cat((y, pred), dim=1)
print(cat)
print(f"Loss ")
loss_tensor = torch.square(torch.abs(torch.atan2(torch.sin(y - pred), torch.cos(y - pred))))
print(loss_tensor)
# Backpropagation and optimization
optimizer.zero_grad()
loss.backward()
......@@ -50,6 +42,8 @@ def train_loop(dataloader, model, loss_fn, optimizer):
del y
torch.cuda.empty_cache()
#print(f" training loss after epoch: {training_loss}, divided by #batches ({size}): {training_loss/size}")
loss = training_loss / size
logging.info(f"Avg training loss: {loss:>7f}")
if config['task'] == 'prosaccade-clf':
......@@ -69,7 +63,9 @@ def validation_loop(dataloader, model, loss_fn):
#get_gpu_memory()
#print(torch.cuda.memory_summary())
num_batches = len(dataloader)
size = len(dataloader.dataset)
val_loss, correct = 0, 0
with torch.no_grad():
for batch, (X, y) in enumerate(dataloader):
......@@ -89,7 +85,7 @@ def validation_loop(dataloader, model, loss_fn):
del y
torch.cuda.empty_cache()
loss = val_loss / size
loss = val_loss / num_batches
logging.info(f"Avg validation loss: {loss:>8f}")
if config['task'] == 'prosaccade-clf':
accuracy = correct / size
......
......@@ -4,6 +4,7 @@ import os
import time
from config import config
from memory_profiler import profile
from torch_models.torch_utils.custom_losses import angle_loss
def get_gpu_memory():
"""
......@@ -44,15 +45,18 @@ def compute_loss(loss_fn, dataloader, pred_list, nb_models):
Output:
Scalar value of the mean loss over all batches of the validation set
"""
loss = []
loss = 0
size = len(dataloader)
for batch, (X, y) in enumerate(dataloader):
if torch.cuda.is_available():
y = y.cuda()
# Average the predictions from the ensemble
pred = pred_list[batch]
pred = torch.div(pred, nb_models).float() # is already on gpu
loss.append(loss_fn(y, pred) / config['batch_size'])
return sum(loss) / len(loss)
loss += loss_fn(y, pred)
return loss / size
#@profile
def compute_accuracy(dataloader, pred_list, nb_models):
......@@ -80,7 +84,6 @@ def compute_accuracy(dataloader, pred_list, nb_models):
correct += (pred == y).float().sum()
return correct / size
def sum_predictions(dataloader, model, model_number, prediction_list):
"""
Predict with the given model and add up the predictions in prediction_list to compute ensemble metrics
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment