Commit 2277e505 authored by Lukas Wolf's avatar Lukas Wolf
Browse files

transfer

parent 435a1428
......@@ -13,4 +13,5 @@ notebook.ipynb
models_scratch.py
/log/*
/images/*
/archive_runs/*
\ No newline at end of file
/archive_runs/*
run.sh
\ No newline at end of file
......@@ -169,6 +169,11 @@ class Siamese_ConvNet(ABC):
return self.model
def fit(self, x, y, verbose=2):
# Prepare the data as tuples for the siamese input
X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=42)
X_train_sac, X_train_fix = X_train[:, :config['max_saccade'], :], X_train[:, config['max_saccade']:, :] # first elements are saccade signal, second fixation
X_val_sac, X_val_fix = X_val[:, :config['max_saccade'], :], X_val[:, config['max_saccade']:, :]
# Define callbacks
csv_logger = CSVLogger(config['batches_log'], append=True, separator=';')
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)
......@@ -176,11 +181,6 @@ class Siamese_ConvNet(ABC):
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_loss', save_best_only=True, mode='auto')
prediction_ensemble = prediction_history(((X_val_sac, X_val_fix), y_val))
# Prepare the data as tuples for the siamese input
X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=42)
X_train_sac, X_train_fix = X_train[:, :config['max_saccade'], :], X_train[:, config['max_saccade']:, :] # first elements are saccade signal, second fixation
X_val_sac, X_val_fix = X_val[:, :config['max_saccade'], :], X_val[:, config['max_saccade']:, :]
logging.info("Saccade siamese input shape: {}".format(X_train_sac[0].shape))
logging.info("Fixation siamese input shape: {}".format(X_train_fix[0].shape))
......
......@@ -42,9 +42,9 @@ If split set to true, the data will be clustered and fed each to a separate arch
finally used for classification.
Cluster can be set to clustering(), clustering2() or clustering3(), where different clusters based on literature are used.
"""
# Choose experiment TODO: create config['experiment'] which can be chosen like the model
# Choose experiment TODO: create config['experiment'] which can be chosen like the model via commenting out
config['gaze-reg'] = False # Set to False if you want to run the saccade classification task
config['prosaccade'] = True
config['prosaccade'] = False
config['calibration-task'] = False
# Choose how much data to use on gaze-reg
......@@ -53,18 +53,18 @@ config['data-fraction'] = 1.0 # Set to 1.0 if you want to use the whole dataset,
# Hyper-parameters and training configuration.
config['learning_rate'] = 1e-4 # fix only: 1e-2, sac only: 1e-3, sac_fix: 1e-3 , fix_sac_fix: 1e-4
config['regularization'] = 5 # fix only: 1e-3, sac only: 1e-2, sac_fix: 1, fix_sac_fix: 5
config['epochs'] = 5
config['epochs'] = 1
config['batch_size'] = 64
# Choose which dataset to run the gaze regression on
#config['data_mode'] = 'fix_only'
#config['data_mode'] = 'sacc_only'
config['data_mode'] = 'fix_only'
#onfig['data_mode'] = 'sacc_only'
#config['data_mode'] = 'sacc_fix'
config['data_mode'] = 'fix_sacc_fix'
#config['data_mode'] = 'fix_sacc_fix'
# Choose model
config['model'] = 'cnn'
#config['model'] = 'inception'
#config['model'] = 'cnn'
config['model'] = 'inception'
#config['model'] = 'eegnet'
#config['model'] = 'deepeye'
#config['model'] = 'xception'
......@@ -74,14 +74,14 @@ config['model'] = 'cnn'
# Choose the kerastuner or an ensemble of models
#config['run'] = 'kerastuner'
config['run'] = 'ensemble'
config['ensemble'] = 5 #number of models in the ensemble method
config['ensemble'] = 1 #number of models in the ensemble method
# Other functions that can be chosen optionally
config['sanity_check'] = False
config['plot_filters'] = False #TODO: fix the plot_filters function
config['plot_model'] = True
# Set loss automatically depending on the dataset/task to run
if config['data_mode'] == 'fix_sacc_fix':
if config['data_mode'] == 'fix_sacc_fix' and config['gaze-reg']:
from utils.losses import angle_loss
config['loss'] = angle_loss
else:
......
This source diff could not be displayed because it is too large. You can view the blob instead.
import tensorflow as tf
import numpy as np
from config import config
from utils.utils import *
from utils.plot import plot_model
import logging
from CNN.CNN import Classifier_CNN
......@@ -11,8 +13,6 @@ from DeepEyeRNN.deepeyeRNN import Classifier_DEEPEYE_RNN
from Xception.Xception import Classifier_XCEPTION
from InceptionTime.Inception import Classifier_INCEPTION
from EEGNet.eegNet import Classifier_EEGNet
import numpy as np
def run(trainX, trainY):
......@@ -59,6 +59,9 @@ def run(trainX, trainY):
for j, pred_epoch in enumerate(pred_ensemble.predhis):
pred[j] = (np.array(pred[j])+np.array(pred_epoch))
if config['plot_model']:
plot_model(classifier.model)
for j, pred_epoch in enumerate(pred):
pred_epoch = (pred_epoch/config['ensemble']).tolist()
loss.append(bce(pred_ensemble.targets,pred_epoch).numpy())
......
......@@ -2,7 +2,7 @@ import tensorflow as tf
from config import config
from utils.utils import *
from utils.losses import angle_loss
from utils.plot import plot_filters
from utils.plot import plot_model
import logging
from CNN.Regression_CNN import Regression_CNN
......@@ -27,13 +27,11 @@ def run(trainX, trainY):
logging.info("Started running " + config['model'] + ". If you want to run other methods please choose another model in the config.py file.")
# Metrics
acc = tf.keras.metrics.BinaryAccuracy()
mse = tf.keras.losses.MeanSquaredError()
hist = None
reg = None
loss = []
accuracy = []
for i in range(config['ensemble']):
print('Beginning model number {}/{} ...'.format(i+1, config['ensemble']))
......@@ -86,13 +84,13 @@ def run(trainX, trainY):
if i == 0:
# store the prediction on the validation set of the first model in pred
pred = pred_ensemble.predhis
# Plot the filters of the conv modules only once for the first model
if config['plot_filters']:
plot_filters(reg.model, config['model_dir'])
# Save the targets of the validation set, only once for first model
targets_val_fname = config['model_dir'] + "/" + "targets_val.txt"
logging.info("targets shape: {}".format(pred_ensemble.targets.shape))
np.savetxt(targets_val_fname, pred_ensemble.targets, delimiter=',')
#targets_val_fname = config['model_dir'] + "/" + "targets_val.txt"
#logging.info("targets shape: {}".format(pred_ensemble.targets.shape))
#np.savetxt(targets_val_fname, pred_ensemble.targets, delimiter=',')
# Plot the model
if config['plot_model']:
plot_model(reg.model)
else:
for j, pred_epoch in enumerate(pred_ensemble.predhis):
# add up the resuls of the different models for each epoch into pred, which is a list of lists (one for each epoch prediction)
......@@ -107,10 +105,6 @@ def run(trainX, trainY):
else:
loss.append(mse(pred_ensemble.targets, pred_epoch).numpy())
if not config['gaze-reg']:
# Compute loss for the classification task
accuracy.append(np.mean((np.array(pred_epoch).reshape(-1) + np.array(pred_ensemble.targets).reshape(-1) - 1)**2))
pred_epoch = np.round(pred_epoch, 0) # round to integral number
# save the ensemble loss to the model directory
......@@ -118,9 +112,9 @@ def run(trainX, trainY):
np.savetxt(loss_fname, loss, delimiter=',')
# save the validation predictions to the model directory
pred_val_fname = config['model_dir'] + "/" + "pred_val.txt"
logging.info("prediction val shape: {}".format(pred[-1]))
np.savetxt(pred_val_fname, pred[-1], delimiter=',') # this is the prediction in the last epoch
#pred_val_fname = config['model_dir'] + "/" + "pred_val.txt"
#logging.info("prediction val shape: {}".format(pred[-1]))
#np.savetxt(pred_val_fname, pred[-1], delimiter=',') # this is the prediction in the last epoch
if(config['ensemble'] == 1):
......
......@@ -4,7 +4,7 @@
#SBATCH --output=log/%j.out # where to store the output (%j is the JOBID), subdirectory must exist
#SBATCH --error=log/%j.err # where to store error messages
#SBATCH --gres=gpu:1
#SBATCH --mem=60G
#SBATCH --mem=40G
echo "Running on host: $(hostname)"
echo "In directory: $(pwd)"
......
......@@ -32,6 +32,10 @@ def get_mat_data(data_dir, verbose=True):
if verbose:
logging.info(X.shape)
logging.info(y.shape)
np.save("./data/precomputed/prosaccade/part_prosaccade_X", X[:500])
np.save("./data/precomputed/prosaccade/part_prosaccade_y", y[:500])
return X, y
......
......@@ -4,6 +4,7 @@ import pandas as pd
from tensorflow import keras
from matplotlib.ticker import FormatStrFormatter
import os
from config import config
def plot_batches_log_loss(model_name):
......@@ -62,3 +63,7 @@ def plot_filters(model, model_dir):
fig.savefig(path + fname, facecolor='white', edgecolor='none')
plt.close()
def plot_model(model, dir=config['model_dir'], show_shapes=True):
pathname = dir + "/model_plot.png"
keras.utils.plot_model(model, to_file=pathname, show_shapes=show_shapes)
......@@ -73,13 +73,13 @@ def get_fix_data(verbose=True):
X = np.array(f[config['trainX_variable']]).T
# Compute how much of the data we want to preprocess and return
testsize = int(config['data-fraction'] * len(X))
#testsize = int(config['data-fraction'] * len(X))
x_list = []
y_list = []
# Run through the data
#for i in tqdm(range(testsize), desc='Loading regression data'):
for i in range(testsize):
for i in range(len(X)):
# Read the datapoint from X and check how long the duration is
ref = X[i][0]
x_datapoint = np.array(f[ref])
......@@ -102,9 +102,8 @@ def get_fix_data(verbose=True):
else:
raise Exception("Choose a valid padding scheme in config.py")
x_list.append([x_datapoint])
y_list.append([y_datapoint])
x_list.append(x_datapoint)
y_list.append(y_datapoint)
X = np.asarray(x_list)
# Reshape data and normalize it
norm = np.linalg.norm(X)
......@@ -118,7 +117,9 @@ def get_fix_data(verbose=True):
logging.info("X training loaded.")
logging.info(X.shape)
#np.savetxt("fix_only_y", y, delimiter=',')
# Save the precomputed data for future usage
#np.save("./data/precomputed/fix_only_X", X)
#np.save("./data/precomputed/fix_only_y", y)
return X, y
......@@ -132,7 +133,7 @@ def get_sacc_data(verbose=True):
event_names = [L_saccade, R_saccade]
# Loop over all directories in /data/full_data and extract and concat the events from all people
rootdir = './data/full_data' # modify it if necessary
rootdir = './data/processing_speed_task_full_data' # modify it if necessary
x_list = []
y_list = []
......@@ -192,7 +193,9 @@ def get_sacc_data(verbose=True):
logging.info("X training loaded.")
logging.info(X.shape)
#np.savetxt("sacc_only_y", y, delimiter=',')
# Save the precomputed data for future usage
#np.save("./data/precomputed/sacc_only_X", X)
#np.save("./data/precomputed/sacc_only_y", y)
return X, y
......@@ -211,7 +214,7 @@ def get_sacc_fix_data(verbose=True):
saccades = [L_saccade, R_saccade]
# Loop over all directories in /data/full_data and extract and concat the events from all people
rootdir = './data/full_data' # modify it if necessary
rootdir = './data/processing_speed_task_full_data' # modify it if necessary
x_list = []
y_list = []
......@@ -303,7 +306,9 @@ def get_sacc_fix_data(verbose=True):
logging.info("X training loaded.")
logging.info(X.shape)
#np.savetxt("sacc_fix_y", y, delimiter=',')
# Save the precomputed data for future usage
#np.save("./data/precomputed/sacc_fix_X", X)
#np.save("./data/precomputed/sacc_fix_y", y)
return X, y
......@@ -323,7 +328,164 @@ def get_fix_sacc_fix_data(verbose=True):
saccades = [L_saccade, R_saccade]
# Loop over all directories in /data/full_data and extract and concat the events from all people
rootdir = './data/full_data' # modify it if necessary
rootdir = './data/processing_speed_task_full_data' # modify it if necessary
x_list = []
y_list = []
for subdir, dirs, files in os.walk(rootdir):
for file in files:
# Get the correct path to the current file
path = os.path.join(subdir, file)
events = load_sEEG_events(path) # access event i via events[i]
data = load_sEEG_data(path)
# We now look for fixation-sacade-fixation triplets without blinks or other actions inbetween
# Extract X and y from sEEG.data and sEEG.events
for i in range(len(events)):
first_event = events[i]
first_event_name = first_event[0][0] # dereference the event name, e.g. 'L_saccade'
if first_event_name not in fixations:
continue # we first want a saccade
# We found a fixation as first event, now look for a following saccade
second_event = events[i + 1]
second_event_name = second_event[0][0]
if second_event_name not in saccades:
continue
# We found a following saccade, now look for a following fixation again
third_event = events[i + 2]
third_event_name = third_event[0][0]
if third_event_name not in fixations:
continue
# We finally found a triplet (segment) without interferences of blinks or pushing buttons
fixation1 = first_event
saccade = second_event
fixation2 = third_event
# Pad the first fixation
fixation1_start_time = int(fixation1[1])
fixation1_end_time = int(fixation1[4])
fixation1_datapoint = np.array(data[fixation1_start_time:fixation1_end_time])
x_len_fix1, y_len_fix1 = fixation1_datapoint.shape
if x_len_fix1 < config['min_fixation'] or x_len_fix1 > config['max_fixation']:
continue
fixation1_padding_size = config['max_fixation'] - x_len_fix1
if config['padding'] == 'zero':
fixation1_datapoint = np.pad(fixation1_datapoint, pad_width=((0,fixation1_padding_size),(0,0)))
elif config['padding'] == 'repeat':
fixation1_datapoint = np.pad(fixation1_datapoint, pad_width=((0,fixation1_padding_size),(0,0)), mode='reflect')
else:
raise Exception("Choose a valid padding scheme in config.py")
# Pad the saccade
saccade_start_time = int(saccade[1])
saccade_end_time = int(saccade[4])
saccade_datapoint = np.array(data[saccade_start_time:saccade_end_time])
x_len_sac, y_len_sac = saccade_datapoint.shape
if x_len_sac < config['min_saccade'] or x_len_sac > config['max_saccade']:
continue
saccade_padding_size = config['max_saccade'] - x_len_sac
if config['padding'] == 'zero':
saccade_datapoint = np.pad(saccade_datapoint, pad_width=((0,saccade_padding_size),(0,0)))
elif config['padding'] == 'repeat':
saccade_datapoint = np.pad(saccade_datapoint, pad_width=((0,saccade_padding_size),(0,0)), mode='reflect')
else:
raise Exception("Choose a valid padding scheme in config.py")
# Pad the second fixation
fixation2_start_time = int(fixation2[1])
fixation2_end_time = int(fixation2[4])
fixation2_datapoint = np.array(data[fixation2_start_time:fixation2_end_time])
x_len_fix2, y_len_fix2 = fixation2_datapoint.shape
if x_len_fix2 < config['min_fixation'] or x_len_fix2 > config['max_fixation']:
continue
fixation2_padding_size = config['max_fixation'] - x_len_fix2
if config['padding'] == 'zero':
fixation2_datapoint = np.pad(fixation2_datapoint, pad_width=((0,fixation2_padding_size),(0,0)))
elif config['padding'] == 'repeat':
fixation2_datapoint = np.pad(fixation2_datapoint, pad_width=((0,fixation2_padding_size),(0,0)), mode='reflect')
else:
raise Exception("Choose a valid padding scheme in config.py")
# Stack the fixation, saccade and fixation as one datapoint
x_datapoint = np.concatenate((fixation1_datapoint, saccade_datapoint), axis=0)
x_datapoint = np.concatenate((x_datapoint, fixation2_datapoint), axis=0)
# Extract the difference vector of labels of the two fixations
fix1_avg_x = float(fixation1[11])
fix1_avg_y = float(fixation1[12])
fix2_avg_x = float(fixation2[11])
fix2_avg_y = float(fixation2[12])
# Compute difference of the fixation coordinates
dx = fix2_avg_x - fix1_avg_x
dy = fix2_avg_y - fix1_avg_y
# Compute polar coordinates of the difference vector
rho, phi = cart2pol(dx, dy)
#y_datapoint = np.array([rho, phi])
#y_datapoint = np.array([dx, dy])
y_datapoint = np.array([phi])
# Append to X and y
x_list.append(x_datapoint)
y_list.append(y_datapoint)
X = np.asarray(x_list)
X = X[:,:,:129] # Cut off the last 4 columns (time, x, y, pupil size)
# Normalize the data
norm = np.linalg.norm(X)
X = X / norm
y = np.asarray(y_list)
if verbose:
logging.info("y training loaded.")
logging.info(y.shape)
logging.info("X training loaded.")
logging.info(X.shape)
# Save the precomputed data for future usage
#np.save("./data/precomputed/fix_sacc_fix_X", X)
#np.save("./data/precomputed/fix_sacc_fix_y", y)
return X, y
def get_calibration_task_fix_sacc_fix_data(verbose=True):
"""
Returns X, y for the gaze regression task, specifically the calibration task dataset, with EEG data X only from triplets of fixation-saccade-fixation
"""
L_saccade = 'L_saccade'
L_fixation = 'L_fixation'
L_blink = 'L_blink'
R_saccade = 'R_saccade'
R_fixation = 'R_fixation'
R_blink = 'R_blink'
blockOn = '55'
blockOff = '56'
current_block = 0
block_list = ['TarPos', 'TarPosBack', 'TarPosFlip', 'TarPosFlipBack', 'TarPos']
fixations = [L_fixation, R_fixation]
saccades = [L_saccade, R_saccade]
blinks = [L_blink, R_blink]
# Loop over all directories and extract and concat the events from all people
rootdir = './data/calibration_task' # modify it if necessary
x_list = []
y_list = []
......@@ -453,7 +615,9 @@ def get_fix_sacc_fix_data(verbose=True):
logging.info("X training loaded.")
logging.info(X.shape)
#np.savetxt("fix_sacc_fix_y", y, delimiter=',')
# Save the precomputed data for future usage
#np.save("./data/precomputed/calibration_task/fix_sacc_fix_X", X)
#np.save("./data/precomputed/calibration_task/fix_sacc_fix_y", y)
return X, y
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment