Commit bbbffb59 authored by Ard Kastrati's avatar Ard Kastrati
Browse files

Merge branch 'master' of https://gitlab.ethz.ch/kard/dl-project

parents 9467452d e4b70d37
......@@ -5,16 +5,6 @@ import logging
from ConvNet import ConvNet
from tensorflow.keras.constraints import max_norm
def run(trainX, trainY):
"""
Starts the CNN and stores the histogram, the plots of loss and accuracy.
"""
logging.info("Starting CNN.")
classifier = Classifier_CNN(input_shape=config['cnn']['input_shape'])
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
class Classifier_CNN(ConvNet):
"""
......@@ -30,4 +20,4 @@ class Classifier_CNN(ConvNet):
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(activation='relu')(x)
x = tf.keras.layers.MaxPool1D()(x)
return x
\ No newline at end of file
return x
from abc import ABC, abstractmethod
from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow.keras as keras
from config import config
from keras.callbacks import CSVLogger
class prediction_history(tf.keras.callbacks.Callback):
def __init__(self, val_data):
self.val_data = val_data
self.predhis = []
self.targets = []
def on_batch_end(self, epoch, logs={}):
x_val, y_val = self.val_data
self.targets.append(y_val)
prediction = self.model.predict(x_val)
self.predhis.append(prediction)
class ConvNet(ABC):
def __init__(self, input_shape, kernel_size=32, nb_filters=32, verbose=True, batch_size=64, use_residual=False, depth=6, preprocessing = False):
def __init__(self, input_shape, kernel_size=32, nb_filters=32, verbose=True, batch_size=64, use_residual=False, depth=6, epochs=2, preprocessing = False):
self.use_residual = use_residual
self.depth = depth
self.callbacks = None
......@@ -16,6 +30,7 @@ class ConvNet(ABC):
self.nb_filters = nb_filters
self.preprocessing = preprocessing
self.input_shape = input_shape
self.epochs = epochs
if config['split']:
self.model = self._split_model()
......@@ -91,5 +106,8 @@ class ConvNet(ABC):
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
hist = self.model.fit(x, y, verbose=1, batch_size=self.batch_size, validation_split=0.2, epochs=1, callbacks=[csv_logger, ckpt, early_stop])
return hist
\ No newline at end of file
X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=42)
pred_ensemble = prediction_history((X_val,y_val))
hist = self.model.fit(X_train, y_train, verbose=1, batch_size=self.batch_size, validation_data=(X_val,y_val),
epochs=self.epochs, callbacks=[csv_logger, ckpt, early_stop,pred_ensemble])
return hist, pred_ensemble
......@@ -5,16 +5,6 @@ import logging
from ConvNet import ConvNet
from tensorflow.keras.constraints import max_norm
def run(trainX, trainY):
"""
Starts the DeepEye and stores the histogram, the plots of loss and accuracy.
"""
logging.info("Starting DeepEye.")
classifier = Classifier_DEEPEYE(input_shape=config['deepeye']['input_shape'])
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
# save_logs(hist, config['model_dir'], config['model'], pytorch=False)
class Classifier_DEEPEYE(ConvNet):
"""
......@@ -99,4 +89,4 @@ class Classifier_DEEPEYE(ConvNet):
x = tf.keras.layers.Concatenate(axis=2)(conv_list)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(activation='relu')(x)
return x
\ No newline at end of file
return x
......@@ -5,16 +5,6 @@ import logging
from ConvNet import ConvNet
from tensorflow.keras.constraints import max_norm
def run(trainX, trainY):
"""
Starts the DeepEye-RNN and stores the histogram, the plots of loss and accuracy.
"""
logging.info("Starting DeepEye with RNN.")
classifier = Classifier_DEEPEYE_RNN(input_shape=config['deepeye-rnn']['input_shape'])
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
class Classifier_DEEPEYE_RNN(ConvNet):
"""
......@@ -69,4 +59,4 @@ class Classifier_DEEPEYE_RNN(ConvNet):
x = tf.keras.layers.Concatenate(axis=2)(conv_list)
x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Activation(activation='relu')(x)
return x
\ No newline at end of file
return x
......@@ -15,16 +15,19 @@ from keras.callbacks import CSVLogger
import numpy as np
import logging
def run(trainX, trainY):
"""
Starts the EEGNet and stores the histogram, the plots of loss and accuracy.
"""
classifier = Classifier_EEGNet()
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
# Newly added lines below
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
from sklearn.model_selection import train_test_split
class prediction_history(tf.keras.callbacks.Callback):
def __init__(self, val_data):
self.val_data = val_data
self.predhis = []
self.targets = []
def on_batch_end(self, epoch, logs={}):
x_val, y_val = self.val_data
self.targets.append(y_val)
prediction = self.model.predict(x_val)
self.predhis.append(prediction)
class Classifier_EEGNet:
"""
......@@ -36,7 +39,7 @@ class Classifier_EEGNet:
def __init__(self, nb_classes=1, chans = config['eegnet']['channels'],
samples = config['eegnet']['samples'], dropoutRate = 0.5, kernLength = 250, F1 = 16,
D = 4, F2 = 256, norm_rate = 0.5, dropoutType = 'Dropout', verbose = True, build = True, X = None):
D = 4, F2 = 256, norm_rate = 0.5, dropoutType = 'Dropout', epochs = 50, verbose = True, build = True, X = None):
self.nb_classes = nb_classes
self.chans = chans
......@@ -48,6 +51,7 @@ class Classifier_EEGNet:
self.F2 = F2
self.norm_rate = norm_rate
self.dropoutType = dropoutType
self.epochs = epochs
self.verbose = verbose
if build:
......@@ -135,5 +139,9 @@ class Classifier_EEGNet:
early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
hist = self.model.fit(eegnet_x, y, verbose=1, validation_split=0.2, epochs=50, callbacks=[csv_logger, ckpt, early_stop])
return hist
X_train, X_val, y_train, y_val = train_test_split(eegnet_x, y, test_size=0.2, random_state=42)
pred_ensemble = prediction_history((X_val,y_val))
hist = self.model.fit(X_train, y_train, verbose=1, validation_data=(X_val,y_val),
epochs=self.epochs, callbacks=[csv_logger, ckpt, early_stop,pred_ensemble])
return hist, pred_ensemble
......@@ -5,18 +5,6 @@ import logging
from ConvNet import ConvNet
def run(trainX, trainY):
"""
Starts the InceptionTime and stores the histogram, the plots of loss and accuracy.
"""
logging.info("Starting InceptionTime.")
classifier = Classifier_INCEPTION(input_shape=config['inception']['input_shape'])
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
class Classifier_INCEPTION(ConvNet):
"""
The InceptionTime architecture used as baseline. This is the architecture explained in the paper
......
This diff is collapsed.
# DL-Project
Predicting eye gaze with DL
Projects that we try: CNN, InceptionTime, EEGNet, DeepEye
Predicting eye gaze direction with DL
Models that we try: CNN, InceptionTime, EEGNet, Xception, DeepEye, and DeepEyeRNN.
## Datasets
## Leonhard Settings
You can download the datasets from this link: https://polybox.ethz.ch/index.php/s/GLe287LxHq3QysW
Please use the command
- 125Hz data: noweEEG.mat
- 500Hz data: all_EEGprocuesan.mat
- labels (0 for left, 1 for right): all_trialinfoprosan.mat
`module load gcc/6.3.0 python_gpu/3.8.5 hdf5/1.10.1`
## Model Configuration
before training the model. Please make sure that the Tensorflow version should be 2.x.
Please configure the config.py file correctly before running the main.py file:
config['data_dir'] : indicates the directory where you stored the data
config['model'] : indicates the model you want to use, choose between 'cnn', 'eegnet', 'inception', 'xception' or'deepeye'
config['downsampled'] : True if you want to use 125 data points per second instead of 500. Default is False
config['split'] : True if you want to run a clustered version of the model, please keep it to False as the clustered version is inneficient
## deepEye3 tuning
nb_filter: [32, 64]
## Parameter Tuning
depth: [9, 12, 20]
Please find the files which are related to our model:
|
| - CNN -- CNN.py
| - DeepEye -- deepeye.py
| - DeepEyeRNN -- deepeyeRNN.py
| - EEGNet -- eegNet.py
| - InceptionTime -- inception.py
| - Xception -- xception.py
| - ConvNet.py
kernel_size:[40, 20]
You can find the architechture of our models in these files. For `CNN`, `DeepEye`, `DeepEyeRNN`, `InceptionTime`, and `Xception`, you should tune the parameter by looking into the `ConvNet.py` file and adjust the parameters (e.g. `self.nb_filter`) accordingly.
residual_jump: [3, 4]
For `EEGNet` model, you should directly look into the `eegNet.py` file and tune the parameters accordingly.
Large depth causes overfitting, same for the number of filters. Kernel size seems to have tiny affect on validation. Residual jump for 4 (i.e. `depth % (res_jump) == (res_jump - 1)`) is not so good in our task, but I think it would be useful for future tasks.
The best setting is **nb_filter == 32, depth == 9, kernel_size == 40, res_jump == 3**
## Running in the Cluster
For [ETH Leonhard](https://scicomp.ethz.ch/wiki/Python_on_Leonhard) users, you can follow these steps:
1. Please use the command `module load gcc/6.3.0 python_gpu/3.8.5 hdf5/1.10.1` before training the model.
2. Edit the _data directory_ and __run directory__ where you _saved the datasets_ and __run the code for saving the logs and outputs__.
3. Use the following command to run the model: `bsub -n 10 -W 4:00 -o experiment_[your_model] -R "rusage[mem=5000, ngpus_excl_p=1]" python ../main.py`
__Note:__ If you want to run the model locally, you should ensure to have Pytorch, Tensorflow==2.x installed.
## Reading the Results
A summary of the model is printed in the `experiment_[your_model]` file generated by the Leonhard cluster.
After each run, a directory with the name `'number'_config['model']` is created under the run directory, where 'number' refers to the time stamp of the training (different for each run of main.py). It contains the weights of the model, a .csv file with the fiting history, the best validation score and an accuracy and loss plot of the training.
......@@ -4,17 +4,6 @@ from utils.utils import *
import logging
from ConvNet import ConvNet
def run(trainX, trainY):
"""
Starts the Xception and stores the histogram, the plots of loss and accuracy.
"""
logging.info("Starting InceptionTime.")
classifier = Classifier_XCEPTION(input_shape=config['inception']['input_shape'])
hist = classifier.fit(trainX, trainY)
plot_loss(hist, config['model_dir'], config['model'], True)
plot_acc(hist, config['model_dir'], config['model'], True)
save_logs(hist, config['model_dir'], config['model'], pytorch=False)
class Classifier_XCEPTION(ConvNet):
"""
......
......@@ -36,19 +36,21 @@ xception: The Xception architecture
deepeye-rnn: The DeepEye RNN architecture
If downsampled set to true, the downsampled data (which have length 125) will be used. Otherwise, full data with length 500 is used.
If split set to true, the data will be clustered and fed each to a separate architecture. The extracted features are concatenated
If split set to true, the data will be clustered and fed each to a separate architecture. The extracted features are concatenated
finally used for classification.
Cluster can be set to clustering(), clustering2() or clustering3(), where different clusters based on literature are used.
"""
# Choosing model
config['model'] = 'inception'
config['model'] = 'deepeye'
config['downsampled'] = False
config['split'] = False
config['cluster'] = clustering2()
config['cluster'] = clustering()
if config['split']:
config['model'] = config['model'] + '_cluster'
config['ensemble'] = 9 #number of models in the ensemble method
config['trainX_file'] = 'noweEEG.mat' if config['downsampled'] else 'all_EEGprocuesan.mat'
config['trainY_file'] = 'all_trialinfoprosan.mat'
config['trainX_variable'] = 'noweEEG' if config['downsampled'] else 'all_EEGprocuesan'
......@@ -79,9 +81,12 @@ config['eegnet']['samples'] = 125 if config['downsampled'] else 500
# Create a unique output directory for this experiment.
timestamp = str(int(time.time()))
model_folder_name = timestamp if config['model'] == '' else timestamp + "_" + config['model']
if config['split']:
model_folder_name += '_cluster'
if config['downsampled']:
model_folder_name += '_downsampled'
if config['ensemble']>1:
model_folder_name += '_ensemble'
config['model_dir'] = os.path.abspath(os.path.join(config['log_dir'], model_folder_name))
if not os.path.exists(config['model_dir']):
......@@ -116,4 +121,4 @@ config['deepeye']['trainX_filename'] = "EEGprocue"
config['deepeye']['trainY_variable1'] = "trialinfopro"
config['deepeye']['trainY_variable2'] = "cues"
config['deepeye']['trainY_filename'] = "trialinfocuelocked"
\ No newline at end of file
config['deepeye']['trainY_filename'] = "trialinfocuelocked"
import tensorflow as tf
from config import config
from utils.utils import *
import logging
from CNN.CNN import Classifier_CNN
from DeepEye.deepeye import Classifier_DEEPEYE
from DeepEyeRNN.deepeyeRNN import Classifier_DEEPEYE_RNN
from Xception.xception import Classifier_XCEPTION
from InceptionTime.inception import Classifier_INCEPTION
from EEGNet.eegNet import Classifier_EEGNet
import numpy as np
def run(trainX, trainY):
"""
Starts the multiples Classifier in the Ensemble and stores the histogram, the plots of loss and accuracy.
validation is of the ensemble model and training just the last one
"""
logging.info("Started running "+config['model']+". If you want to run other methods please choose another model in the config.py file.")
# acc = tf.keras.metrics.BinaryAccuracy()
bce = tf.keras.losses.BinaryCrossentropy()
loss=[]
accuracy=[]
for i in range(config['ensemble']):
print('beginning model number {}/{} ...'.format(i,config['ensemble']))
if config['model'] == 'deepeye':
classifier = Classifier_DEEPEYE(input_shape = config['deepeye']['input_shape'])
elif config['model'] == 'cnn':
classifier = Classifier_CNN(input_shape = config['cnn']['input_shape'])
elif config['model'] == 'eegnet':
classifier = Classifier_EEGNet()
elif config['model'] == 'inception':
classifier = Classifier_INCEPTION(input_shape=config['inception']['input_shape'])
elif config['model'] == 'xception' :
classifier = Classifier_XCEPTION(input_shape=config['inception']['input_shape'])
elif config['model'] == 'deepeye-rnn':
classifier = Classifier_DEEPEYE_RNN(input_shape=config['deepeye-rnn']['input_shape'])
else:
logging.info('Cannot start the program. Please choose one model in the config.py file')
hist, pred_ensemble = classifier.fit(trainX,trainY)
if i == 0:
pred = pred_ensemble.predhis
else:
for j, pred_epoch in enumerate(pred_ensemble.predhis):
pred[j] = (np.array(pred[j])+np.array(pred_epoch))
for j, pred_epoch in enumerate(pred):
pred_epoch = (pred_epoch/config['ensemble']).tolist()
loss.append(bce(pred_ensemble.targets[j],pred_epoch).numpy())
pred_epoch = np.round(pred_epoch,0)
accuracy.append(np.mean((np.array(pred_epoch).reshape(-1)+np.array(pred_ensemble.targets[j]).reshape(-1)-1)**2))
if config['ensemble']>1:
config['model']+='_ensemble'
if config['split']:
config['model'] = config['model'] + '_cluster'
hist.history['val_loss'] = loss
hist.history['val_accuracy'] = accuracy
plot_loss(hist, config['model_dir'], config['model'], val = True)
plot_acc(hist, config['model_dir'], config['model'], val = True)
save_logs(hist, config['model_dir'], config['model'], pytorch = False)
from config import config
import time
from CNN import CNN
from utils import IOHelper
from DeepEye import deepeye
from DeepEyeRNN import deepeyeRNN
from Xception import xception
from InceptionTime import inception
from EEGNet import eegNet
from ensemble import run
import numpy as np
import scipy
from scipy import io
import h5py
import logging
import time
def main():
logging.basicConfig(filename=config['info_log'], level=logging.INFO)
......@@ -18,36 +14,10 @@ def main():
# try:
trainX, trainY = IOHelper.get_mat_data(config['data_dir'], verbose=True)
if config['model'] == 'cnn' or config['model'] == 'cnn_cluster':
logging.info("Started running CNN. If you want to run other methods please choose another model in the config.py file.")
CNN.run(trainX, trainY)
elif config['model'] == 'eegnet' or config['model'] == 'eegnet_cluster':
logging.info(
"Started running EEGNet. If you want to run other methods please choose another model in the config.py file.")
eegnet_x = np.transpose(trainX, (0, 2, 1))
logging.info(eegnet_x.shape)
eegNet.run(trainX=eegnet_x, trainY=trainY)
elif config['model'] == 'inception' or config['model'] == 'inception_cluster':
logging.info("Started running InceptionTime. If you want to run other methods please choose another model in the config.py file.")
inception.run(trainX=trainX, trainY=trainY)
elif config['model'] == 'xception' or config['model'] == 'xception_cluster':
logging.info("Started running XceptionTime. If you want to run other methods please choose another model in the config.py file.")
xception.run(trainX=trainX, trainY=trainY)
elif config['model'] == 'deepeye' or config['model'] == 'deepeye_cluster':
logging.info("Started running DeepEye. If you want to run other methods please choose another model in the config.py file.")
deepeye.run(trainX=trainX, trainY=trainY)
elif config['model'] == 'deepeye-rnn' or config['model'] == 'deepeye-rnn_cluster':
logging.info("Started running deepeye-rnn. If you want to run other methods please choose another model in the config.py file.")
deepeyeRNN.run(trainX=trainX, trainY=trainY)
else:
logging.info('Cannot start the program. Please choose one model in the config.py file')
if config['model'] == 'eegnet' or config['model'] == 'eegnet_cluster':
trainX = np.transpose(trainX, (0, 2, 1))
logging.info(trainX.shape)
run(trainX,trainY)
logging.info("--- Runtime: %s seconds ---" % (time.time() - start_time))
logging.info('Finished Logging')
......
Accuracy,Loss
0.5900683403015137,0.6709842085838318
loss,accuracy,val_loss,val_accuracy
0.6709842085838318,0.5900683403015137,0.6837942004203796,0.6404416561126709
best_model_train_loss,best_model_val_loss,best_model_train_acc,best_model_val_acc
0.706407904624939,0.6844089031219482,0.5264661908149719,0.5702930688858032
Accuracy,Loss
0.5264661908149719,0.706407904624939
loss,accuracy,val_loss,val_accuracy
0.706407904624939,0.5264661908149719,0.6844089031219482,0.5702930688858032
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment