Merge branch 'master' of https://gitlab.ethz.ch/kard/dl-project

CNN/CNN.py

@@ -5,16 +5,6 @@ import logging
 from ConvNet import ConvNet
 from tensorflow.keras.constraints import max_norm
 
-def run(trainX, trainY):
-    """
-    Starts the CNN and stores the histogram, the plots of loss and accuracy.
-    """
-    logging.info("Starting CNN.")
-    classifier = Classifier_CNN(input_shape=config['cnn']['input_shape'])
-    hist = classifier.fit(trainX, trainY)
-    plot_loss(hist, config['model_dir'], config['model'], True)
-    plot_acc(hist, config['model_dir'], config['model'], True)
-    save_logs(hist, config['model_dir'], config['model'], pytorch=False)
 
 class Classifier_CNN(ConvNet):
     """
@@ -30,4 +20,4 @@ class Classifier_CNN(ConvNet):
         x = tf.keras.layers.BatchNormalization()(x)
         x = tf.keras.layers.Activation(activation='relu')(x)
         x = tf.keras.layers.MaxPool1D()(x)
-        return x
\ No newline at end of file
+        return x

ConvNet.py

 from abc import ABC, abstractmethod
 
+from sklearn.model_selection import train_test_split
 import tensorflow as tf
 import tensorflow.keras as keras
 from config import config
 from keras.callbacks import CSVLogger
 
+class prediction_history(tf.keras.callbacks.Callback):
+    def __init__(self, val_data):
+        self.val_data = val_data
+        self.predhis = []
+        self.targets = []
+
+    def on_batch_end(self, epoch, logs={}):
+        x_val, y_val = self.val_data
+        self.targets.append(y_val)
+        prediction = self.model.predict(x_val)
+        self.predhis.append(prediction)
+
+
 class ConvNet(ABC):
-    def __init__(self, input_shape, kernel_size=32, nb_filters=32, verbose=True, batch_size=64, use_residual=False, depth=6, preprocessing = False):
+    def __init__(self, input_shape, kernel_size=32, nb_filters=32, verbose=True, batch_size=64, use_residual=False, depth=6, epochs=2, preprocessing = False):
         self.use_residual = use_residual
         self.depth = depth
         self.callbacks = None
@@ -16,6 +30,7 @@ class ConvNet(ABC):
         self.nb_filters = nb_filters
         self.preprocessing = preprocessing
         self.input_shape = input_shape
+        self.epochs = epochs
 
         if config['split']:
             self.model = self._split_model()
@@ -91,5 +106,8 @@ class ConvNet(ABC):
         early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
         ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
         ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
-        hist = self.model.fit(x, y, verbose=1, batch_size=self.batch_size, validation_split=0.2, epochs=1, callbacks=[csv_logger, ckpt, early_stop])
-        return hist
\ No newline at end of file
+        X_train, X_val, y_train, y_val = train_test_split(x, y, test_size=0.2, random_state=42)
+        pred_ensemble = prediction_history((X_val,y_val))
+        hist = self.model.fit(X_train, y_train, verbose=1, batch_size=self.batch_size, validation_data=(X_val,y_val),
+        epochs=self.epochs, callbacks=[csv_logger, ckpt, early_stop,pred_ensemble])
+        return hist, pred_ensemble

DeepEye/deepeye.py

@@ -5,16 +5,6 @@ import logging
 from ConvNet import ConvNet
 from tensorflow.keras.constraints import max_norm
 
-def run(trainX, trainY):
-    """
-    Starts the DeepEye and stores the histogram, the plots of loss and accuracy.
-    """
-    logging.info("Starting DeepEye.")
-    classifier = Classifier_DEEPEYE(input_shape=config['deepeye']['input_shape'])
-    hist = classifier.fit(trainX, trainY)
-    plot_loss(hist, config['model_dir'], config['model'], True)
-    plot_acc(hist, config['model_dir'], config['model'], True)
-    # save_logs(hist, config['model_dir'], config['model'], pytorch=False)
 
 class Classifier_DEEPEYE(ConvNet):
     """
@@ -99,4 +89,4 @@ class Classifier_DEEPEYE(ConvNet):
         x = tf.keras.layers.Concatenate(axis=2)(conv_list)
         x = tf.keras.layers.BatchNormalization()(x)
         x = tf.keras.layers.Activation(activation='relu')(x)
-        return x
\ No newline at end of file
+        return x

DeepEyeRNN/deepeyeRNN.py

@@ -5,16 +5,6 @@ import logging
 from ConvNet import ConvNet
 from tensorflow.keras.constraints import max_norm
 
-def run(trainX, trainY):
-    """
-    Starts the DeepEye-RNN and stores the histogram, the plots of loss and accuracy.
-    """
-    logging.info("Starting DeepEye with RNN.")
-    classifier = Classifier_DEEPEYE_RNN(input_shape=config['deepeye-rnn']['input_shape'])
-    hist = classifier.fit(trainX, trainY)
-    plot_loss(hist, config['model_dir'], config['model'], True)
-    plot_acc(hist, config['model_dir'], config['model'], True)
-    save_logs(hist, config['model_dir'], config['model'], pytorch=False)
 
 class Classifier_DEEPEYE_RNN(ConvNet):
     """
@@ -69,4 +59,4 @@ class Classifier_DEEPEYE_RNN(ConvNet):
         x = tf.keras.layers.Concatenate(axis=2)(conv_list)
         x = tf.keras.layers.BatchNormalization()(x)
         x = tf.keras.layers.Activation(activation='relu')(x)
-        return x
\ No newline at end of file
+        return x

EEGNet/eegNet.py

@@ -15,16 +15,19 @@ from keras.callbacks import CSVLogger
 import numpy as np
 import logging
 
-def run(trainX, trainY):
-    """
-    Starts the EEGNet and stores the histogram, the plots of loss and accuracy.
-    """
-    classifier = Classifier_EEGNet()
-    hist = classifier.fit(trainX, trainY)
-    plot_loss(hist, config['model_dir'], config['model'], True)
-    plot_acc(hist, config['model_dir'], config['model'], True)
-    # Newly added lines below
-    save_logs(hist, config['model_dir'], config['model'], pytorch=False)
+from sklearn.model_selection import train_test_split
+
+class prediction_history(tf.keras.callbacks.Callback):
+    def __init__(self, val_data):
+        self.val_data = val_data
+        self.predhis = []
+        self.targets = []
+
+    def on_batch_end(self, epoch, logs={}):
+        x_val, y_val = self.val_data
+        self.targets.append(y_val)
+        prediction = self.model.predict(x_val)
+        self.predhis.append(prediction)
 
 class Classifier_EEGNet:
     """
@@ -36,7 +39,7 @@ class Classifier_EEGNet:
 
     def __init__(self, nb_classes=1, chans = config['eegnet']['channels'],
             samples = config['eegnet']['samples'], dropoutRate = 0.5, kernLength = 250, F1 = 16,
-            D = 4, F2 = 256, norm_rate = 0.5, dropoutType = 'Dropout', verbose = True, build = True, X = None):
+            D = 4, F2 = 256, norm_rate = 0.5, dropoutType = 'Dropout', epochs = 50, verbose = True, build = True, X = None):
 
         self.nb_classes = nb_classes
         self.chans = chans
@@ -48,6 +51,7 @@ class Classifier_EEGNet:
         self.F2 = F2
         self.norm_rate = norm_rate
         self.dropoutType = dropoutType
+        self.epochs = epochs
         self.verbose = verbose
 
         if build:
@@ -135,5 +139,9 @@ class Classifier_EEGNet:
         early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
         ckpt_dir = config['model_dir'] + '/' + config['model'] + '_' + 'best_model.h5'
         ckpt = tf.keras.callbacks.ModelCheckpoint(ckpt_dir, verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
-        hist = self.model.fit(eegnet_x, y, verbose=1, validation_split=0.2, epochs=50, callbacks=[csv_logger, ckpt, early_stop])
-        return hist
+        X_train, X_val, y_train, y_val = train_test_split(eegnet_x, y, test_size=0.2, random_state=42)
+        pred_ensemble = prediction_history((X_val,y_val))
+        hist = self.model.fit(X_train, y_train, verbose=1, validation_data=(X_val,y_val),
+        epochs=self.epochs, callbacks=[csv_logger, ckpt, early_stop,pred_ensemble])
+
+        return hist, pred_ensemble

InceptionTime/inception.py

@@ -5,18 +5,6 @@ import logging
 from ConvNet import ConvNet
 
 
-def run(trainX, trainY):
-    """
-    Starts the InceptionTime and stores the histogram, the plots of loss and accuracy.
-    """
-    logging.info("Starting InceptionTime.")
-    classifier = Classifier_INCEPTION(input_shape=config['inception']['input_shape'])
-    hist = classifier.fit(trainX, trainY)
-    plot_loss(hist, config['model_dir'], config['model'], True)
-    plot_acc(hist, config['model_dir'], config['model'], True)
-    save_logs(hist, config['model_dir'], config['model'], pytorch=False)
-
-
 class Classifier_INCEPTION(ConvNet):
     """
     The InceptionTime architecture used as baseline. This is the architecture explained in the paper

README.md

 # DL-Project
 
-Predicting eye gaze with DL
-    Projects that we try: CNN, InceptionTime, EEGNet, DeepEye
+Predicting eye gaze direction with DL
+	
+Models that we try: CNN, InceptionTime, EEGNet, Xception, DeepEye, and DeepEyeRNN.
 
 
+## Datasets
 
-## Leonhard Settings
+You can download the datasets from this link: https://polybox.ethz.ch/index.php/s/GLe287LxHq3QysW
 
-Please use the command
+ - 125Hz data: noweEEG.mat
+ - 500Hz data: all_EEGprocuesan.mat
+ - labels (0 for left, 1 for right): all_trialinfoprosan.mat
 
-`module load gcc/6.3.0 python_gpu/3.8.5 hdf5/1.10.1`
+## Model Configuration
 
-before training the model. Please make sure that the Tensorflow version should be 2.x.
+Please configure the config.py file correctly before running the main.py file:   
+  
+config['data_dir']    : indicates the directory where you stored the data  
+config['model']       : indicates the model you want to use, choose between 'cnn', 'eegnet', 'inception', 'xception' or'deepeye'  
+config['downsampled'] : True if you want to use 125 data points per second instead of 500. Default is False  
+config['split']       : True if you want to run a clustered version of the model, please keep it to False as the clustered version is inneficient  
 
-## deepEye3 tuning
 
-nb_filter: [32, 64]
+## Parameter Tuning
 
-depth: [9, 12, 20]
+Please find the files which are related to our model:
+	
+	|
+	| - CNN -- CNN.py
+	| - DeepEye -- deepeye.py
+	| - DeepEyeRNN -- deepeyeRNN.py
+	| - EEGNet -- eegNet.py
+	| - InceptionTime -- inception.py
+	| - Xception -- xception.py
+	| - ConvNet.py
 
-kernel_size:[40, 20]
+You can find the architechture of our models in these files. For `CNN`, `DeepEye`, `DeepEyeRNN`, `InceptionTime`, and `Xception`, you should tune the parameter by looking into the `ConvNet.py` file and adjust the parameters (e.g. `self.nb_filter`) accordingly.
 
-residual_jump: [3, 4]
+For `EEGNet` model, you should directly look into the `eegNet.py` file and tune the parameters accordingly.
 
-Large depth causes overfitting, same for the number of filters. Kernel size seems to have tiny affect on validation. Residual jump for 4 (i.e. `depth % (res_jump) == (res_jump - 1)`) is not so good in our task, but I think it would be useful for future tasks.
 
-The best setting is **nb_filter == 32, depth == 9, kernel_size == 40, res_jump == 3**
+## Running in the Cluster
+
+For [ETH Leonhard](https://scicomp.ethz.ch/wiki/Python_on_Leonhard) users, you can follow these steps:
+	
+	1. Please use the command `module load gcc/6.3.0 python_gpu/3.8.5 hdf5/1.10.1` before training the model.
+	
+	2. Edit the _data directory_ and __run directory__ where you _saved the datasets_ and __run the code for saving the logs and outputs__.
+	
+	3. Use the following command to run the model: `bsub -n 10 -W 4:00 -o experiment_[your_model] -R "rusage[mem=5000, ngpus_excl_p=1]" python ../main.py`
+
+__Note:__  If you want to run the model locally, you should ensure to have Pytorch, Tensorflow==2.x installed.
+	   
+
+## Reading the Results
+
+A summary of the model is printed in the `experiment_[your_model]` file generated by the Leonhard cluster.
+
+After each run, a directory with the name `'number'_config['model']` is created under the run directory, where 'number' refers to the time stamp of the training (different for each run of main.py). It contains the weights of the model, a .csv file with the fiting history, the best validation score and an accuracy and loss plot of the training.
+
 

Xception/xception.py

@@ -4,17 +4,6 @@ from utils.utils import *
 import logging
 from ConvNet import ConvNet
 
-def run(trainX, trainY):
-    """
-     Starts the Xception and stores the histogram, the plots of loss and accuracy.
-    """
-    logging.info("Starting InceptionTime.")
-    classifier = Classifier_XCEPTION(input_shape=config['inception']['input_shape'])
-    hist = classifier.fit(trainX, trainY)
-    plot_loss(hist, config['model_dir'], config['model'], True)
-    plot_acc(hist, config['model_dir'], config['model'], True)
-    save_logs(hist, config['model_dir'], config['model'], pytorch=False)
-
 
 class Classifier_XCEPTION(ConvNet):
     """

config.py

@@ -36,19 +36,21 @@ xception: The Xception architecture
 deepeye-rnn: The DeepEye RNN architecture
 
 If downsampled set to true, the downsampled data (which have length 125) will be used. Otherwise, full data with length 500 is used.
-If split set to true, the data will be clustered and fed each to a separate architecture. The extracted features are concatenated 
+If split set to true, the data will be clustered and fed each to a separate architecture. The extracted features are concatenated
 finally used for classification.
 Cluster can be set to clustering(), clustering2() or clustering3(), where different clusters based on literature are used.
 """
 
 # Choosing model
-config['model'] = 'inception'
+config['model'] = 'deepeye'
 config['downsampled'] = False
 config['split'] = False
-config['cluster'] = clustering2()
+config['cluster'] = clustering()
 if config['split']:
     config['model'] = config['model'] + '_cluster'
 
+config['ensemble'] = 9 #number of models in the ensemble method
+
 config['trainX_file'] = 'noweEEG.mat' if config['downsampled'] else 'all_EEGprocuesan.mat'
 config['trainY_file'] = 'all_trialinfoprosan.mat'
 config['trainX_variable'] = 'noweEEG' if config['downsampled'] else 'all_EEGprocuesan'
@@ -79,9 +81,12 @@ config['eegnet']['samples'] = 125 if config['downsampled'] else 500
 # Create a unique output directory for this experiment.
 timestamp = str(int(time.time()))
 model_folder_name = timestamp if config['model'] == '' else timestamp + "_" + config['model']
-
+if config['split']:
+     model_folder_name += '_cluster'
 if config['downsampled']:
     model_folder_name += '_downsampled'
+if config['ensemble']>1:
+    model_folder_name += '_ensemble'
 
 config['model_dir'] = os.path.abspath(os.path.join(config['log_dir'], model_folder_name))
 if not os.path.exists(config['model_dir']):
@@ -116,4 +121,4 @@ config['deepeye']['trainX_filename'] = "EEGprocue"
 
 config['deepeye']['trainY_variable1'] = "trialinfopro"
 config['deepeye']['trainY_variable2'] = "cues"
-config['deepeye']['trainY_filename'] = "trialinfocuelocked"
\ No newline at end of file
+config['deepeye']['trainY_filename'] = "trialinfocuelocked"

ensemble.py

+import tensorflow as tf
+from config import config
+from utils.utils import *
+import logging
+
+from CNN.CNN import Classifier_CNN
+from DeepEye.deepeye import Classifier_DEEPEYE
+from DeepEyeRNN.deepeyeRNN import Classifier_DEEPEYE_RNN
+from Xception.xception import Classifier_XCEPTION
+from InceptionTime.inception import Classifier_INCEPTION
+from EEGNet.eegNet import Classifier_EEGNet
+import numpy as np
+
+
+
+def run(trainX, trainY):
+    """
+    Starts the  multiples Classifier in the Ensemble and stores the histogram, the plots of loss and accuracy.
+    validation is of the ensemble model and training just the last one
+    """
+    logging.info("Started running "+config['model']+". If you want to run other methods please choose another model in the config.py file.")
+
+    # acc = tf.keras.metrics.BinaryAccuracy()
+    bce = tf.keras.losses.BinaryCrossentropy()
+
+    loss=[]
+    accuracy=[]
+
+    for i in range(config['ensemble']):
+        print('beginning model number {}/{} ...'.format(i,config['ensemble']))
+        if config['model'] == 'deepeye':
+            classifier = Classifier_DEEPEYE(input_shape = config['deepeye']['input_shape'])
+        elif config['model'] == 'cnn':
+            classifier = Classifier_CNN(input_shape = config['cnn']['input_shape'])
+        elif config['model'] == 'eegnet':
+            classifier = Classifier_EEGNet()
+        elif config['model'] == 'inception':
+            classifier = Classifier_INCEPTION(input_shape=config['inception']['input_shape'])
+        elif config['model'] == 'xception' :
+            classifier = Classifier_XCEPTION(input_shape=config['inception']['input_shape'])
+        elif config['model'] == 'deepeye-rnn':
+            classifier = Classifier_DEEPEYE_RNN(input_shape=config['deepeye-rnn']['input_shape'])
+        else:
+            logging.info('Cannot start the program. Please choose one model in the config.py file')
+
+        hist, pred_ensemble = classifier.fit(trainX,trainY)
+        if i == 0:
+            pred = pred_ensemble.predhis
+        else:
+            for j, pred_epoch in enumerate(pred_ensemble.predhis):
+                pred[j] = (np.array(pred[j])+np.array(pred_epoch))
+
+    for j, pred_epoch in enumerate(pred):
+        pred_epoch = (pred_epoch/config['ensemble']).tolist()
+        loss.append(bce(pred_ensemble.targets[j],pred_epoch).numpy())
+        pred_epoch = np.round(pred_epoch,0)
+        accuracy.append(np.mean((np.array(pred_epoch).reshape(-1)+np.array(pred_ensemble.targets[j]).reshape(-1)-1)**2))
+
+    if config['ensemble']>1:
+        config['model']+='_ensemble'
+    if config['split']:
+        config['model'] = config['model'] + '_cluster'
+
+    hist.history['val_loss'] = loss
+    hist.history['val_accuracy'] = accuracy
+    plot_loss(hist, config['model_dir'], config['model'], val = True)
+    plot_acc(hist, config['model_dir'], config['model'], val = True)
+    save_logs(hist, config['model_dir'], config['model'], pytorch = False)

main.py

 from config import config
-import time
-from CNN import CNN
-from utils import IOHelper
-from DeepEye import deepeye
-from DeepEyeRNN import deepeyeRNN
-from Xception import xception
-from InceptionTime import inception
-from EEGNet import eegNet
+from ensemble import run
 import numpy as np
+import scipy
+from scipy import io
+import h5py
 import logging
-
+import time
 
 def main():
     logging.basicConfig(filename=config['info_log'], level=logging.INFO)
@@ -18,36 +14,10 @@ def main():
     # try:
     trainX, trainY = IOHelper.get_mat_data(config['data_dir'], verbose=True)
 
-    if config['model'] == 'cnn' or config['model'] == 'cnn_cluster':
-        logging.info("Started running CNN. If you want to run other methods please choose another model in the config.py file.")
-        CNN.run(trainX, trainY)
-
-    elif config['model'] == 'eegnet' or config['model'] == 'eegnet_cluster':
-        logging.info(
-            "Started running EEGNet. If you want to run other methods please choose another model in the config.py file.")
-        eegnet_x = np.transpose(trainX, (0, 2, 1))
-        logging.info(eegnet_x.shape)
-        eegNet.run(trainX=eegnet_x, trainY=trainY)
-
-    elif config['model'] == 'inception' or config['model'] == 'inception_cluster':
-        logging.info("Started running InceptionTime. If you want to run other methods please choose another model in the config.py file.")
-        inception.run(trainX=trainX, trainY=trainY)
-
-    elif config['model'] == 'xception' or config['model'] == 'xception_cluster':
-        logging.info("Started running XceptionTime. If you want to run other methods please choose another model in the config.py file.")
-        xception.run(trainX=trainX, trainY=trainY)
-
-    elif config['model'] == 'deepeye' or config['model'] == 'deepeye_cluster':
-        logging.info("Started running DeepEye. If you want to run other methods please choose another model in the config.py file.")
-        deepeye.run(trainX=trainX, trainY=trainY)
-
-    elif config['model'] == 'deepeye-rnn' or config['model'] == 'deepeye-rnn_cluster':
-        logging.info("Started running deepeye-rnn. If you want to run other methods please choose another model in the config.py file.")
-        deepeyeRNN.run(trainX=trainX, trainY=trainY)
-
-    else:
-        logging.info('Cannot start the program. Please choose one model in the config.py file')
-
+    if config['model'] == 'eegnet' or config['model'] == 'eegnet_cluster':
+        trainX = np.transpose(trainX, (0, 2, 1))
+        logging.info(trainX.shape)
+    run(trainX,trainY)
     logging.info("--- Runtime: %s seconds ---" % (time.time() - start_time))
     logging.info('Finished Logging')
 

runs/1609157751_inception/inception_df_metrics.csv

-Accuracy,Loss
-0.5900683403015137,0.6709842085838318

runs/1609157751_inception/inception_history.csv

-loss,accuracy,val_loss,val_accuracy
-0.6709842085838318,0.5900683403015137,0.6837942004203796,0.6404416561126709

runs/1609159406_inception/inception_df_best_model.csv

-best_model_train_loss,best_model_val_loss,best_model_train_acc,best_model_val_acc
-0.706407904624939,0.6844089031219482,0.5264661908149719,0.5702930688858032

runs/1609159406_inception/inception_df_metrics.csv

-Accuracy,Loss
-0.5264661908149719,0.706407904624939

runs/1609159406_inception/inception_history.csv

-loss,accuracy,val_loss,val_accuracy
-0.706407904624939,0.5264661908149719,0.6844089031219482,0.5702930688858032