To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 6ef639cc authored by Ard Kastrati's avatar Ard Kastrati
Browse files

First state of preparation module

parent 5b81d9fc
......@@ -9,3 +9,4 @@ noweEEG.mat
/runs/*
/kerasTunerResults/*
./lsf*
/data/*
......@@ -9,19 +9,17 @@ config = dict()
##################################################################
# Please note that the following fields will be set by our scripts to re-train and re-evaluate our model.
# Where experiment results are stored.
config['log_dir'] = './runs/'
# Path to training, validation and test data folders.
# PS: Note that we have to upload the data to the server!!!
# config['data_dir'] = '/cluster/home/your_username/data/'
config['data_dir'] = './'
config['measured_data_dir'] = './data/measured/'
config['prepared_data_dir'] = './data/prepared/'
# Path of root
config['root_dir'] = '.'
##################################################################
# You can modify the rest or add new fields as you need.
# Hyper-parameters and training configuration.
config['learning_rate'] = 1e-4
......@@ -50,7 +48,6 @@ config['cluster'] = clustering()
config['ensemble'] = 5 #number of models in the ensemble method
config['trainX_file'] = 'noweEEG.mat' if config['downsampled'] else 'all_EEGprocuesan.mat'
config['trainY_file'] = 'all_trialinfoprosan.mat'
config['trainX_variable'] = 'noweEEG' if config['downsampled'] else 'all_EEGprocuesan'
......@@ -77,7 +74,6 @@ config['inception']['input_shape'] = (125, 129) if config['downsampled'] else (5
config['deepeye']['input_shape'] = (125, 129) if config['downsampled'] else (500, 129)
config['deepeye-rnn']['input_shape'] = (125, 129) if config['downsampled'] else (500, 129)
config['xception']['input_shape'] = (125, 129) if config['downsampled'] else (500, 129)
config['eegnet']['channels'] = 129
config['eegnet']['samples'] = 125 if config['downsampled'] else 500
......@@ -98,30 +94,3 @@ if not os.path.exists(config['model_dir']):
config['info_log'] = config['model_dir'] + '/' + 'info.log'
config['batches_log'] = config['model_dir'] + '/' + 'batches.log'
# Before we were loading directly the data from the server, or used the code to merge them.
# Deprecated now. We don't really use the functions anymore in the IOHelper that make use of the following configurations.
config['cnn']['trainX_variable1'] = "EEGprocue"
config['cnn']['trainX_variable2'] = "data"
config['cnn']['trainX_filename'] = "EEGprocue"
config['cnn']['trainY_variable1'] = "trialinfopro"
config['cnn']['trainY_variable2'] = "cues"
config['cnn']['trainY_filename'] = "trialinfocuelocked"
config['inception']['trainX_variable1'] = "EEGprocue"
config['inception']['trainX_variable2'] = "data"
config['inception']['trainX_filename'] = "EEGprocue"
config['inception']['trainY_variable1'] = "trialinfopro"
config['inception']['trainY_variable2'] = "cues"
config['inception']['trainY_filename'] = "trialinfocuelocked"
config['deepeye']['trainX_variable1'] = "EEGprocue"
config['deepeye']['trainX_variable2'] = "data"
config['deepeye']['trainX_filename'] = "EEGprocue"
config['deepeye']['trainY_variable1'] = "trialinfopro"
config['deepeye']['trainY_variable2'] = "cues"
config['deepeye']['trainY_filename'] = "trialinfocuelocked"
##################################################################
# Data preparation configurations
import time
import os
import numpy as np
preparation_config = dict()
# The task for which we want to prepare the data. Possible choices that are implemented so far are:
# 'LR_task' (dataset: 'antisaccade'):
# 'Direction_task' (dataset: 'dots' or 'processing_speed'):
# 'Position_task' (dataset: 'dots'):
# 'Segmentation_task' (dataset: 'antisaccade', 'dots', or 'processing_speed'):
preparation_config['task'] = 'LR_task'
preparation_config['dataset'] = 'antisaccade'
# We provide two types of preprocessing on the dataset (minimal preprocessing and maximal preprocessing). Choices are
# 'max'
# 'min'
preparation_config['preprocessing'] = 'max' # or min
# We provide also dataset where features are extracted
# (typically used for training with standard machine learning methods).
# The feature extraction that we have implemented is hilbert transformed data for phase and amplitude.
preparation_config['feature_extraction'] = True
# Maybe for later we can also include the bandpassed data on
# top of the feature extracted data (this is not implemented yet).
preparation_config['including_bandpass_data'] = False # or True (for later)
##################################################################################
##################################################################################
##################################################################################
##################################################################################
# We prepare some helper variables to locate the correct datasets that we need and to use them.
preparation_config['preprocessing_path'] = 'synchronized_' + preparation_config['preprocessing'] if preparation_config['feature_extraction'] else 'synchronized_' + preparation_config['preprocessing']
preparation_config['LOAD_ANTISACCADE_PATH'] = '../data/measured/antisaccade_task_data/' + preparation_config['preprocessing_path'] + '/'
preparation_config['SAVE_ANTISACCADE_PATH'] = '../data/prepared/antisaccade_task_data/' + preparation_config['preprocessing_path'] + '/'
preparation_config['ANTISACCADE_FILE_PATTERN'] = '[go]ip_..._AS_EEG.mat'
preparation_config['LOAD_DOTS_PATH'] = '../data/measured/dots_data/' + preparation_config['preprocessing_path'] + '/'
preparation_config['SAVE_DOTS_PATH'] = '../data/prepared/dots_data/' + preparation_config['preprocessing_path'] + '/'
preparation_config['DOTS_FILE_PATTERN'] = '(ep|EP).._DOTS._EEG.mat'
preparation_config['LOAD_PROCESSING_SPEED_PATH'] = '../data/measured/processing_speed_data/' + preparation_config['preprocessing_path'] + '/'
preparation_config['SAVE_PROCESSING_SPEED_PATH'] = '../data/prepared/processing_speed_data/' + preparation_config['preprocessing_path'] + '/'
preparation_config['PROCESSING_SPEED_FILE_PATTERN'] = '..._WI2_EEG.mat'
####################################################################################
# Internal information about each dataset (antisaccade, dots, processing_speeed)
preparation_config['saccade_trigger'] = ['L_saccade', 'R_saccade']
preparation_config['fixation_trigger'] = ['L_fixation', 'R_fixation']
preparation_config['blink_trigger'] = ['L_blink', 'R_blink']
# Anti-saccade dataset
preparation_config['antisaccade'] = dict()
preparation_config['antisaccade']['cue_trigger'] = ['10', '11']
preparation_config['antisaccade']['matlab_struct'] = 'EEG'
#Dots dataset
preparation_config['dots'] = dict()
preparation_config['dots']['cue_trigger'] = list(map(str, range(1, 28))) + list(map(str, range(101, 128)))
preparation_config['dots']['end_cue_trigger'] =['41']
preparation_config['dots']['matlab_struct'] = 'sEEG'
preparation_config['dots']['tar_pos'] = np.array([
[400, 300], [650, 500], [400, 100], [100, 450], [700, 450], [100, 500],
[200, 350], [300, 400], [100, 150], [150, 500], [150, 100], [700, 100],
[300, 200], [100, 100], [700, 500], [500, 400], [600, 250], [650, 100],
[400, 300], [200, 250], [400, 500], [700, 150], [500, 200], [100, 300],
[700, 300], [600, 350], [400, 300]
])
# Processing speed dataset
preparation_config['processing_speed'] = dict()
preparation_config['processing_speed']['matlab_struct'] = 'sEEG'
preparation_config['matlab_struct'] = preparation_config[preparation_config['dataset']]['matlab_struct']
#Maybe we should do logging here as well ...
\ No newline at end of file
import os
import numpy as np
import h5py
import scipy.io
import pandas as pd
import re
from tqdm import tqdm
from preparation_config import preparation_config as config
class Preparator:
def __init__(self, load_directory='./', save_directory='./', load_file_pattern='*', save_file_name='all.npz',verbose=False):
self.load_directory = load_directory
self.save_directory = save_directory
self.load_file_pattern = re.compile(load_file_pattern)
self.save_file_name = save_file_name
self.extract_pattern = None
self.extract_pattern = None
self.start_time = None
self.length_time = None
self.start_channel = None
self.end_channel = None
self.on_blocks = None
self.off_blocks = None
self.filters = []
self.ignore_events = []
self.labels = []
self.verbose = verbose
self.padding = True
if self.verbose:
print("Preparator is initialized with: ")
print("Directory to load data: " + self.load_directory)
print("Directory to save data: " + self.save_directory)
print("Looking for file that match: " + load_file_pattern)
print("Will store the merged file with name: " + self.save_file_name)
def extract_data_at_events(self, extract_pattern, name_start_time, start_time, name_length_time, length_time, start_channel, end_channel, padding=True):
self.extract_pattern = extract_pattern
self.start_time = start_time
self.length_time = length_time
self.start_channel = start_channel
self.end_channel = end_channel
self.padding = padding
if self.verbose:
print("Preparator is instructed to look for events that match structure: " + str(self.extract_pattern))
print("Time dimension -- Cut start info: " + name_start_time)
print("Time dimension -- Cut length info: " + name_length_time)
print("Channel dimension -- Cut start info: " + str(start_channel))
print("Channel dimension -- Cut end info: " + str(end_channel))
def blocks(self, on_blocks, off_blocks):
self.on_blocks = on_blocks
self.off_blocks = off_blocks
if self.verbose:
print("Blocks to be used are: " + str(on_blocks))
print("Blocks to be ignored are: " + str(off_blocks))
def addFilter(self, name, f):
self.filters.append((name, f))
if self.verbose: print('Preparator is instructed to use filter: ' + name)
def addLabel(self, name, f):
self.labels.append((name, f))
if self.verbose: print('Preparator is instructed to use label: ' + name)
def ignoreEvent(self, name, f):
self.ignore_events.append((name, f))
if self.verbose: print('Preparator is instructed to ignore the event: ' + name)
def run(self):
if self.verbose: print("Starting collecting data.")
all_EEG = []
all_labels = []
subj_counter = 1
progress = tqdm(sorted(os.listdir(self.load_directory)))
for subject in progress:
if os.path.isdir(self.load_directory + subject):
# skip subject which doesn't follow the sequence: 20.P - 30.A - 30.A - 30.A - 20.P
# if subject == 'BY2':
# continue
# if subject == 'EP18':
# break
cur_dir = self.load_directory + subject + '/'
debug = 0
for f in sorted(os.listdir(cur_dir)):
# skip the non-matching files
print(f)
if not self.load_file_pattern.match(f):
continue
print(subject)
if debug == 1:
break
debug = 1 - debug
progress.set_description('Loading ' + f)
# load the mat file
events = None
if h5py.is_hdf5(cur_dir + f):
hdf5file = h5py.File(cur_dir + f, 'r')
EEG = hdf5file[list(h5py.File(cur_dir + f, 'r').keys())[1]]
events = self._load_hdf5_events(EEG)
else:
EEG = scipy.io.loadmat(cur_dir + f)['sEEG'][0]
events = self._load_v5_events(EEG)
events = self._ignore_events(events)
if self.verbose: print(events)
select = self._filter_blocks(events)
select &= self._filter_events(events)
trials = self._extract_events(EEG, events, select)
labels = self._extract_labels(events, select, subj_counter)
all_EEG.append(trials)
all_labels.append(labels)
subj_counter += 1
# save the concatenated arrays
print('Saving data...')
EEG = np.concatenate(all_EEG, axis=0)
labels = np.concatenate(all_labels, axis=0)
print(EEG.shape)
print(labels.shape)
np.savez(self.save_directory + self.save_file_name, EEG=EEG, labels=labels)
def _load_v5_events(self, EEG):
if self.verbose: print("Loading the events from the subject. ")
# extract the useful event data
events = pd.DataFrame()
events['type'] = [''.join(map(str, el[0][0:])).strip() for el in EEG[0]['event'][0]['type']]
# if self.verbose: print(events)
events['latency'] = [el[0][0] for el in EEG[0]['event'][0]['latency']]
# if self.verbose: print(events)
events['amplitude'] = [el[0][0] for el in EEG[0]['event'][0]['sac_amplitude']]
# if self.verbose: print(events)
events['start_x'] = [el[0][0] for el in EEG[0]['event'][0]['sac_startpos_x']]
# if self.verbose: print(events)
events['end_x'] = [el[0][0] for el in EEG[0]['event'][0]['sac_endpos_x']]
# if self.verbose: print(events)
events['start_y'] = [el[0][0] for el in EEG[0]['event'][0]['sac_startpos_y']]
# if self.verbose: print(events)
events['end_y'] = [el[0][0] for el in EEG[0]['event'][0]['sac_endpos_y']]
# if self.verbose: print(events)
events['duration'] = [el[0][0] for el in EEG[0]['event'][0]['duration']]
# if self.verbose: print(events)
events['avgpos_x'] = [el[0][0] for el in EEG[0]['event'][0]['fix_avgpos_x']]
# if self.verbose: print(events)
events['avgpos_y'] = [el[0][0] for el in EEG[0]['event'][0]['fix_avgpos_y']]
# if self.verbose: print(events)
events['endtime'] = [el[0][0] for el in EEG[0]['event'][0]['endtime']]
if self.verbose:
print("Events loaded are: ")
print(events)
return events
def _load_hdf5_events(self, EEG):
if self.verbose: print("Loading the events from the subject. ")
# extract the useful event data
events = pd.DataFrame()
events['type'] = [''.join(map(chr, EEG[ref][:, 0])).strip() for ref in EEG['event']['type'][:, 0]]
#if self.verbose: print(events)
events['latency'] = [EEG[ref][0, 0] for ref in EEG['event']['latency'][:, 0]]
# if self.verbose: print(events)
events['amplitude'] = [EEG[ref][0, 0] for ref in EEG['event']['sac_amplitude'][:, 0]]
#if self.verbose: print(events)
events['start_x'] = [EEG[ref][0, 0] for ref in EEG['event']['sac_startpos_x'][:, 0]]
#if self.verbose: print(events)
events['end_x'] = [EEG[ref][0, 0] for ref in EEG['event']['sac_endpos_x'][:, 0]]
#if self.verbose: print(events)
events['start_y'] = [EEG[ref][0, 0] for ref in EEG['event']['sac_startpos_y'][:, 0]]
#if self.verbose: print(events)
events['end_y'] = [EEG[ref][0, 0] for ref in EEG['event']['sac_endpos_y'][:, 0]]
#if self.verbose: print(events)
events['duration'] = [EEG[ref][0, 0] for ref in EEG['event']['duration'][:, 0]]
#if self.verbose: print(events)
events['avgpos_x'] = [EEG[ref][0, 0] for ref in EEG['event']['fix_avgpos_x'][:, 0]]
#if self.verbose: print(events)
events['avgpos_y'] = [EEG[ref][0, 0] for ref in EEG['event']['fix_avgpos_y'][:, 0]]
# if self.verbose: print(events)
events['endtime'] = [EEG[ref][0, 0] for ref in EEG['event']['endtime'][:, 0]]
if self.verbose:
print("Events loaded are: ")
print(events)
return events
def _filter_blocks(self, events):
if self.verbose: print("Filtering the blocks: ")
select = events['type'].apply(lambda x: True)
if self.on_blocks is None or self.off_blocks is None:
return select
for i, event in enumerate(events['type']):
if event in self.on_blocks:
select.iloc[i] = True
elif event in self.off_blocks:
select.iloc[i] = False
elif i > 0:
select.iloc[i] = select.iloc[i-1]
if self.verbose: print(list(zip(range(1, len(select) + 1), select)))
return select
def _ignore_events(self, events):
ignore = events['type'].apply(lambda x: False)
for name, f in self.ignore_events:
if self.verbose: print("Applying: " + name)
ignore |= f(events)
if self.verbose: print(list(zip(range(1, len(ignore) + 1), ignore)))
select = ignore.apply(lambda x: not x)
if self.verbose: print(list(zip(range(1, len(select) + 1), select)))
return events.loc[select]
def _filter_events(self, events):
if self.verbose: print("Filtering the events: ")
select = events['type'].apply(lambda x: True)
for i, event in enumerate(self.extract_pattern):
select &= events['type'].shift(-i).isin(event)
if self.verbose: print(list(zip(range(1, len(select) + 1), select)))
for name, f in self.filters:
if self.verbose: print("Applying filter: " + name)
select &= f(events)
if self.verbose: print(list(zip(range(1, len(select) + 1), select)))
return select
def _extract_events(self, EEG, events, select): # needs to be able to pad
if self.verbose: print("Extracting data from the interested events: ")
all_trials = []
# extract the useful data
print(EEG['data'])
data = np.array(EEG['data'], dtype='float')
start = self.start_time(events).loc[select]
length = events['type'].apply(lambda x: self.length_time).loc[select]
end_block = events['latency'].shift(-len(self.extract_pattern)).loc[select]
print(start)
print(length)
print(end_block)
for s, l, e in zip(start, length, end_block):
if s + l > e and self.padding:
#Need to pad since, the required length is bigger then the last block
print(str(s) + ", " + str(l) + ", " + str(e) + " that is need to pad")
unpadded_data = data[int(s - 1):int(e - 1), (self.start_channel - 1):self.end_channel]
x_len, y_len = unpadded_data.shape
padding_size = int(s + l - e)
append_data = np.pad(unpadded_data, pad_width=((0, padding_size), (0, 0)), mode='reflect')
print(append_data)
else:
append_data = data[int(s - 1):int(s + l - 1), (self.start_channel - 1):self.end_channel]
all_trials.append(append_data)
all_trials = np.array(all_trials)
# if self.verbose: print("Extracted all this data from this participant.")
# if self.verbose: print(all_trials)
return all_trials
def _extract_labels(self, events, select, subj_counter):
if self.verbose: print("Extracting the labels for each trial.")
if self.verbose: print("Appending the subject counter. ")
# append subject IDs to the full list and then the labels
nr_trials = events.loc[select].shape[0]
labels = np.full((nr_trials, 1), subj_counter)
if self.verbose: print(labels)
for name, f in self.labels:
if self.verbose: print("Appending the next label: " + name)
labels = np.concatenate((labels, np.asarray(f(events).loc[select]).reshape(-1,1)), axis=1)
if self.verbose: print(labels)
return labels
"""
assert((self.fixed_length is not None and self.length is not None) == False)
if self.fixed_length is not None:
if self.verbose: print("Fixed length, setting the duration to this fixed length " + str(self.fixed_length))
start = events['latency']
duration = events['latency'].apply(lambda x: self.fixed_length)
end = events['latency'].apply(lambda x: (x + self.fixed_length))
if self.verbose: print(start)
if self.verbose: print(duration)
if self.verbose: print(end)
elif self.length is not None:
if self.verbose: print("The length to be extracted is: " + str(self.length))
start = events['latency']
duration = events['latency'].apply(lambda x: self.length)
end = events['endtime'].shift(-len(self.extract_pattern) + 1)
if self.verbose: print(start)
if self.verbose: print(duration)
if self.verbose: print(end)
else:
raise ValueError("Please specify either length or fixed length")
# Get the valid trials
start_trial = start.loc[select]
duration_trial = duration.loc[select]
end_trial = end.loc[select]
if self.verbose:
print("Getting the valid trials start and their duration.")
print(start_trial)
print(duration_trial)
print(end_trial)
# Cut the data
if self.verbose: print("Cutting the data.")
all_trials = []
# extract the useful data
data = np.array(EEG['data'], dtype='float')
for start, duration, end in zip(start_trial, duration_trial, end_trial):
if end - start > duration:
# Need to cut
if self.verbose: print("Cutting the data since too long")
start = start + (end - start - duration)/2
print("Starting " + str(start-1) + " and ending in " + str(start + duration - 1))
all_trials.append(data[int(start - 1):int(start + duration - 1), :129])
elif end - start < duration:
if self.verbose: print("Padding the data since too short")
# Need to pad
unpadded_data = data[int(start - 1):int(end - start - 1), :]
x_len, y_len = unpadded_data.shape
padding_size = duration - x_len
padded_data = np.pad(unpadded_data, pad_width=((0, padding_size), (0, 0)))
all_trials.append(padded_data[:,:129])
else: # They are equal
if self.verbose: print("No cutting or padding is needed.")
all_trials.append(data[int(start - 1):int(start + duration - 1), :129])
all_trials = np.array(all_trials)
if self.verbose: print("Extracted all this data from this participant.")
if self.verbose: print(all_trials)
return all_trials
"""
\ No newline at end of file
from data_preparation.preparator import Preparator
from preparation_config import preparation_config as config
import numpy as np
def left_right_task_data_preparation():
# We use the antisaccade dataset to extract the data for left and right benchmark task.
saccade = config['saccade_trigger']
fixation = config['fixation_trigger']
cue = config['antisaccade']['cue_trigger']
preparator = Preparator(load_directory=config['LOAD_ANTISACCADE_PATH'], save_directory=config['SAVE_ANTISACCADE_PATH'],
load_file_pattern=config['ANTISACCADE_FILE_PATTERN'], save_file_name='LeftRightTask_data.npz', verbose=True)
preparator.extract_data_at_events(extract_pattern=[cue], name_start_time='Beginning of cue', start_time=lambda events: events['latency'],
name_length_time='Size blocks of 500', length_time=500,
start_channel=1, end_channel=129, padding=False)
preparator.blocks(on_blocks=['20'], off_blocks=['30']) # take only blocks of pro-saccade
preparator.addFilter(name='Keep immediate saccade', f=lambda events: events['type'].isin(cue) & events['type'].shift(-1).isin(saccade))
preparator.addFilter(name='Keep right direction', f=lambda events: (events['type'].isin(['10']) & events['type'].shift(-1).isin(saccade) & (events['end_x'].shift(-1) < 400))
| (events['type'].isin(['11']) & events['type'].shift(-1).isin(saccade) & (events['end_x'].shift(-1) > 400)))
preparator.addFilter(name='Keep saccade if it comes after a delay', f=lambda events: events['latency'].shift(-1) - events['latency'] > 50)
preparator.addFilter(name='Keep only the ones with big enough amplitude', f=lambda events: events['amplitude'].shift(-1) > 2)
preparator.addLabel(name='Giving label 0 for left and 1 for right', f=lambda events: events['type'].apply(lambda x: 0 if x == '10' else 1))
preparator.run()
def left_right_task_feature_extraction_data_preparation():
# We use the antisaccade dataset to extract the data for left and right benchmark task.
saccade = config['saccade_trigger']
cue = config['antisaccade']['cue_trigger']
fixation = config['fixation_trigger']
preparator = Preparator(load_directory=config['LOAD_ANTISACCADE_PATH'], save_directory=config['SAVE_ANTISACCADE_PATH'],
load_file_pattern=config['ANTISACCADE_FILE_PATTERN'], save_file_name='LeftRightTask_data_feature_extracted.npz', verbose=True)
preparator.extract_data_at_events(extract_pattern=[saccade], name_start_time='Beginning of saccade', start_time=lambda events: events['latency'],
name_length_time='Fixed blocks of 1', length_time=1,
start_channel=1, end_channel=129, padding=False) # we are interested only on the saccade events where the feature were extracted (hilbert trafo)
preparator.blocks(on_blocks=['20'], off_blocks=['30']) # take only blocks of pro-saccade
preparator.addFilter(name='Keep saccade that has previously a cue', f=lambda events: events['type'].isin(saccade) & events['type'].shift(1).isin(cue))
preparator.addFilter(name='Keep right direction', f=lambda events: (events['type'].shift(1).isin(['10']) & events['type'].isin(saccade) & (events['end_x'] < 400))
| (events['type'].shift(1).isin(['11']) & events['type'].isin(saccade) & (events['end_x'] > 400)))
preparator.addFilter(name='Keep saccade if it comes after a delay', f=lambda events: events['latency'] - events['latency'].shift(1) > 50)
preparator.addFilter(name='Keep only the ones with big enough amplitude', f=lambda events: events['amplitude'] > 2)
preparator.addLabel(name='Giving label 0 for left and 1 for right', f=lambda events: events['type'].shift(1).apply(lambda x: 0 if x == '10' else 1))
preparator.run()
def direction_task_data_preparation():
# We use the 'dots' dataset for direction task
fixation = config['fixation_trigger']
saccade = config['saccade_trigger']
cue = config['dots']['cue_trigger']
end_cue = config['dots']['end_cue_trigger']
preparator = Preparator(load_directory=config['LOAD_DOTS_PATH'], save_directory=config['SAVE_DOTS_PATH'],
load_file_pattern=config['DOTS_FILE_PATTERN'], save_file_name='DirectionTask_data.npz', verbose=True)
# no padding, but cut 500 somewhere in between
# we are interested only on the 5-triggers (fixation 41 cue saccade fixation) and we cut 500 data points in the middle
preparator.extract_data_at_events(extract_pattern=[fixation, end_cue, cue, saccade, fixation], name_start_time='250 timepoints before the saccade',
start_time=lambda events: (events['latency'].shift(-3) - 250),
name_length_time='Fixed blocks of 500', length_time=500,
start_channel=1, end_channel=129, padding=False)
preparator.blocks(on_blocks=['55'], off_blocks=['56']) # take only blocks 55
preparator.ignoreEvent(name='Ignore microsaccades', f=(lambda events: events['type'].isin(saccade) & (events['amplitude'] < 2)))
preparator.ignoreEvent(name='Ignore microfixations', f=(lambda events: events['type'].isin(fixation) & (events['duration'] < 150)))