To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 80fda68c authored by Ard Kastrati's avatar Ard Kastrati
Browse files

Added the functionality to load also v5 files

parent da9d210c
......@@ -11,8 +11,8 @@ preparation_config = dict()
# 'Direction_task' (dataset: 'dots' or 'processing_speed'):
# 'Position_task' (dataset: 'dots'):
# 'Segmentation_task' (dataset: 'antisaccade', 'dots', or 'processing_speed'):
preparation_config['task'] = 'Position_task'
preparation_config['dataset'] = 'dots'
preparation_config['task'] = 'LR_task'
preparation_config['dataset'] = 'antisaccade'
# We provide two types of preprocessing on the dataset (minimal preprocessing and maximal preprocessing). Choices are
# 'max'
......
......@@ -90,21 +90,25 @@ class Preparator:
progress.set_description('Loading ' + f)
# load the mat file
events = None
data = None
# preparator.py - line 93
if h5py.is_hdf5(cur_dir + f):
if self.verbose: print("It is a HDF5 file. All is fine.")
hdf5file = h5py.File(cur_dir + f, 'r')
EEG = hdf5file[list(h5py.File(cur_dir + f, 'r').keys())[1]]
EEG = hdf5file[list(hdf5file.keys())[1]] # removal of a repeated h5py.File() call here
events = self._load_hdf5_events(EEG)
data = np.array(EEG['data'], dtype='float')
else:
# EEG = scipy.io.loadmat(cur_dir + f)['sEEG'][0]
# events = self._load_v5_events(EEG)
raise NotImplementedError("Matlab v5 files cannot be loaded. I still have to implement this.")
matfile = scipy.io.loadmat(cur_dir + f)
EEG = matfile[list(matfile.keys())[3]][0,0] #eventually at the end to remove the repetition in the load method
events = self._load_v5_events(EEG)
data = np.array(EEG['data'], dtype='float').T
events = self._ignore_events(events)
if self.verbose: print(events)
select = self._filter_blocks(events)
select &= self._filter_events(events)
trials = self._extract_events(EEG, events, select)
trials = self._extract_events(data, events, select)
labels = self._extract_labels(events, select, subj_counter)
all_EEG.append(trials)
......@@ -121,32 +125,32 @@ class Preparator:
print(labels.shape)
np.savez(self.save_directory + self.save_file_name, EEG=EEG, labels=labels)
# THIS IS NOT FINISHED
def _load_v5_events(self, EEG):
if self.verbose: print("Loading the events from the subject. ")
# extract the useful event data
events = pd.DataFrame()
events['type'] = [''.join(map(str, el[0][0:])).strip() for el in EEG[0]['event'][0]['type']]
events['type'] = [el[0].strip() for el in EEG['event'][0]['type']]
# if self.verbose: print(events)
events['latency'] = [el[0][0] for el in EEG[0]['event'][0]['latency']]
events['latency'] = [el[0, 0] for el in EEG['event'][0]['latency']]
# if self.verbose: print(events)
events['amplitude'] = [el[0][0] for el in EEG[0]['event'][0]['sac_amplitude']]
events['amplitude'] = [el[0, 0] for el in EEG['event'][0]['sac_amplitude']]
# if self.verbose: print(events)
events['start_x'] = [el[0][0] for el in EEG[0]['event'][0]['sac_startpos_x']]
events['start_x'] = [el[0, 0] for el in EEG['event'][0]['sac_startpos_x']]
# if self.verbose: print(events)
events['end_x'] = [el[0][0] for el in EEG[0]['event'][0]['sac_endpos_x']]
events['end_x'] = [el[0, 0] for el in EEG['event'][0]['sac_endpos_x']]
# if self.verbose: print(events)
events['start_y'] = [el[0][0] for el in EEG[0]['event'][0]['sac_startpos_y']]
events['start_y'] = [el[0, 0] for el in EEG['event'][0]['sac_startpos_y']]
# if self.verbose: print(events)
events['end_y'] = [el[0][0] for el in EEG[0]['event'][0]['sac_endpos_y']]
events['end_y'] = [el[0, 0] for el in EEG['event'][0]['sac_endpos_y']]
# if self.verbose: print(events)
events['duration'] = [el[0][0] for el in EEG[0]['event'][0]['duration']]
events['duration'] = [el[0, 0] for el in EEG['event'][0]['duration']]
# if self.verbose: print(events)
events['avgpos_x'] = [el[0][0] for el in EEG[0]['event'][0]['fix_avgpos_x']]
events['avgpos_x'] = [el[0, 0] for el in EEG['event'][0]['fix_avgpos_x']]
# if self.verbose: print(events)
events['avgpos_y'] = [el[0][0] for el in EEG[0]['event'][0]['fix_avgpos_y']]
events['avgpos_y'] = [el[0, 0] for el in EEG['event'][0]['fix_avgpos_y']]
# if self.verbose: print(events)
events['endtime'] = [el[0][0] for el in EEG[0]['event'][0]['endtime']]
events['endtime'] = [el[0, 0] for el in EEG['event'][0]['endtime']]
if self.verbose:
print("Events loaded are: ")
......@@ -225,13 +229,12 @@ class Preparator:
if self.verbose: print(list(zip(range(1, len(select) + 1), select)))
return select
def _extract_events(self, EEG, events, select): # needs to be able to pad
def _extract_events(self, data, events, select): # needs to be able to pad
if self.verbose: print("Extracting data from the interested events: ")
all_trials = []
# extract the useful data
if self.verbose: print(EEG['data'])
data = np.array(EEG['data'], dtype='float')
if self.verbose: print(data)
start = self.start_time(events).loc[select]
length = events['type'].apply(lambda x: self.length_time).loc[select]
......@@ -239,18 +242,18 @@ class Preparator:
if self.verbose: print(start)
if self.verbose: print(length)
if self.verbose: print(end_block)
for s, l, e in zip(start, length, end_block):
assert(e > s)
if s + l > e and self.padding:
#Need to pad since, the required length is bigger then the last block
if self.verbose: print(str(s) + ", " + str(l) + ", " + str(e) + " that is need to pad")
unpadded_data = data[int(s - 1):int(e - 1), (self.start_channel - 1):self.end_channel]
x_len, y_len = unpadded_data.shape
unpaded_data = data[int(s - 1):int(e - 1), (self.start_channel - 1):self.end_channel]
padding_size = int(s + l - e)
append_data = np.pad(unpadded_data, pad_width=((0, padding_size), (0, 0)), mode='reflect')
append_data = np.pad(unpaded_data, pad_width=((0, padding_size), (0, 0)), mode='reflect')
if self.verbose: print(append_data)
else:
append_data = data[int(s - 1):int(s + l - 1), (self.start_channel - 1):self.end_channel]
all_trials.append(append_data)
all_trials = np.array(all_trials)
......
......@@ -3,7 +3,7 @@ from preparation_config import preparation_config as config
import numpy as np
def left_right_task_data_preparation(feature_extraction = False):
def left_right_task_data_preparation(feature_extraction = False, verbose=False):
# We use the antisaccade dataset to extract the data for left and right benchmark task.
saccade = config['saccade_trigger']
fixation = config['fixation_trigger']
......@@ -13,7 +13,7 @@ def left_right_task_data_preparation(feature_extraction = False):
preparator = Preparator(load_directory=config['LOAD_ANTISACCADE_PATH'],
save_directory=config['SAVE_PATH'],
load_file_pattern=config['ANTISACCADE_FILE_PATTERN'],
save_file_name=config['output_name'], verbose=True)
save_file_name=config['output_name'], verbose=verbose)
preparator.extract_data_at_events(extract_pattern=[cue, saccade, fixation], name_start_time='Beginning of cue', start_time=lambda events: events['latency'],
name_length_time='Size blocks of 500', length_time=500,
start_channel=1, end_channel=129, padding=False)
......@@ -21,7 +21,7 @@ def left_right_task_data_preparation(feature_extraction = False):
preparator = Preparator(load_directory=config['LOAD_ANTISACCADE_PATH'],
save_directory=config['SAVE_PATH'],
load_file_pattern=config['ANTISACCADE_HILBERT_FILE_PATTERN'],
save_file_name=config['output_name'], verbose=True)
save_file_name=config['output_name'], verbose=verbose)
preparator.extract_data_at_events(extract_pattern=[cue, saccade, fixation], name_start_time='At saccade on-set', start_time=lambda events: events['latency'].shift(-1),
name_length_time='Fixed blocks of 1', length_time=1,
start_channel=1, end_channel=129, padding=False)
......@@ -34,7 +34,7 @@ def left_right_task_data_preparation(feature_extraction = False):
preparator.addLabel(name='Giving label 0 for left and 1 for right', f=lambda events: events['type'].apply(lambda x: 0 if x == '10' else 1))
preparator.run()
def direction_task_data_preparation(feature_extraction=False):
def direction_task_data_preparation(feature_extraction=False, verbose=False):
# We use the 'dots' dataset for direction task
fixation = config['fixation_trigger']
saccade = config['saccade_trigger']
......@@ -44,7 +44,7 @@ def direction_task_data_preparation(feature_extraction=False):
if not feature_extraction:
preparator = Preparator(load_directory=config['LOAD_DOTS_PATH'], save_directory=config['SAVE_PATH'],
load_file_pattern=config['DOTS_FILE_PATTERN'], save_file_name=config['output_name'],
verbose=True)
verbose=verbose)
# no padding, but cut 500 somewhere in between
# we are interested only on the 5-triggers (fixation 41 cue saccade fixation) and we cut 500 data points in the middle
preparator.extract_data_at_events(extract_pattern=[fixation, end_cue, cue, saccade, fixation],
......@@ -55,7 +55,7 @@ def direction_task_data_preparation(feature_extraction=False):
else:
preparator = Preparator(load_directory=config['LOAD_DOTS_PATH'], save_directory=config['SAVE_PATH'],
load_file_pattern=config['DOTS_HILBERT_FILE_PATTERN'], save_file_name=config['output_name'],
verbose=True)
verbose=verbose)
# we are interested only on the 5-triggers (fixation 41 cue saccade fixation) and we cut 1 point at saccade on-set
preparator.extract_data_at_events(extract_pattern=[fixation, end_cue, cue, saccade, fixation],
name_start_time='At saccade on-set',
......@@ -71,7 +71,7 @@ def direction_task_data_preparation(feature_extraction=False):
preparator.addLabel(name='angle', f=lambda events: np.arctan2(events['end_y'].shift(-3) - events['start_y'].shift(-3), events['end_x'].shift(-3) - events['start_x'].shift(-3)))
preparator.run()
def direction_with_processing_speed_dataset(feature_extraction=False):
def direction_with_processing_speed_dataset(feature_extraction=False, verbose=False):
# fixation, saccade, fixation
# saccade should be bigger than 0.5 (amplitude) (Repetition)
# We use the 'processing_speed' dataset for this direction task
......@@ -80,14 +80,14 @@ def direction_with_processing_speed_dataset(feature_extraction=False):
if not feature_extraction:
preparator = Preparator(load_directory=config['LOAD_PROCESSING_SPEED_PATH'], save_directory=config['SAVE_PATH'],
load_file_pattern=config['PROCESSING_SPEED_FILE_PATTERN'], save_file_name=config['output_name'], verbose=True)
load_file_pattern=config['PROCESSING_SPEED_FILE_PATTERN'], save_file_name=config['output_name'], verbose=verbose)
preparator.extract_data_at_events(extract_pattern=[fixation, saccade, fixation], name_start_time='at the fixation start',
start_time=lambda events: (events['latency']),
name_length_time='Fixed blocks of 500', length_time=500,
start_channel=1, end_channel=129, padding=True) # we are interested only on the triplets, fixation saccade fixation
else:
preparator = Preparator(load_directory=config['LOAD_PROCESSING_SPEED_PATH'], save_directory=config['SAVE_PATH'],
load_file_pattern=config['PROCESSING_SPEED_HILBERT_FILE_PATTERN'], save_file_name=config['output_name'], verbose=True)
load_file_pattern=config['PROCESSING_SPEED_HILBERT_FILE_PATTERN'], save_file_name=config['output_name'], verbose=verbose)
preparator.extract_data_at_events(extract_pattern=[fixation, saccade, fixation], name_start_time='at saccade onset', start_time=lambda events: (events['latency'].shift(-1)),
name_length_time='Size 1', length_time=1,
start_channel=1, end_channel=129, padding=False) # we are interested only on saccade on-set (where the data is feature extracted by hilbert trafo)
......@@ -95,17 +95,17 @@ def direction_with_processing_speed_dataset(feature_extraction=False):
preparator.addFilter(name='Keep only long enough fixations1', f=lambda events:events['duration'] > 50)
preparator.addFilter(name='Keep only big enough saccade', f=lambda events: events['amplitude'].shift(-1) > 0.5)
preparator.addFilter(name='Keep only long enough fixations2', f=lambda events: events['duration'].shift(-2) > 50)
preparator.addLabel(name='angle', f=lambda events: np.sqrt((events['end_x'].shift(-1) - events['start_x'].shift(-1)) ** 2 + (events['end_y'].shift(-1) - events['start_y'].shift(-1)) ** 2))
preparator.addLabel(name='amplitude', f=lambda events: np.arctan2(events['end_y'].shift(-1) - events['start_y'].shift(-1), events['end_x'].shift(-1) - events['start_x'].shift(-1)))
preparator.addLabel(name='amplitude', f=lambda events: np.sqrt((events['end_x'].shift(-1) - events['start_x'].shift(-1)) ** 2 + (events['end_y'].shift(-1) - events['start_y'].shift(-1)) ** 2))
preparator.addLabel(name='angle', f=lambda events: np.arctan2(events['end_y'].shift(-1) - events['start_y'].shift(-1), events['end_x'].shift(-1) - events['start_x'].shift(-1)))
preparator.run()
def position_task_data_preparation(feature_extraction):
def position_task_data_preparation(feature_extraction, verbose=False):
# We use the 'dots' dataset for position task
fixation = config['fixation_trigger']
if not feature_extraction:
preparator = Preparator(load_directory=config['LOAD_DOTS_PATH'], save_directory=config['SAVE_PATH'],
load_file_pattern=config['DOTS_FILE_PATTERN'], save_file_name=config['output_name'], verbose=True)
load_file_pattern=config['DOTS_FILE_PATTERN'], save_file_name=config['output_name'], verbose=verbose)
# no padding, but cut 500 somewhere in between
preparator.extract_data_at_events(extract_pattern=[fixation], name_start_time='At fixation start', start_time=lambda events: (events['latency']),
name_length_time='Fixed blocks of 500', length_time=500,
......@@ -119,7 +119,7 @@ def position_task_data_preparation(feature_extraction):
preparator.addLabel(name='y_position', f=lambda events: events['avgpos_y'])
preparator.run()
def segmentation_task_data_preparation(feature_extraction=False):
def segmentation_task_data_preparation(feature_extraction=False, verbose=False):
raise NotImplementedError("Segmentation task is not implemented yet.")
def main():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment