To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit e35e5537 authored by stehess's avatar stehess
Browse files

initial commit of source code for publication

parent d07e324f
"""
Stephan Wegner & Vithurjan Visuvalingam
pdz, ETH Zürich
2020
This file calls all files in the project, dependent on the definitions in definitions.py
"""
import os
import definitions
print('mode:',definitions.mode)
if definitions.operation['2DCNN_Inference']==1:
"""
(1)
2DCNN inference
input: video.avi (60 Hz), mask_rcnn_hands.h5 + mask_rcnn_yps_data_acc.h5 + video_times.txt (start-end-times)
variables: detection_min_confidence(obj + hands)
output: name.csv(obj+hands/frame), name_masked.avi(30 Hz, cut start-end), name_black_avi(30 Hz, cut start-end)
times: t_0*, t_-1*
"""
os.chdir("src\TwoDCNN\models")
os.system("python 2DCNN_inference.py")
#Return to Root directory
i = 0
while i <= 2:
os.chdir(os.path.dirname(os.getcwd()))
i += 1
if definitions.operation['extract_features']==1:
"""
(2)
extract features
input: GT import, videos (masked, black)
times GT: t_0, t_-1 -> adapted to t_0*, t_-1*
times videos: t_0*, t_-1*
ouput: image extraction with 30 Hz
ID = f"{n}_{cur_frame_time}"
data\datasets\extracted_images\train_val\blacked_mask\ID.png
data\datasets\extracted_images\test\blacked_mask\ID.png
link: label & ID (n_frame-time)
train_val_filled_values_id_label_map.csv
test_filled_values_id_label_map.csv
"""
os.chdir("src\ThreeDCNN\dataset_creation")
os.system("python extract_features.py --mode %s" % definitions.mode)
# Return to Root directory
i = 0
while i <= 2:
os.chdir(os.path.dirname(os.getcwd()))
i += 1
if definitions.operation['create_segments']==1:
"""
(3)
create segments
input calls train_val_filled_values_id_label_map.csv
Output: l.336 creates train_val_prop_dataset.csv file for training of NN
l.356 creates train_val_classification_dataset.csv file for training of NN
l.359 creates test_segment_dataset.csv
"""
os.chdir("src\ThreeDCNN\dataset_creation")
os.system("python create_segments.py --mode %s" % definitions.mode)
# Return to Root directory
i = 0
while i <= 2:
os.chdir(os.path.dirname(os.getcwd()))
i += 1
if definitions.operation['3DCNN_train_class']==1 and definitions.mode == 'train':
"""
(4)
train classification NN
input l.23 calls train_val_classification_dataset.csv
output *i_TCNN_class_acc_*_pycharm.h5
"""
os.chdir("src\ThreeDCNN\models\classification_network")
for a in range(definitions.length): #
filter=definitions.hyperparam[0][0][a]
kernel = definitions.hyperparam[1][0][a]
optimizer = definitions.hyperparam[2][0][a]
activation = definitions.hyperparam[3][0][a]
weight= definitions.hyperparam[4][0][a]
print('a', a)
i=definitions.start_fold # in definition: decision, which fold is start fold (0,1,2,3,4)
while i < definitions.runs:
start = definitions.starts[i]
end = definitions.ends[i]
print('startfold, start:', i, ',', start)
os.system("python train_model.py --filter %s --kernel %s --optimizer %s --activation %s --weight %s --start %s --end %s --start_epoch %i --model %s --learning_rate %f"
% (filter, kernel, optimizer, activation, weight, start, end, definitions.start_epoch, definitions.model, definitions.learning_rate))
i+=1
# Return to Root directory
i = 0
while i <= 3:
os.chdir(os.path.dirname(os.getcwd()))
i += 1
if definitions.operation['3DCNN_predict']==1 and definitions.mode == 'test':
"""
(5)
predict on test set
input: test nums, path to classification NN, test images, test segments
output: segment and singe-frame level:
figures (confusion matrix, classification report, ), predictions (classification scores)
csv files with predictions
"""
os.chdir("src\ThreeDCNN\models\prediction")
os.system("python predict.py --mode %s" % definitions.mode)
# Return to Root directory
i=0
while i<=3:
os.chdir(os.path.dirname(os.getcwd()))
i+=1
if definitions.operation['post-processing']==1 and definitions.mode == 'test':
"""
(6)
post-process results from prediction on test test
input: predictions, OOI hits
output: figures
"""
os.chdir("src\ThreeDCNN\models\post_processing")
os.system("python post_process.py")
# Return to Root directory
i = 0
while i <= 3:
os.chdir(os.path.dirname(os.getcwd()))
i += 1
\ No newline at end of file
from tensorflow.keras import optimizers
""" Optimizer chooses from this table (values in classification_network\train.py)
#0: sgd1 = optimizers.SGD(lr=0.001, decay=0.005, momentum=0.9, nesterov=True)
#1: sgd = optimizers.SGD(lr=0.01, decay=0.05, momentum=0.9, nesterov=True)
#2: rmsprop = optimizers.RMSprop(learning_rate=0.001, rho=0.9)
#3: adam = optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, amsgrad=False)
#4: adagrad=optimizers.Adagrad(learning_rate=0.001, initial_accumulator_value=0.1, epsilon=1e-7) #well suited for sparse data
"""
#optimizer=[sgd1, sgd, rmsprop, adam]
param_grid = {'filter_base': [32], #32,48,64
'kernel_base': [3], #,5
'optimizer': [4],#1,,3 #1:sgd, 2: rmsprop 3:adam, 4: adagrad
'activation':[1],#,2 #1: relu, 2: tanh (seems to not work out)
'weight': [6]} #5,
opti=[]
filter=[]
kernel=[]
activation=[]
weight=[]
for i in range(len(param_grid['filter_base'])):
for j in range(len(param_grid['kernel_base'])):
for k in range(len(param_grid['optimizer'])):
for l in range(len(param_grid['activation'])):
for m in range(len(param_grid['weight'])):
filter.append(param_grid['filter_base'][i])
kernel.append(param_grid['kernel_base'][j])
opti.append(param_grid['optimizer'][k])
activation.append((param_grid['activation'][l]))
weight.append(((param_grid['weight'][m])))
length=len(filter)
hyperparam=[[filter], [kernel], [opti], [activation], [weight]]
\ No newline at end of file
"""
Stephan Wegner & Vithurjan Visuvalingam
pdz, ETH Zürich
2020
This file contains all functions to create segments for 3DCNN from the extract features and images
"""
import pandas as pd
import numpy as np
import argparse
import os
import sys
i=0
while i<=2:
os.chdir(os.path.dirname(os.getcwd()))
i+=1
sys.path.append(os.getcwd())
import definitions
from src.ThreeDCNN.dataset_creation import utils
if __name__ == '__main__':
print("start create segments")
ROOT_DIR = definitions.ROOT_DIR
# Args
parser = argparse.ArgumentParser(description='Create segments for training or test set')
parser.add_argument('--segment_input_dir', default=os.path.join(ROOT_DIR, 'data/datasets'))
parser.add_argument('--segment_output_dir', default=os.path.join(ROOT_DIR, 'data/datasets/filled_values_segments'))
parser.add_argument('--mode', required=True)
args = parser.parse_args()
if args.mode == 'train':
id_label_mapping = pd.read_csv(args.segment_input_dir + "\\" + r"train_val_filled_values_id_label_map.csv")
first_video_number = definitions.train_val_nums[0]
path_to_save_segments = args.segment_output_dir + "\\" + r"train_val"
elif args.mode == 'test':
id_label_mapping = pd.read_csv(args.segment_input_dir + "\\" + r"test_filled_values_id_label_map.csv")
first_video_number = definitions.test_nums[0]
path_to_save_segments = args.segment_output_dir+ "\\" + r"test"
path_to_save_gt_segments = args.segment_output_dir+ "\\" + r"test" + '\\GT_segments'
else:
print("chose --mode = train or test")
ids = id_label_mapping['ids'] #ids = num_video_frame-time[ms]
labels = id_label_mapping['labels'] #labels = 0,1,2,3 for linked frames
# get start points of the videos and the video numbers of each video
indexes, num_videos = utils.get_start_end_list_of_videos(ids) #indexes = index in list-of-ids for start and end of video, num_videos = num_video
# datastructure needed for index retrieval of given ID
start_indexes_of_videos = dict(zip(num_videos, indexes[:len(indexes) - 1]))
segment_size = 16 # number of frames per segment
# overlap E (0,1)
overlap = 0.5 #50% overlap
length = 16 # timespan of the segments, e.g 16 = 0.5 s
classes = []
for i in range(len(definitions.HEC_classes)):
classes.append(i)
parameter_segments = [[length, overlap]] # it is also possible to create segments of different length, just add more sets of parameters
"""
# Create set of segments with length 0.5 s
"""
label_segments=[]
segments =[]
i=0
while i < len(parameter_segments):
length = parameter_segments[i][0]
overlap = parameter_segments[i][1]
label, segment = utils.labels_segments(indexes=indexes, ids=ids, length=length, segment_size=segment_size, overlap=overlap, labels_raw=labels, classes= classes, start_indexes_of_videos= start_indexes_of_videos)
label_segments = label_segments + label
segments = segments + segment
i += 1
# name each segment according to its video number its starting point and its length
ids_predict = []
for segment in segments:
ids_predict.append(segment[0] + "_" + str(round(utils.time_stamp(segment[-1]) - utils.time_stamp(segment[0]))))
# save the snapshot ids corresponding to a segment in a directory
for i in range(len(segments)):
np.save(os.path.join(path_to_save_segments, str(ids[i])), np.array(segments[i]))
# create a dataframe having all information about the data generated and save it
segment_dataset = pd.DataFrame(data={"ids": ids_predict, "labels": label_segments})
if args.mode == 'train':
segment_dataset.to_csv(ROOT_DIR + "\\" + r"data\datasets\train_val_segment_dataset.csv", sep=',', index=False)
elif args.mode == 'test':
segment_dataset.to_csv(ROOT_DIR + "\\" + r"data\datasets\test_segment_dataset.csv", sep=',', index=False)
else:
print("chose --mode = train or test")
\ No newline at end of file
"""
Stephan Wegner & Vithurjan Visuvalingam
pdz, ETH Zürich
2020
This file contains all functions to create segments for 3DCNN from the extract features and images
"""
import pandas as pd
import numpy as np
import argparse
import os
import random
import sys
i=0
while i<=2:
os.chdir(os.path.dirname(os.getcwd()))
i+=1
sys.path.append(os.getcwd())
from definitions import ROOT_DIR
from definitions import train_val_nums
from definitions import test_nums
# create a dictionary from snapshot ids
#########################
# Functions #
#########################
# METADATA
# get timestamp of a snapshot
def time_stamp(ID):
return float(ID.split('_')[1])
# given the ID return the video number it comes from
def video_number(ID):
return ID.split('_')[0]
# SEGMENTS
# generate the segments given a video
def get_segments_from_video(index_to_start, index_to_stop, list_of_ids, segment_length, segment_size, overlap):
segments = []
current_segment = []
step_size_segment = (1-overlap) * segment_length
j = 0
while index_to_start + segment_length - 1 < index_to_stop:
current_segment = []
j = 0
while j < segment_length:
current_segment.append(list_of_ids[index_to_start + j])
j = j + segment_length / segment_size #use every 2nd frame in 1s
segments.append(current_segment)
index_to_start = index_to_start + step_size_segment
return segments
# get start points of videos
def get_start_end_list_of_videos(list_of_ids):
num_videos = []
cur_id = '-1'
indexes = []
for i in range(len(list_of_ids)):
ID = video_number(list_of_ids[i])
if ID != cur_id:
indexes.append(i)
num_videos.append(ID)
cur_id = ID
indexes.append(len(list_of_ids))
return indexes, num_videos
# return a list of all the segments from the given videos
def get_segments_of_all_videos(start_of_videos, list_of_ids, segment_length, segment_size, overlap):
i = 1
segments_list = []
current_list = []
while i < len(start_of_videos):
to_start = start_of_videos[i - 1]
to_end = start_of_videos[i]
current_list = get_segments_from_video(to_start, to_end, list_of_ids, segment_length, segment_size, overlap)
for elem in current_list:
segments_list.append(elem)
i = i + 1
return segments_list
# in a sorted list given the ID find the index of it
def index_of_id(ID):
num_video = video_number(ID)
time = time_stamp(ID)
start_index = start_indexes_of_videos[num_video]
offset = round(time / (100 / 3))
return int(start_index + offset)
def labels_segments(indexes, ids, length, segment_size, overlap, labels_raw, classes):
segment_list = []
current_list = get_segments_of_all_videos(start_of_videos = indexes, list_of_ids=ids, segment_length=length, segment_size=segment_size, overlap=overlap)
for elem in current_list:
segment_list.append(elem)
labels_seg = []
for i in range(len(segment_list)):
dataset_labels_segment = []
for j in range(segment_size):
dataset_labels_segment.append(labels_raw[index_of_id(segment_list[i][j])])
"""
#Choose the most dominant label for segment
labels_segment = np.bincount(dataset_labels_segment)
labels_segment = np.argmax(labels_segment)
"""
# Choose the most dominant label for segment, if it represents at least 75% of all frames in segment, else 0 (BG)
occ = []
for i in range(len(classes)):
occ.append(dataset_labels_segment.count(classes[i]))
label = []
for i in range(len(occ)):
if occ[i] >= 0.75 * segment_size:
label.append(1)
else:
label.append(0)
labels_segment = np.argmax(label) #labels_segment = 0, 1, 2 or 3
labels_seg.append(labels_segment)
segments = segment_list
return labels_seg, segments
#########################
# Code #
#########################
if __name__ == '__main__':
print("start create segments")
# Args
parser = argparse.ArgumentParser(description='Create segments for training or test set')
parser.add_argument('--segment_input_dir', default=os.path.join(ROOT_DIR, 'data/datasets'))
parser.add_argument('--segment_output_dir', default=os.path.join(ROOT_DIR, 'data/datasets/filled_values_segments'))
parser.add_argument('--mode', required=True)
args = parser.parse_args()
if args.mode == 'train':
id_label_mapping = pd.read_csv(args.segment_input_dir + "\\" + r"train_val_filled_values_id_label_map.csv")
first_video_number = train_val_nums[0]
path_to_save_segments = args.segment_output_dir + "\\" + r"train_val"
elif args.mode == 'test':
id_label_mapping = pd.read_csv(args.segment_input_dir + "\\" + r"test_filled_values_id_label_map.csv")
first_video_number = test_nums[0]
path_to_save_segments = args.segment_output_dir+ "\\" + r"test"
else:
print("chose --mode = train or test")
ids = id_label_mapping['ids'] #ids = num_video_frame-time[ms]
labels = id_label_mapping['labels'] #labels = 0,1,2,3 for linked frames
# get start points of the videos and the video numbers of each video
indexes, num_videos = get_start_end_list_of_videos(ids) #indexes = index in list-of-ids for start and end of video, num_videos = num_video
# datastructure needed for index retrieval of given ID
start_indexes_of_videos = dict(zip(num_videos, indexes[:len(indexes) - 1]))
length_go = 64 # timespan of the segments for guiding & oberving classes
length_d = 16 # timespan of the segments for directing class
# stepsize E (0,1)
overlap_go = 0.75 #75% overlap
overlap_d = 0.5 #75% overlap
segment_size = 16
classes = [0, 1, 2, 3]
parameter_segments = [[length_d, overlap_d]]#[length_go, overlap_go],
"""
# Create two sets of segments with length 0.5 s and 2 s
"""
label_segments=[]
segments =[]
i=0
while i < len(parameter_segments):
length = parameter_segments[i][0]
overlap = parameter_segments[i][1]
label, segment = labels_segments(indexes=indexes, ids=ids, length=length, segment_size=segment_size, overlap=overlap, labels_raw=labels, classes= classes)
label_segments = label_segments + label
segments = segments + segment
i += 1
"""
label_segments_with_0 = label_segments
segments_with_0 = segments
index = [i for i in range(len(label_segments)) if label_segments[i] == 0]
for i in range(len(index)):
ind = index[i]
index = [x - 1 for x in index]
del label_segments[ind]
del segments[ind]
label_segments = [x - 1 for x in label_segments]
print('len label & segments', len(label_segments), len(segments))
"""
# name each segment according to its video number its starting point and its length
ids = []
for segment in segments:
ids.append(segment[0] + "_" + str(round(time_stamp(segment[-1]) - time_stamp(segment[0]))))
# save the snapshot ids corresponding to a segment in a directory
for i in range(len(segments)):
np.save(os.path.join(path_to_save_segments, str(ids[i])), np.array(segments[i]))
"""
ids_with_0 = []
for segment_with_0 in segments_with_0:
ids_with_0.append(segment_with_0[0] + "_" + str(round(time_stamp(segment_with_0[-1]) - time_stamp(segment_with_0[0]))))
"""
index_0 = [i for i in range(len(label_segments)) if label_segments[i] == 0]
index_1 = [i for i in range(len(label_segments)) if label_segments[i] > 0]
"""
random.shuffle(index_1)
index_1 = index_1[:len(index_0)]
print('len index 0,1', len(index_0), len(index_1))
print('ids idx 0',ids[index_1[0]])
print(len(ids))
"""
label_segments_prop = []
ids_prop = []
i=0
while i < len(index_0):
label_segments_prop.append(0)
ids_prop.append(ids[index_0[i]])
i+=1
j=0
while j < len(index_1):
label_segments_prop.append(1)
ids_prop.append(ids[index_1[i]])
j+=1
# create a dataframe having all information about the data generated and save it
segment_dataset = pd.DataFrame(data={"ids": ids, "labels": label_segments})
segment_dataset_prop = pd.DataFrame(data={"ids": ids_prop, "labels": label_segments_prop})
if args.mode == 'train':
segment_dataset.to_csv(ROOT_DIR + "\\" + r"data\datasets\classification_NN\train_val_go_classification_dataset.csv", sep=',', index=False)
segment_dataset_prop.to_csv(ROOT_DIR + "\\" + r"data\datasets\proposal_NN\train_val_proposal_dataset.csv", sep=',',index=False)
elif args.mode == 'test':
segment_dataset.to_csv(ROOT_DIR + "\\" + r"data\datasets\test_go_segment_dataset.csv", sep=',', index=False)
segment_dataset_prop.to_csv(ROOT_DIR + "\\" + r"data\datasets\test_proposal_dataset.csv", sep=',',index=False)
else:
print("chose --mode = train or test")
\ No newline at end of file
"""
Stephan Wegner & Vithurjan Visuvalingam
pdz, ETH Zürich
2020
This file contains all functions to extract features and images from the masked videos.
"""
import cv2
import pandas as pd
import argparse
import os
import sys
i=0
while i<=2:
os.chdir(os.path.dirname(os.getcwd()))
i+=1
sys.path.append(os.getcwd())
import definitions
from src.ThreeDCNN.dataset_creation import utils
def extract_features(args, n, path_to_save_images): #performes loop n times
# load GT labels, bring to needed format with t_0 = t_start
name='behaviour_ground_truth_' + definitions.name_base + str(n)
GT_INPUT_DIR = args.gt_input_dir