To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit e35e5537 authored by stehess's avatar stehess
Browse files

initial commit of source code for publication

parent d07e324f
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Flatten, Dense, Conv3D, MaxPooling3D, Dropout, BatchNormalization, LSTM, Reshape
#3D CNN Input: [batch size (None), volumes (16,128,128), channels (3)]
#LSTM Input: [batch, timesteps, features]
def classification_model(filter_base, kernel_base, activation, initializer, regularizer, optimizer, target):
model = Sequential()
model.add(Conv3D(filter_base, kernel_size=kernel_base, activation=activation, input_shape=(16, 128, 128, 3), kernel_initializer=initializer, kernel_regularizer=regularizer))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(BatchNormalization())
model.add(Conv3D(2*filter_base, kernel_size=kernel_base, padding='same', activation=activation, kernel_initializer=initializer, kernel_regularizer=regularizer))
#model.add(Conv3D(2 * filter_base, kernel_size=kernel_base, padding='same', activation=activation, kernel_initializer=initializer, kernel_regularizer=regularizer))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(BatchNormalization())
model.add(Flatten())
model.add(Dense(256, activation=activation, kernel_initializer=initializer, kernel_regularizer=regularizer))
model.add(Dropout(0.2))
model.add(Dense(target, activation='softmax', kernel_initializer=tf.keras.initializers.glorot_normal()))
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['categorical_accuracy'])
model.summary()
return model
def classification_model_lstm(filter_base, kernel_base, activation, initializer, regularizer, optimizer, target):
model = Sequential()
model.add(Conv3D(filter_base, kernel_size=kernel_base, activation=activation, input_shape=(16, 128, 128, 3), kernel_initializer=initializer, kernel_regularizer=regularizer))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(BatchNormalization())
model.add(Conv3D(2 * filter_base, kernel_size=kernel_base, padding='same', activation=activation, kernel_initializer=initializer, kernel_regularizer=regularizer))
model.add(Conv3D(2 * filter_base, kernel_size=kernel_base, padding='same', activation=activation, kernel_initializer=initializer, kernel_regularizer=regularizer))
model.add(MaxPooling3D(pool_size=(2, 2, 2)))
model.add(BatchNormalization())
model.add(Reshape((-1, 61504)))
model.add(LSTM(units=256, activation='tanh', recurrent_activation='sigmoid', use_bias=True, kernel_initializer='glorot_normal'))
model.add(Dense(256, activation=activation, kernel_initializer=initializer, kernel_regularizer=regularizer))
model.add(Dense(target, activation='softmax', kernel_initializer=initializer))
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['categorical_accuracy'])
model.summary()
return model
\ No newline at end of file
"""
Stephan Wegner & Vithurjan Visuvalingam
pdz, ETH Zürich
2020
This file contains all functions to train the classification NN
Part of the software code for stop and restart training is based on the pyimageserach article by Adrian Rosebrock on September 23, 2019
https://www.pyimagesearch.com/2019/09/23/keras-starting-stopping-and-resuming-training/
"""
print('start train classification NN')
import numpy as np
import pandas as pd
from sklearn.metrics import classification_report, confusion_matrix
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras import optimizers, regularizers
from tensorflow.keras.backend import set_session
from tensorflow.keras.models import load_model
import tensorflow.keras.backend as K
import random
import os
import sys
from time import gmtime, strftime
import argparse
import classification_model as cm
i = 0
while i <= 3:
os.chdir(os.path.dirname(os.getcwd()))
i += 1
sys.path.append(os.getcwd())
import definitions
from src.ThreeDCNN.models.data_generator.ThreeDimCNN_datagenerator import ThreeDimCNN_datagenerator
from src.ThreeDCNN.models.classification_network import utils
ROOT_DIR = definitions.ROOT_DIR
HEC_labels = np.arange(len(definitions.HEC_classes))
# paths to load and store data
path_to_id_label_mapping = ROOT_DIR + "\\" + r"data\datasets\train_val_segment_dataset.csv"
path_to_store_model = ROOT_DIR + "\\" + r"models\ThreeDCNN\classification_NN"
path_to_store_figures = ROOT_DIR + "\\" + r"reports\figures"
path_to_store_training_plots = path_to_store_figures + '\\temp_acc_loss\\live_acc_loss_training'
path_to_store_training_model = path_to_store_model+ '\\'+'temp_training'
# load id label mapping
df_id_label_map = pd.read_csv(path_to_id_label_mapping)
ids = list(df_id_label_map['ids']) # ids: video-num_frame-time[ms]_segment-length
labels = list(df_id_label_map['labels']) # labels: 0,1,2,3,4 (BG, Guiding, Directing, Checking, Observing)
# create labels dictionary
dict_labels = dict(zip(ids, labels))
triple_ids = [ID.split('_') for ID in ids] # triple_ids[i] = ['num_video', 'start_time_segment', 'segment_length']
num_triple_ids = [[float(a), float(b), float(c)] for [a, b, c] in triple_ids] # convert string to float
s_num_triple_ids = sorted(num_triple_ids)
sorted_ids = [f"{int(a)}" + "_" + f"{float(b)}" + "_" + f"{int(c)}" for [a, b, c] in s_num_triple_ids]
sorted_labels = [dict_labels[ID] for ID in sorted_ids]
ids = sorted_ids
labels = sorted_labels
# sort the background and activity segments
# Creates a dictionary, which stores position of every segment in list of labels with label i,
# e.g. index_label = {'index_label_0': [0,1,2,3,21,...], 'index_label_1': [4,5,6,...], ...}
index_label = {}
for i in range(len(HEC_labels)):
index_label['index_label_{0}'.format(i)] = [j for j in range(len(labels)) if labels[j] == i]
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='train classification NN')
parser.add_argument('--filter')
parser.add_argument('--kernel')
parser.add_argument('--optimizer')
parser.add_argument('--activation')
parser.add_argument('--weight')
parser.add_argument('--start')
parser.add_argument('--end')
parser.add_argument("-m", "--model", type=str, help="path to *specific* model checkpoint to load")
parser.add_argument("-s", "--start_epoch", type=int, default=0, help="epoch to restart training at")
parser.add_argument("-lr", "--learning_rate", type=float, default=1e-3, help="epoch to restart training at")
argument = parser.parse_args()
# different optimizers
sgd1 = optimizers.SGD(lr=0.01, decay=0.05, momentum=0.9, nesterov=True) # momentum = 0.9 is standard value, keep
sgd = optimizers.SGD(lr=0.001, decay=0.005, momentum=0.9, nesterov=True)
rmsprop = optimizers.RMSprop(learning_rate=0.001, rho=0.9)
adam = optimizers.Adam(learning_rate=0.00001, beta_1=0.9, beta_2=0.999, amsgrad=False, clipnorm=5)
adagrad = optimizers.Adagrad(learning_rate=1e-3, initial_accumulator_value=0.1, epsilon=1e-7,
clipnorm=10) # well suited for sparse data
optimizer = [sgd1, sgd, rmsprop, adam, adagrad]
filter_base = int(argument.filter) # e.g. 16,32,48, 64
kernel_base = int(argument.kernel) # e.g. 3,5
optimizer = optimizer[int(argument.optimizer)]
activation = int(argument.activation)
weight = int(argument.weight) # obsolete when using cw_i
if activation == 1:
activation = 'relu'
initializer = tf.keras.initializers.he_normal() # activation = 'relu'
# initializer = tf.keras.initializers.he_uniform()
elif activation == 2:
activation = 'tanh'
initializer = tf.keras.initializers.glorot_normal() # activation = 'thanh'
# initializer= tf.keras.initializers.glorot_uniform()
else:
print('no activation chosen')
regularizer = regularizers.l1_l2(l1=1e-4, l2=1e-4)
# *** create the training and validation set using k-fold cross validation ***
start = float(argument.start) # if k =5, start=[0.01, 0.21, 0.41, 0.61, 0.81]
end = float(argument.end)
val = {}
train_start = {}
train_end = {}
train = {}
# creates a dictionary for the validation set and training set. The training set is composed of data before start and after end point of the vlaidation set
for i in range(len(HEC_labels)):
# e.g. index_label = {'index_label_0': [0,1,2,3,21,...], 'index_label_1': [4,5,6,...], ...}
index_label_i = index_label["index_label_{0}".format(i)]
val["val_{0}".format(i)] = index_label_i[int(len(index_label_i) * start):int(len(index_label_i) * end)]
train_start["train_{0}_1".format(i)] = index_label_i[:int(len(index_label_i) * start)]
train_end["train_{0}_2".format(i)] = index_label_i[int(len(index_label_i) * end):]
train["train_{0}".format(i)] = train_start["train_{0}_1".format(i)] + train_end["train_{0}_2".format(i)]
if definitions.undersampling == True:
lens_trains = []
# Shuffles all entries in the training set dictionary
for i in range(len(HEC_labels)):
random.shuffle(train["train_{0}".format(i)])
lens_trains.append(len(train["train_{0}".format(i)]))
# Keeps only first entries for each HEC class, so there is the same number of entries
for i in range(len(HEC_labels)):
train["train_{0}".format(i)] = train["train_{0}".format(i)][:min(lens_trains)]
train_indexes = []
val_indexes = []
# Combines HEC class subsets for training and validation datasets respectively
for i in range(len(HEC_labels)):
train_indexes += train["train_{0}".format(i)]
val_indexes += val["val_{0}".format(i)]
# Use indexes (positions in list of labels) to find the corresponding ids for train and val
train_ids = [ids[i] for i in train_indexes] # train_ids[i] = [num-video_start-time-segment_segment-length]
val_ids = [ids[i] for i in val_indexes] # val_ids[i] = [num-video_start-time-segment_segment-length]
random.shuffle(train_ids)
random.shuffle(val_ids)
# class weights
class_weight = {0: 1,
1: 1,
2: 1,
3: 1,
4: 1
}
elif definitions.undersampling == False:
# store position of every segment in list of labels, which belongs to train or val set
train_indexes = []
val_indexes = []
for i in range(len(HEC_labels)):
train_indexes += train["train_{0}".format(i)]
val_indexes += val["val_{0}".format(i)]
# Use indexes (positions in list of labels) to find the corresponding ids for train and val
train_ids = [ids[i] for i in train_indexes] # train_ids[i] = [num-video_start-time-segment_segment-length]
val_ids = [ids[i] for i in val_indexes] # val_ids[i] = [num-video_start-time-segment_segment-length]
random.shuffle(train_ids)
random.shuffle(val_ids)
# class-imbalance: class weights according to the number of representations in the validation dataset, with the lowest class weight for the most prominent class
# creates a dictionary e.g. len_val = {'len_val0': 100, 'len_val1': 20, ...}
len_val = {}
for i in range(len(HEC_labels)):
len_val["len_val{0}".format(i)] = len(val["val_{0}".format(i)])
print('[INFO] len val',len_val)
len_val_array = []
for i in range(len(HEC_labels)):
len_val_array.append(len_val["len_val{0}".format(i)])
cw = {}
for i in range(len(HEC_labels)):
cw["cw_{0}".format(i)] = float(max(len_val_array) / len_val["len_val{0}".format(i)])
class_weight = {}
for i in range(len(HEC_labels)):
class_weight[i] = cw["cw_{0}".format(i)] * cw["cw_{0}".format(i)]
else:
print('Please select undersampling mode = True or False')
print("[INFO] cw's:", class_weight)
# create partition dictionary
dict_partition = {}
dict_partition['train'] = train_ids
dict_partition['validation'] = val_ids
# create the parameters for the train and validation generators
params = {'dim_feature_sequence': (16, 128, 128, 3), # 16 images, 128x128 px, rgb
'batch_size': 10, # 10 works for NVIDIA GeForce RTX 2080 SUPER, 8.0 GB memory
'n_classes': len(definitions.HEC_classes),
'shuffle': False} # Training and validation set is already shuffled
# create the train, validation and test generator
training_generator = ThreeDimCNN_datagenerator(dict_partition['train'], dict_labels, **params) # i_generator[0][1].shape = batch_size x 4 labels
validation_generator = ThreeDimCNN_datagenerator(dict_partition['validation'], dict_labels, **params) # i_generator[0][0].shape = batch_size x 16 (128x128px) images
print('validation_generator[0][1]', validation_generator[0][1].shape) # batch_size x 4 labels
print('validation_generator[0][0]', validation_generator[0][0].shape) # batch_size x 16 128x128px images
####################
# Create the model #
####################
# train the model and save the history data
# set configuration for tensorflow
config = tf.ConfigProto(
gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.4)
)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
set_session(session)
if argument.model == 'None':
print("[INFO] compiling model...")
model = cm.classification_model(filter_base=filter_base, kernel_base=kernel_base, activation=activation, initializer=initializer, regularizer=regularizer, optimizer=optimizer, target= len(definitions.HEC_classes))
#model = cm.classification_model_lstm(filter_base=filter_base, kernel_base=kernel_base, activation=activation, initializer=initializer, regularizer=regularizer, optimizer=optimizer, target = len(definitions.HEC_classes))
else:
# load the checkpoint from disk
print("[INFO] loading {}...".format(argument.model))
model = load_model(argument.model)
# update the learning rate
print("[INFO] old learning rate: {}".format(K.get_value(model.optimizer.lr)))
K.set_value(model.optimizer.lr, argument.learning_rate)
print("[INFO] new learning rate: {}".format(K.get_value(model.optimizer.lr)))
EarylStopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, min_delta=0, restore_best_weights=True)
EpochCheckpoint = utils.EpochCheckpoint(outputPath=path_to_store_training_model, every=5, startAt=argument.start_epoch)
TrainingMonitor = utils.TrainingMonitor(figPath=path_to_store_training_plots, startAt=argument.start_epoch)
history = model.fit_generator(generator=training_generator,
validation_data=validation_generator,
use_multiprocessing=True,
epochs=definitions.epochs,
workers=0,
class_weight=class_weight,
callbacks=[EarylStopping, EpochCheckpoint, TrainingMonitor]
)
time = strftime("%Y%b%d_%H%M", gmtime())
identifier = str(start) + '-' + str(filter_base) + '-' + str(kernel_base) + '-' + str(
argument.optimizer) + '-' + str(argument.activation) + '-' + str(argument.weight)
print(identifier)
# plot the training performance
plt.plot(history.history['categorical_accuracy'])
plt.plot(history.history['val_categorical_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.savefig(path_to_store_figures + f"\\acc\\" + time + '_' + identifier + "_acc_small.png")
plt.clf()
# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
# plt.show()
plt.savefig(path_to_store_figures + f"\\loss\\" + time + '_' + identifier + f"_loss_small.png")
plt.clf()
# get the loss and accuracy of the model
loss, acc = model.evaluate_generator(generator=validation_generator, use_multiprocessing=True, workers=0)
# print the confusion matrix and classification report of the model performance
Y_pred = model.predict_generator(validation_generator)
y_pred = np.argmax(Y_pred, axis=1)
target_names = definitions.HEC_classes
Ypred_0 = []
for i in range(len(Y_pred)):
Ypred_0.append(Y_pred[i][0])
Ypred_1 = []
for i in range(len(Y_pred)):
Ypred_1.append(Y_pred[i][1])
Ypred_2 = []
for i in range(len(Y_pred)):
Ypred_2.append(Y_pred[i][2])
Ypred_3 = []
for i in range(len(Y_pred)):
Ypred_3.append(Y_pred[i][3])
Ypred_4 = []
for i in range(len(Y_pred)):
Ypred_4.append(Y_pred[i][4])
y_true = []
for i in range(int(len(y_pred) / params['batch_size'])):
temp = []
for j in range(params['batch_size']):
temp = np.append(temp, int(np.argmax(validation_generator[i][1][j]))) # validation_generator[0][1].shape = batch_size x 4 labels
y_true = np.append(y_true, temp)
t_dict = {'labels pred': y_pred, 'labels true': y_true, 'val id': val_ids[:len(y_pred)],
'value pred 0': Ypred_0,'value pred 1': Ypred_1,'value pred 2': Ypred_2 , 'value pred 3': Ypred_3, 'value pred 4': Ypred_4}
df = pd.DataFrame(t_dict)
df.to_csv(
path_to_store_figures + f"\\ComparisonPredTrue\\" + time + '_' + identifier + r'classification_predictions-true.csv',
sep=',', index=False)
plt.figure(figsize=[10.0, 6.3])
plt.cool()
plt.plot(y_true, 'b*')
plt.plot(y_pred, 'r+')
plt.title('comparison pred vs true')
plt.ylabel('label')
plt.xlabel('sequence')
plt.legend(['y_true', 'y_pred'])
plt.savefig(path_to_store_figures + f"\\ComparisonPredTrue\\" + time + '_' + identifier + f"_loss_small.png")
plt.clf()
print('acc', i, acc)
print('loss', i, loss)
""" """
print('Confusion Matrix')
conf_mat = confusion_matrix(y_true, y_pred, labels=np.arange(len(definitions.HEC_classes)))
print(conf_mat)
print('Classification Report')
class_rep = classification_report(y_true, y_pred, target_names=target_names)
print(class_rep)
df_conv = pd.DataFrame.from_dict(conf_mat)
df_conv.to_csv(path_to_store_figures + f"\\ConfusionMat\\" + time + '_' + identifier + f".csv")
path = path_to_store_figures + f"\\ClassificationRep\\" + time + '_' + identifier + f".txt"
text_file = open(path, 'w')
n = text_file.write(class_rep)
text_file.close()
# save the model
model.save(path_to_store_model + "\\" + time + '_' + str(start) + '_' + r'TCNN_class_acc_' + str(acc) + '.h5',
include_optimizer=True)
del model
del history
\ No newline at end of file
"""
Stephan Wegner & Vithurjan Visuvalingam
pdz, ETH Zürich
2020
This file contains all functions to train the classification NN
"""
print('start train classification NN')
import numpy as np
import pandas as pd
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from keras.wrappers.scikit_learn import KerasClassifier
from tensorflow.keras import optimizers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Flatten, Dense, Conv3D, MaxPooling3D
from tensorflow.keras.backend import set_session
import random
import os
import sys
from time import strftime, gmtime
i=0
while i<=3:
os.chdir(os.path.dirname(os.getcwd()))
i+=1
sys.path.append(os.getcwd())
from definitions import ROOT_DIR
from src.ThreeDCNN.models.data_generator.ThreeDimCNN_datagenerator import ThreeDimCNN_datagenerator
#set configuration for tensorflow
config = tf.ConfigProto(
gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.4)
# device_count = {'GPU': 1}
)
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
set_session(session)
# paths to load and store data
path_to_id_label_mapping = ROOT_DIR + "\\" + r"data\datasets\classification_NN\train_val_classification_dataset.csv"
path_to_store_model = ROOT_DIR + "\\" + r"models\ThreeDCNN\classification_NN"
path_to_store_figures = ROOT_DIR + "\\" + r"reports\figures"
# load id label mapping
df_id_label_map = pd.read_csv(path_to_id_label_mapping)
ids = list(df_id_label_map['ids'])
labels = list(df_id_label_map['labels'])
# create labels dictionary
dict_labels = dict(zip(ids, labels))
triple_ids = [ID.split('_') for ID in ids]
num_triple_ids = [[float(a), float(b), float(c)] for [a, b, c] in triple_ids]
s_num_triple_ids = sorted(num_triple_ids)
sorted_ids = [f"{int(a)}" + "_" + f"{float(b)}" + "_" + f"{int(c)}" for [a, b, c] in s_num_triple_ids]
sorted_labels = [dict_labels[ID] for ID in sorted_ids]
ids = sorted_ids
labels = sorted_labels
acc_per_fold = []
loss_per_fold = []
# sort the background and activity segments
index_label_0 = [i for i in range(len(labels)) if labels[i] == 0]
index_label_1 = [i for i in range(len(labels)) if labels[i] == 1]
index_label_2 = [i for i in range(len(labels)) if labels[i] == 2]
index_label_3 = [i for i in range(len(labels)) if labels[i] == 3]
# *** k-fold cross validation ***
"""
k=5
step=1/k
i=0
while i<k:
"""
# create the training and validation set
# create the training and validation set, 80/20 split, simple hold-out split
train_0 = index_label_0[:int(len(index_label_0) * 0.9)] #NEW
val_0 = index_label_0[int(len(index_label_0) * 0.9):] #NEW
train_1 = index_label_1[:int(len(index_label_1) * 0.9)] #NEW
val_1 = index_label_1[int(len(index_label_1) * 0.9):] #NEW
train_2 = index_label_2[:int(len(index_label_2) * 0.9)] #NEW
val_2 = index_label_2[int(len(index_label_2) * 0.9):] #NEW
train_3 = index_label_3[:int(len(index_label_3) * 0.9)] #NEW
val_3 = index_label_3[int(len(index_label_3) * 0.9):] #NEW
"""
start = round((1 / 100 + 10 * step * i / 10), 2)
end = round((start + step - 1 / 100), 2)
val_0 = index_label_0[int(len(index_label_0)*start):int(len(index_label_0)*end)]
train_0_1 = index_label_0[:int(len(index_label_0)*start)]
train_0_2 = index_label_0[int(len(index_label_0)*end):]
val_1 = index_label_1[int(len(index_label_1) * start):int(len(index_label_1) * end)]
train_1_1 = index_label_1[:int(len(index_label_1) * start)]
train_1_2 = index_label_1[int(len(index_label_1) * end):]
val_2 = index_label_2[int(len(index_label_2) * start):int(len(index_label_2) * end)]
train_2_1 = index_label_2[:int(len(index_label_2) * start)]
train_2_2 = index_label_2[int(len(index_label_2) * end):]
val_3 = index_label_3[int(len(index_label_3) * start):int(len(index_label_3) * end)]
train_3_1 = index_label_3[:int(len(index_label_3) * start)]
train_3_2 = index_label_3[int(len(index_label_3) * end):]
"""
train_indexes = train_1 + train_2 + train_3
val_indexes = val_0 + val_1 + val_2 + val_3
train_ids = [ids[i] for i in train_indexes]
val_ids = [ids[i] for i in val_indexes]
random.shuffle(train_ids)
random.shuffle(val_ids)