utils.py 5.8 KB
Newer Older
zpgeng's avatar
zpgeng committed
1
2
import matplotlib
matplotlib.use('Agg')
okiss's avatar
okiss committed
3
4
import pandas as pd
from config import config
5
import matplotlib.pyplot as plt
6
7
import seaborn as sns
import numpy as np
8
import torch
9
import pandas as pd
okiss's avatar
okiss committed
10

11
import os
okiss's avatar
okiss committed
12
13
14
15
from subprocess import call
import operator
import shutil

16
sns.set_style('darkgrid')
Ard Kastrati's avatar
Ard Kastrati committed
17
import logging
18

okiss's avatar
okiss committed
19
def plot_acc(hist, output_directory, model, val=False):
20
21
22
    '''
    plot the accuracy against the epochs during training
    '''
Ard Kastrati's avatar
Ard Kastrati committed
23
24
    epochs = len(hist.history['accuracy'])
    epochs = np.arange(epochs)
25
    plt.figure()
Ard Kastrati's avatar
Ard Kastrati committed
26
    plt.title(model + ' accuracy')
Ard Kastrati's avatar
Ard Kastrati committed
27
    plt.plot(epochs, hist.history['accuracy'],'b-',label='training')
28
    if val:
Ard Kastrati's avatar
Ard Kastrati committed
29
        plt.plot(epochs, hist.history['val_accuracy'],'g-',label='validation')
okiss's avatar
okiss committed
30

31
32
33
    plt.legend()
    plt.xlabel('epochs')
    plt.ylabel('Accuracy')
Ard Kastrati's avatar
Ard Kastrati committed
34
35
    plt.savefig(output_directory + '/' + model + '_accuracy.png')
    # plt.show()
Ard Kastrati's avatar
Ard Kastrati committed
36
37
    logging.info(10*'*'+'\n')

38

okiss's avatar
okiss committed
39
def plot_loss(hist, output_directory, model, val=False):
Ard Kastrati's avatar
Ard Kastrati committed
40
41
    epochs = len(hist.history['accuracy'])
    epochs = np.arange(epochs)
42
    plt.figure()
Ard Kastrati's avatar
Ard Kastrati committed
43
    plt.title(model + ' loss')
okiss's avatar
okiss committed
44
    plt.plot(epochs, hist.history['loss'], 'b-', label='training')
45
    if val:
okiss's avatar
okiss committed
46
47
48
        plt.plot(epochs, hist.history['val_loss'],'g-',label='validation')


49
50
    plt.legend()
    plt.xlabel('epochs')
okiss's avatar
okiss committed
51
    plt.ylabel('Binary Cross Entropy')
Ard Kastrati's avatar
Ard Kastrati committed
52
53
    plt.savefig(output_directory + '/' + model + '_loss.png')
    # plt.show()
54

Ard Kastrati's avatar
Ard Kastrati committed
55

Ard Kastrati's avatar
Ard Kastrati committed
56
def plot_loss_torch(loss, output_directory, model):
okiss's avatar
okiss committed
57
58
    epochs=np.arange(len(loss))
    plt.figure()
Ard Kastrati's avatar
Ard Kastrati committed
59
60
    plt.title(model + ' loss')
    plt.plot(epochs, loss, 'b-', label='training')
okiss's avatar
okiss committed
61
62
63
    plt.legend()
    plt.xlabel('epochs')
    plt.ylabel('Binary Cross Entropy')
Ard Kastrati's avatar
Ard Kastrati committed
64
65
    plt.savefig(output_directory + '/' + model + 'loss.png')
    # plt.show()
okiss's avatar
okiss committed
66
67


okiss's avatar
okiss committed
68
69
70
def cp_dir(source, target):
    call(['cp', '-a', source, target])

okiss's avatar
okiss committed
71
def comparison_plot():
okiss's avatar
okiss committed
72

okiss's avatar
okiss committed
73
    run_dir = os.getcwd()+'/results/'
okiss's avatar
okiss committed
74
    plt.figure()
okiss's avatar
okiss committed
75
    plt.title('Comparison of the Validation accuracy' )
okiss's avatar
okiss committed
76
77
78
    plt.grid(True)
    plt.xlabel('epochs')
    plt.ylabel('accuracy (%)')
okiss's avatar
okiss committed
79
80
81
82
83
84
85
86
87
88
89
90
    
    for experiment in os.listdir(run_dir):
        position = experiment.find('ensemble')
        if position != -1:

            name_split = experiment.split('_')
            name=name_split[1]
            for i in range(2,len(name_split)):
                name += '_'
                name += name_split[i]

            summary = pd.read_csv(run_dir+experiment+'/'+name+'_history.csv')
okiss's avatar
okiss committed
91
            acc = 100 * summary['val_accuracy']
okiss's avatar
okiss committed
92
            plt.plot(acc,'-',label=name)
okiss's avatar
okiss committed
93
94

    plt.legend()
okiss's avatar
okiss committed
95
    plt.savefig(run_dir+'/comparison_accuracy.png')
okiss's avatar
okiss committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131


def select_best_model():
    results = {}
    model = {}
    run_dir = config['log_dir']
    #get best model in runs for all model_name
    for experiment in os.listdir(run_dir):
        number,name = experiment.split('_',1)
        if os.path.isdir(run_dir+experiment):
            try:
                summary = pd.read_csv(run_dir+experiment+'/'+name+'_df_best_model.csv')
                acc = float(summary['val_accuracy'])
                if not (name in results.keys()):

                    results[name] = acc
                    model[name] = number
                else:
                    if acc > results[name]:
                        results[name] = acc
                        model[name] = number
            except FileNotFoundError:
                pass

    # update the best model in th results folder with the ones in runs
    for name in model.keys():
        if os.path.exists(os.getcwd()+'/results/'+'best_'+name) and os.path.isdir(os.getcwd()+'/results/'+'best_'+name):
            acc = float(pd.read_csv(os.getcwd()+'/results/'+'best_'+name+'/'+name+'_df_best_model.csv')['val_accuracy'])
            if acc < results[name]:
                shutil.rmtree(os.getcwd()+'/results/'+'best_'+name)
                cp_dir(run_dir+model[name]+'_'+name,os.getcwd()+'/results/')
                os.rename(os.getcwd()+'/results/'+model[name]+'_'+name, os.getcwd()+'/results/'+'best_'+name)
            else:
                pass


okiss's avatar
okiss committed
132
# Save the logs
133
def save_logs(hist, output_directory, model, pytorch=False):
Ard Kastrati's avatar
Ard Kastrati committed
134
    # os.mkdir(output_directory)
135
    if pytorch:
zpgeng's avatar
zpgeng committed
136
137
138
139
        try:
            hist_df = pd.DataFrame(hist)
            hist_df.to_csv(output_directory + '/' + model + '_' + 'history.csv', index=False)
        except:
Ard Kastrati's avatar
Ard Kastrati committed
140
            return
141
142

    else:
zpgeng's avatar
zpgeng committed
143
144
145
        try:
            hist_df = pd.DataFrame(hist.history)
            hist_df.to_csv(output_directory + '/' + model + '_' + 'history.csv', index=False)
okiss's avatar
okiss committed
146

zpgeng's avatar
zpgeng committed
147
148
149
            #df_metrics = {'Accuracy': hist_df['accuracy'], 'Loss': hist_df['loss']}
            #df_metrics = pd.DataFrame(df_metrics)
            #df_metrics.to_csv(output_directory + '/' + model + '_' + 'df_metrics.csv', index=False)
150

zpgeng's avatar
zpgeng committed
151
            index_best_model = hist_df['val_accuracy'].idxmax()
zpgeng's avatar
zpgeng committed
152
            row_best_model = hist_df.loc[index_best_model]
153

zpgeng's avatar
zpgeng committed
154
155
            df_best_model = pd.DataFrame(data=np.zeros((1, 4), dtype=np.float), index=[0],
                    columns=['best_model_train_loss', 'best_model_val_loss', 'best_model_train_acc', 'best_model_val_acc'])
okiss's avatar
okiss committed
156

zpgeng's avatar
zpgeng committed
157
158
            df_best_model['best_model_train_loss'] = row_best_model['loss']
            df_best_model['best_model_val_loss'] = row_best_model['val_loss']
zpgeng's avatar
zpgeng committed
159
160
            df_best_model['best_model_train_acc'] = row_best_model['accuracy']
            df_best_model['best_model_val_acc'] = row_best_model['val_accuracy']
okiss's avatar
okiss committed
161

zpgeng's avatar
zpgeng committed
162
163
164
            df_best_model.to_csv(output_directory + '/' + model + '_' + 'df_best_model.csv', index=False)
        except:
            return
165

Ard Kastrati's avatar
Ard Kastrati committed
166

167
# Save the model parameters (newly added without debugging)
168
169
170
171
172
173
174
# def save_model_param(classifier, output_directory, model, pytorch=False):
#     try:
#         if pytorch:
#             torch.save(classifier.state_dict(), output_directory + '/' + model + '_' + 'model.pth')
#         else:
#             classifier.save(output_directory + '/' + model + '_' + 'model.h5')
#     except:
okiss's avatar
okiss committed
175
#         return