Reworked plot functions for ML part.

Added plots for training accuracy in ML part.
Paths should now be handled in platform independent way.
General bugfixes.

slopestabilityML/GBC/gbc_run.py

@@ -22,10 +22,11 @@ def gbc_run(test_results, random_seed):
     clf = ensemble.GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)
 
     # Train classifier
-    accuracy_labels, accuracy_score = slopestabilityML.run_classification(test_training, test_prediction, test_results,
-                                                                          clf, 'GBC')
+    accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
+        slopestabilityML.run_classification(test_training, test_prediction, test_results, clf, 'GBC')
 
     # Plot
-    slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'GBC')
+    slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'GBC_prediction')
+    slopestabilityML.plot_results(accuracy_labels_training, accuracy_score_training, 'GBC_training')
 
-    return accuracy_score, accuracy_labels
+    return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training

slopestabilityML/SGD/sgd_run.py

@@ -21,10 +21,11 @@ def sgd_run(test_results, random_seed):
     clf = SGDClassifier(loss="hinge", penalty="l2", max_iter=5)
 
     # Train classifier
-    accuracy_labels, accuracy_score = slopestabilityML.run_classification(test_training, test_prediction, test_results,
-                                                                          clf, 'SGD')
+    accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
+        slopestabilityML.run_classification(test_training, test_prediction, test_results, clf, 'SGD ')
 
     # Plot
-    slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'SGD')
+    slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'SGD_prediction')
+    slopestabilityML.plot_results(accuracy_labels_training, accuracy_score_training, 'SGD_training')
 
-    return accuracy_score, accuracy_labels
+    return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training

slopestabilityML/SVM/svm_run.py

@@ -28,30 +28,12 @@ def svm_run(test_results, random_seed):
     clf = svm.SVC(gamma=0.001, C=100, kernel='linear')
 
     # Train classifier
-    accuracy_labels, accuracy_score = slopestabilityML.run_classification(test_training, test_prediction, test_results,
-                                                                          clf, 'SVM')
-
-    # for test_name in test_training:
-    #
-    #     # Prepare data
-    #     x_train, y_train = slopestabilityML.preprocess_data(test_results[test_name])
-    #
-    #     # Train classifier
-    #     clf.fit(x_train, y_train)
-    #
-    # # Predict with classfier
-    # for test_name_pred in test_prediction:
-    #
-    #     # Prepare data
-    #     x_question, y_answer = slopestabilityML.preprocess_data(test_results[test_name_pred])
-    #
-    #     y_pred = clf.predict(x_question)
-    #
-    #     # Evaluate result
-    #     accuracy_score.append(len(np.where(y_pred == y_answer)) / len(y_answer) * 100)
-    #     accuracy_labels.append(test_name_pred)
+    accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
+        slopestabilityML.run_classification(test_training, test_prediction, test_results, clf, 'SVM')
+
 
     # Plot
-    slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'SVM')
+    slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'SVM_prediction')
+    slopestabilityML.plot_results(accuracy_labels_training, accuracy_score_training, 'SVM_training')
 
-    return accuracy_score, accuracy_labels
+    return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training

slopestabilityML/combine_results.py

@@ -7,19 +7,44 @@ Created on 19.01.2021
 """
 
 import matplotlib.pyplot as plt
+from pathlib import Path
 
 
 def combine_results(ml_results):
 
+    # TODO avoid reusing the same code twice
+    # Predictions
     fig = plt.figure()
-    #ax = fig.subplots(1)
-    fig.suptitle('Accuracy of different ML methods')
+    ax = fig.subplots(1)
+    fig.suptitle('Accuracy of different ML methods: predictions')
 
-    for method_name in ml_results.keys():
-        plt.scatter(ml_results[method_name]['labels'], ml_results[method_name]['score'], label=method_name)
+    for method_name in sorted(ml_results.keys()):
+        plt.plot(ml_results[method_name]['labels'], ml_results[method_name]['score'], marker='x',
+                 label=method_name)
 
-    plt.legend(loc='best')
+    plt.xlabel('Test name')
+    plt.setp(ax.get_xticklabels(), rotation=45)
+    plt.ylabel('Correct points [%]')
+    plt.legend(loc='lower right')
 
-    fig.savefig('results/figures/ML_summary.eps')
-    fig.savefig('results/figures/ML_summary.png')
-    fig.savefig('results/figures/ML_summary.pdf')
\ No newline at end of file
+    fig.savefig(Path('results/figures/ML_summary_prediction.eps'))
+    fig.savefig(Path('results/figures/ML_summary_prediction.png'))
+    fig.savefig(Path('results/figures/ML_summary_prediction.pdf'))
+
+    # Training
+    fig = plt.figure()
+    ax = fig.subplots(1)
+    fig.suptitle('Accuracy of different ML methods - training')
+
+    for method_name in sorted(ml_results.keys()):
+        plt.plot(ml_results[method_name]['labels_training'], ml_results[method_name]['score_training'], marker='x',
+                 label=method_name)
+
+    plt.xlabel('Test name')
+    plt.setp(ax.get_xticklabels(), rotation=45)
+    plt.ylabel('Correct points [%]')
+    plt.legend(loc='lower right')
+
+    fig.savefig(Path('results/figures/ML_summary_training.eps'))
+    fig.savefig(Path('results/figures/ML_summary_training.png'))
+    fig.savefig(Path('results/figures/ML_summary_training.pdf'))

slopestabilityML/plot_results.py

@@ -7,19 +7,23 @@ Created on 17.01.2021
 """
 
 import matplotlib.pyplot as plt
+from pathlib import Path
 
 
 def plot_results(accuracy_labels, accuracy_score, clf_name):
 
+    clf_name_title = clf_name.replace("_", " ")
+
     fig = plt.figure()
     ax = plt.subplot(111)
-    ax.scatter(accuracy_labels, accuracy_score)
+    ax.plot(accuracy_labels, accuracy_score, marker='x')
+    plt.setp(ax.get_xticklabels(), rotation=45)
     plt.xlabel('Test name')
     plt.ylabel('Correct points [%]')
-    plt.title(clf_name+' classification accuracy')
+    plt.title(clf_name_title+' accuracy score')
     print('plot script is executed')
-    fig.savefig('results/figures/ML/'+clf_name+'.eps')
-    fig.savefig('results/figures/ML/'+clf_name+'.pdf')
-    fig.savefig('results/figures/ML/'+clf_name+'.png')
+    fig.savefig(Path('results/figures/ML/'+clf_name+'.eps'))
+    fig.savefig(Path('results/figures/ML/'+clf_name+'.pdf'))
+    fig.savefig(Path('results/figures/ML/'+clf_name+'.png'))
 
     return

slopestabilityML/preprocess_data.py

@@ -12,7 +12,7 @@ import pandas as pd
 def preprocess_data(data_set):
 
     x_train = data_set.drop(['X', 'Y', 'Z', 'INM', 'CLASS'], axis='columns')
-    print(x_train)
+    #print(x_train)
     #y_train = data_set.drop(['X', 'Y', 'Z', 'INM'], axis='columns')
     y_train = pd.DataFrame(data_set['CLASS'])
 

slopestabilityML/run_all_tests.py

@@ -7,27 +7,34 @@ Created on 19.01.2021
 """
 
 import slopestabilityML
-#import slopestabilityML.SVM.svm_run
-#import slopestabilityML.GBC.gbc_run
 
 
-def run_all_tests(test_results):
+# import slopestabilityML.SVM.svm_run
+# import slopestabilityML.GBC.gbc_run
+
 
+def run_all_tests(test_results):
     random_seed = 999
 
     ml_results = {}
 
     print('Running SVM...')
-    svm_accuracy_score, svm_accuracy_labels = slopestabilityML.SVM.svm_run(test_results, random_seed)
-    ml_results['svm'] = {'score': svm_accuracy_score, 'labels': svm_accuracy_labels}
+    svm_accuracy_score, svm_accuracy_labels, svm_accuracy_score_training, svm_accuracy_labels_training = \
+        slopestabilityML.SVM.svm_run(test_results, random_seed)
+    ml_results['svm'] = {'score': svm_accuracy_score, 'labels': svm_accuracy_labels,
+                         'score_training': svm_accuracy_score_training, 'labels_training': svm_accuracy_labels_training}
 
     print('Running GBC...')
-    gbc_accuracy_score, gbc_accuracy_labels = slopestabilityML.GBC.gbc_run(test_results, random_seed)
-    ml_results['gbc'] = {'score': gbc_accuracy_score, 'labels': gbc_accuracy_labels}
+    gbc_accuracy_score, gbc_accuracy_labels, gbc_accuracy_score_training, gbc_accuracy_labels_training = \
+        slopestabilityML.GBC.gbc_run(test_results, random_seed)
+    ml_results['gbc'] = {'score': gbc_accuracy_score, 'labels': gbc_accuracy_labels,
+                         'score_training': gbc_accuracy_score_training, 'labels_training': gbc_accuracy_labels_training}
 
     print('Running SGD...')
-    sgd_accuracy_score, sgd_accuracy_labels = slopestabilityML.SGD.sgd_run(test_results, random_seed)
-    ml_results['sgd'] = {'score': sgd_accuracy_score, 'labels': sgd_accuracy_labels}
+    sgd_accuracy_score, sgd_accuracy_labels, sgd_accuracy_score_training, sgd_accuracy_labels_training = \
+        slopestabilityML.SGD.sgd_run(test_results, random_seed)
+    ml_results['sgd'] = {'score': sgd_accuracy_score, 'labels': sgd_accuracy_labels,
+                         'score_training': sgd_accuracy_score_training, 'labels_training': sgd_accuracy_labels_training}
 
     slopestabilityML.combine_results(ml_results)
     print('end')

slopestabilityML/run_classification.py

@@ -16,6 +16,7 @@ import matplotlib.pyplot as plt
 from matplotlib import ticker
 from scipy import interpolate
 import slopestabilitytools
+from pathlib import Path
 
 
 def run_classification(test_training, test_prediction, test_results, clf, clf_name):
@@ -23,6 +24,9 @@ def run_classification(test_training, test_prediction, test_results, clf, clf_na
     accuracy_score = []
     accuracy_labels = []
 
+    accuracy_score_training = []
+    accuracy_labels_training = []
+
     num_feat = ['RES', 'SEN']
     #cat_feat = ['CLASS']
 
@@ -44,6 +48,10 @@ def run_classification(test_training, test_prediction, test_results, clf, clf_na
         # print(type(x_train))
         # print(type(y_train))
         clf_pipeline_UM.fit(x_train, y_train)
+        score_training = clf_pipeline_UM.score(x_train, y_train)
+
+        accuracy_score_training.append(score_training * 100)
+        accuracy_labels_training.append(test_name)
 
     # Predict with classifier
     for test_name_pred in test_prediction:
@@ -110,14 +118,14 @@ def run_classification(test_training, test_prediction, test_results, clf, clf_na
         cb[2].locator = tick_locator
         cb[2].update_ticks()
 
-        fig.savefig('results/figures/ML/eps/{}_ML_{}_class_res.eps'.format(test_name_pred, clf_name))
-        fig.savefig('results/figures/ML/png/{}_ML_{}_class_res.png'.format(test_name_pred, clf_name))
-        fig.savefig('results/figures/ML/pdf/{}_ML_{}_class_res.pdf'.format(test_name_pred, clf_name))
+        fig.savefig(Path('results/figures/ML/prediction/eps/{}_ML_{}_class_res.eps'.format(test_name_pred, clf_name)))
+        fig.savefig(Path('results/figures/ML/prediction/png/{}_ML_{}_class_res.png'.format(test_name_pred, clf_name)))
+        fig.savefig(Path('results/figures/ML/prediction/pdf/{}_ML_{}_class_res.pdf'.format(test_name_pred, clf_name)))
 
         # Evaluate result
         #accuracy_score.append(len(np.where(y_pred == y_answer.to_numpy())) / len(y_answer.to_numpy()) * 100)
-        accuracy_score.append(score)
+        accuracy_score.append(score*100)
         accuracy_labels.append(test_name_pred)
 
-    return accuracy_labels, accuracy_score
+    return accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training
 

slopestabilityML/split_dataset.py

@@ -21,4 +21,4 @@ def split_dataset(test_names, random_seed):
 
     test_training = slopestabilitytools.set_diff(list(test_names), set(test_prediction))
 
-    return test_training, test_prediction
+    return sorted(test_training), sorted(test_prediction)

slopestabilitytools/folder_structure/check_create_folder.py

@@ -14,7 +14,7 @@ def check_create_folder(folder_path):
     if os.path.isdir(folder_path):
         print("Folder for figures exists!")
     else:
-        os.mkdir(folder_path)
+        os.makedirs(folder_path)
         print("Created folder for figures!")
 
     return worked

slopestabilitytools/folder_structure/create_folder_for_test.py

@@ -7,12 +7,13 @@ Created on 15.01.2021
 """
 
 import os
+from pathlib import Path
 from .check_create_folder import check_create_folder
 
 
 def create_folder_for_test(test_name):
 
-    folder_path = os.getcwd() + '/results/results/%'.format(test_name)
+    folder_path = Path(os.getcwd() / '/results/results/%'.format(test_name))
     is_success = check_create_folder(folder_path)
 
     return is_success

slopestabilitytools/folder_structure/create_folder_structure.py

@@ -7,6 +7,7 @@ Created on 15.01.2021
 """
 
 import os
+from pathlib import Path
 from .check_create_folder import check_create_folder
 
 
@@ -15,27 +16,41 @@ def create_folder_structure():
     is_success = True
 
     # Folder for figures
-    folder_path = os.getcwd()+'/results/'
+    folder_path = Path(os.getcwd()+'/results/')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/figures/'
+    folder_path = Path(os.getcwd() + '/results/figures/')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/figures/pdf'
+    folder_path = Path(os.getcwd() + '/results/figures/pdf')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/figures/eps'
+    folder_path = Path(os.getcwd() + '/results/figures/eps')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/figures/png'
+    folder_path = Path(os.getcwd() + '/results/figures/png')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/figures/ML/pdf'
+    folder_path = Path(os.getcwd() + '/results/figures/ML/pdf')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/figures/ML/eps'
+    folder_path = Path(os.getcwd() + '/results/figures/ML/eps')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/figures/ML/png'
+    folder_path = Path(os.getcwd() + '/results/figures/ML/png')
+    is_success = check_create_folder(folder_path)
+    folder_path = Path(os.getcwd() + '/results/figures/ML/training/')
+    is_success = check_create_folder(folder_path)
+    folder_path = Path(os.getcwd() + '/results/figures/ML/training/pdf')
+    is_success = check_create_folder(folder_path)
+    folder_path = Path(os.getcwd() + '/results/figures/ML/training/eps')
+    is_success = check_create_folder(folder_path)
+    folder_path = Path(os.getcwd() + '/results/figures/ML/training/png')
+    is_success = check_create_folder(folder_path)
+    folder_path = Path(os.getcwd() + '/results/figures/ML/prediction/pdf')
+    is_success = check_create_folder(folder_path)
+    folder_path = Path(os.getcwd() + '/results/figures/ML/prediction/eps')
+    is_success = check_create_folder(folder_path)
+    folder_path = Path(os.getcwd() + '/results/figures/ML/prediction/png')
     is_success = check_create_folder(folder_path)
 
     # Folder for results
-    folder_path = os.getcwd()+'/results/'
+    folder_path = Path(os.getcwd()+'/results/')
     is_success = check_create_folder(folder_path)
-    folder_path = os.getcwd() + '/results/results/'
+    folder_path = Path(os.getcwd() + '/results/results/')
     is_success = check_create_folder(folder_path)
 
     return is_success

slopestabilitytools/plot_and_save.py

@@ -12,6 +12,7 @@ from matplotlib import ticker
 from scipy import interpolate
 import numpy as np
 import slopestabilitytools
+from pathlib import Path
 
 
 def plot_and_save(test_name, test_result, plot_title):
@@ -75,9 +76,9 @@ def plot_and_save(test_name, test_result, plot_title):
     cb[2].locator = tick_locator
     cb[2].update_ticks()
 
-    fig.savefig('results/figures/eps/{}_in_inv_diff.eps'.format(test_name))
-    fig.savefig('results/figures/png/{}_in_inv_diff.png'.format(test_name))
-    fig.savefig('results/figures/pdf/{}_in_inv_diff.pdf'.format(test_name))
+    fig.savefig(Path('results/figures/eps/{}_in_inv_diff.eps'.format(test_name)))
+    fig.savefig(Path('results/figures/png/{}_in_inv_diff.png'.format(test_name)))
+    fig.savefig(Path('results/figures/pdf/{}_in_inv_diff.pdf'.format(test_name)))
 
     # Plot coverage
     cb_cov = []
@@ -93,8 +94,8 @@ def plot_and_save(test_name, test_result, plot_title):
     cb_cov.locator = tick_locator
     cb_cov.update_ticks()
 
-    fig_cov.savefig('results/figures/eps/{}_cov.eps'.format(test_name))
-    fig_cov.savefig('results/figures/png/{}_cov.png'.format(test_name))
-    fig_cov.savefig('results/figures/pdf/{}_cov.pdf'.format(test_name))
+    fig_cov.savefig(Path('results/figures/eps/{}_cov.eps'.format(test_name)))
+    fig_cov.savefig(Path('results/figures/png/{}_cov.png'.format(test_name)))
+    fig_cov.savefig(Path('results/figures/pdf/{}_cov.pdf'.format(test_name)))
 
     return