Added "council of precogs".

slopestabilityML/ADABOOST/adaboost_run.py

@@ -21,11 +21,11 @@ def adaboost_run(test_results, random_seed):
                              n_estimators=20, random_state=0)
 
     # Train classifier
-    accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
+    result_class, accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
         slopestabilityML.run_classification(test_training, test_prediction, test_results, clf, 'ADA')
 
     # Plot
     slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'ADA_prediction')
     slopestabilityML.plot_results(accuracy_labels_training, accuracy_score_training, 'ADA_training')
 
-    return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training
+    return result_class, accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training

slopestabilityML/GBC/gbc_run.py

-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on 19.01.2021
-
-@author: Feliks Kiszkurno
-"""
-
-from sklearn import ensemble
-import slopestabilityML.plot_results
-import slopestabilityML.split_dataset
-import slopestabilityML.run_classification
-
-# TODO: as in svm_run
-
-
-def gbc_run(test_results, random_seed):
-
-    test_training, test_prediction = slopestabilityML.split_dataset(test_results.keys(), random_seed)
-
-    # Create classifier
-    clf = ensemble.GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)
-
-    # Train classifier
-    accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
-        slopestabilityML.run_classification(test_training, test_prediction, test_results, clf, 'GBC')
-
-    # Plot
-    slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'GBC_prediction')
-    slopestabilityML.plot_results(accuracy_labels_training, accuracy_score_training, 'GBC_training')
-
-    return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training

slopestabilityML/KNN/__init__.py

-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on 
-
-@author: 
-"""
-
-# https://letsfigureout.com/2020/03/08/nearest-neighbor-algorithm-with-python-and-numpy/

slopestabilityML/KNN/knn_run.py

@@ -20,11 +20,11 @@ def knn_run(test_results, random_seed):
     clf = KNeighborsClassifier(n_neighbors=2)
 
     # Train classifier
-    accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
+    result_class, accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
         slopestabilityML.run_classification(test_training, test_prediction, test_results, clf, 'KNN')
 
     # Plot
     slopestabilityML.plot_results(accuracy_labels, accuracy_score, 'KNN_prediction')
     slopestabilityML.plot_results(accuracy_labels_training, accuracy_score_training, 'KNN_training')
 
-    return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training
\ No newline at end of file
+    return result_class, accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training
\ No newline at end of file

slopestabilityML/MGC/mgc_run.py

@@ -25,31 +25,34 @@ def mgc_run(test_results, random_seed):
     clf = 0
 
     # Train classifier
+    result_class = {}
     for test_name in test_prediction:
         classes = slopestabilityML.MGC.max_grad_classi(test_results[test_name])
         ids_temp = np.argwhere(classes >= 0.5)
         classes[ids_temp] = 1
         ids_temp = np.argwhere(classes < 0.5)
         classes[ids_temp] = 0
+        result_class[test_name] = classes
         classes_correct = test_results[test_name]['CLASS'].to_numpy()
         score_prediction = len(np.argwhere(classes == classes_correct)) / len(classes_correct)
         accuracy_score_prediction.append(score_prediction * 100)
         accuracy_labels_prediction.append(test_name)
         slopestabilityML.plot_class_res(test_results, test_name, classes_correct, classes, 'MGC_prediction')
 
-    for test_name in test_training:
-        classes = slopestabilityML.MGC.max_grad_classi(test_results[test_name])
-        ids_temp = np.argwhere(classes >= 0.5)
-        classes[ids_temp] = 1
-        ids_temp = np.argwhere(classes < 0.5)
-        classes[ids_temp] = 0
-        classes_correct = test_results[test_name]['CLASS'].to_numpy()
-        score_prediction = len(np.argwhere(classes == classes_correct)) / len(classes_correct)
-        accuracy_score_training.append(score_prediction * 100)
-        accuracy_labels_training.append(test_name)
-        slopestabilityML.plot_class_res(test_results, test_name, classes_correct, classes, 'MGC_training')
+
+    # for test_name in test_training:
+    #     classes = slopestabilityML.MGC.max_grad_classi(test_results[test_name])
+    #     ids_temp = np.argwhere(classes >= 0.5)
+    #     classes[ids_temp] = 1
+    #     ids_temp = np.argwhere(classes < 0.5)
+    #     classes[ids_temp] = 0
+    #     classes_correct = test_results[test_name]['CLASS'].to_numpy()
+    #     score_prediction = len(np.argwhere(classes == classes_correct)) / len(classes_correct)
+    #     accuracy_score_training.append(score_prediction * 100)
+    #     accuracy_labels_training.append(test_name)
+    #     slopestabilityML.plot_class_res(test_results, test_name, classes_correct, classes, 'MGC_training')
 
     # Plot
     slopestabilityML.plot_results(accuracy_labels_prediction, accuracy_score_prediction, 'MGC_training')
 
-    return accuracy_score_prediction, accuracy_labels_prediction, accuracy_score_training, accuracy_labels_training
+    return result_class, accuracy_score_prediction, accuracy_labels_prediction, accuracy_score_training, accuracy_labels_training

slopestabilityML/ask_committee.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on 04.04.2021
+
+@author: Feliks Kiszkurno
+"""
+
+import numpy as np
+
+import settings
+import slopestabilityML
+
+
+def ask_committee(ml_result_class, test_results, *, random_seed=False):
+    
+    results = np.array([])
+    classes_correct = {}
+    for test_name in sorted(test_results.keys()):
+        if settings.settings['norm_class'] is True:
+            class_in = test_results[test_name]['CLASSN']
+        elif settings.settings['norm_class'] is False:
+            class_in = test_results[test_name]['CLASS']
+        else:
+            print('I don\'t know which class to use! Exiting...')
+            exit(0)
+        classes_correct[test_name] = class_in
+        
+    # First create list of datasets and verify if all classifiers has been run over the same sets
+    test_names_all = []
+    for method_name in sorted(ml_result_class.keys()):
+        test_names_all_temp = []
+        for test_name in sorted(ml_result_class[method_name].keys()):
+            test_names_all_temp.append(test_name)
+        test_names_all = set(test_names_all_temp)
+        #if sorted(test_names_all) == sorted(test_names_all_temp):
+        #    test_names_all = sorted(test_names_all_temp)
+        #else:
+        #    print("Mismatch in datasets!")
+        #    exit(0)
+
+    # Combine results for each data set from each classifier into an array
+    results_test = {}
+    for test_name in test_names_all:
+        results = np.zeros([len(ml_result_class[list(ml_result_class.keys())[0]][test_name]), len( sorted(ml_result_class.keys()))])
+        method_id = 0
+        for method_name in sorted(ml_result_class.keys()):
+            results[:, method_id] = ml_result_class[method_name][test_name].T.reshape([len(ml_result_class[method_name][test_name])])
+            method_id = method_id + 1
+        results_test[test_name] = results
+
+    # Execute the voting
+    results_voting = {}
+    for test_name in sorted(results_test.keys()):
+        test_result = results_test[test_name]
+        shape_test = test_result.shape
+        result_rows = shape_test[0]
+        result_cols = shape_test[1]
+        result_voted = np.zeros([result_rows, 1])
+        for row_id in range(result_rows):
+            row_temp = test_result[row_id, :]
+            result_voted[row_id] = np.bincount(row_temp.astype(int)).argmax()
+        results_voting[test_name] = result_voted
+
+    # Compare voted result with correct result
+    accuracy_score = []
+    accuracy_labels = []
+    accuracy_score_training = []
+    accuracy_labels_training = []
+
+    # if random_seed is not False:
+    #     test_training, test_prediction = slopestabilityML.split_dataset(test_results.keys(), random_seed)
+    #     tests_ordered = {'train': test_training, 'pred': test_prediction}
+    # else:
+    test_training = []
+    test_prediction = results_voting.keys()
+    tests_ordered = {'train': test_training, 'pred': test_prediction}
+
+    for test_group in sorted(tests_ordered.keys()):
+        for test_name in tests_ordered[test_group]:
+            classes_correct = class_in.to_numpy()
+            classes_correct = classes_correct.reshape([len(results_voting[test_name]), 1])
+            score = len(np.argwhere(results_voting[test_name] == classes_correct)) / len(classes_correct)
+            if test_group is 'train':
+                accuracy_score_training.append(score * 100)
+                accuracy_labels_training.append(test_name)
+            elif test_group is 'pred':
+                accuracy_score.append(score * 100)
+                accuracy_labels.append(test_name)
+
+    return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training

slopestabilityML/run_classi.py

@@ -80,6 +80,8 @@ def run_classification(test_training, test_prediction, test_results, clf, clf_na
         accuracy_score_training.append(score_training * 100)
         accuracy_labels_training.append(test_name)
 
+    result_class = {}
+
     # Predict with classifier
     for test_name_pred in test_prediction:
         # Prepare data
@@ -87,6 +89,7 @@ def run_classification(test_training, test_prediction, test_results, clf, clf_na
 
         # y_pred = clf_pipeline.score(x_question, y_answer)
         y_pred = clf_pipeline.predict(x_question)
+        result_class[test_name_pred] = y_pred
         # print(y_pred)
         score = clf_pipeline.score(x_question, y_answer)
         print('score: '+str(score))
@@ -107,5 +110,5 @@ def run_classification(test_training, test_prediction, test_results, clf, clf_na
         accuracy_score.append(score*100)
         accuracy_labels.append(test_name_pred)
 
-    return accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training
+    return result_class, accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training
 

slopestabilityML/run_every.py

@@ -19,36 +19,42 @@ def run_all_tests(test_results):
     random_seed = 999
 
     ml_results = {}
+    ml_results_class = {}
 
     print('Running SVM...')
-    svm_accuracy_score, svm_accuracy_labels, svm_accuracy_score_training, svm_accuracy_labels_training = \
+    svm_result_class, svm_accuracy_score, svm_accuracy_labels, svm_accuracy_score_training, svm_accuracy_labels_training = \
         slopestabilityML.SVM.svm_run(test_results, random_seed)
     ml_results['svm'] = {'score': svm_accuracy_score, 'labels': svm_accuracy_labels,
                          'score_training': svm_accuracy_score_training, 'labels_training': svm_accuracy_labels_training}
+    ml_results_class['svm'] = svm_result_class
 
     print('Running GBC...')
-    gbc_accuracy_score, gbc_accuracy_labels, gbc_accuracy_score_training, gbc_accuracy_labels_training = \
+    gbc_result_class, gbc_accuracy_score, gbc_accuracy_labels, gbc_accuracy_score_training, gbc_accuracy_labels_training = \
         slopestabilityML.GBC.gbc_run(test_results, random_seed)
     ml_results['gbc'] = {'score': gbc_accuracy_score, 'labels': gbc_accuracy_labels,
                          'score_training': gbc_accuracy_score_training, 'labels_training': gbc_accuracy_labels_training}
+    ml_results_class['gbc'] = gbc_result_class
 
     print('Running SGD...')
-    sgd_accuracy_score, sgd_accuracy_labels, sgd_accuracy_score_training, sgd_accuracy_labels_training = \
+    sgd_result_class, sgd_accuracy_score, sgd_accuracy_labels, sgd_accuracy_score_training, sgd_accuracy_labels_training = \
         slopestabilityML.SGD.sgd_run(test_results, random_seed)
     ml_results['sgd'] = {'score': sgd_accuracy_score, 'labels': sgd_accuracy_labels,
                          'score_training': sgd_accuracy_score_training, 'labels_training': sgd_accuracy_labels_training}
+    ml_results_class['sgd'] = sgd_result_class
 
     print('Running KNN...')
-    knn_accuracy_score, knn_accuracy_labels, knn_accuracy_score_training, knn_accuracy_labels_training = \
+    knn_result_class, knn_accuracy_score, knn_accuracy_labels, knn_accuracy_score_training, knn_accuracy_labels_training = \
         slopestabilityML.KNN.knn_run(test_results, random_seed)
     ml_results['KNN'] = {'score': knn_accuracy_score, 'labels': knn_accuracy_labels,
                          'score_training': knn_accuracy_score_training, 'labels_training': knn_accuracy_labels_training}
+    ml_results_class['knn'] = knn_result_class
 
     print('Running ADABOOST...')
-    ada_accuracy_score, ada_accuracy_labels, ada_accuracy_score_training, ada_accuracy_labels_training = \
+    ada_result_class, ada_accuracy_score, ada_accuracy_labels, ada_accuracy_score_training, ada_accuracy_labels_training = \
         slopestabilityML.ADABOOST.adaboost_run(test_results, random_seed)
     ml_results['ADA'] = {'score': ada_accuracy_score, 'labels': ada_accuracy_labels,
                          'score_training': ada_accuracy_score_training, 'labels_training': ada_accuracy_labels_training}
+    ml_results_class['ada'] = ada_result_class
 
     # print('Running RVM...')
     # rvm_accuracy_score, rvm_accuracy_labels, rvm_accuracy_score_training, rvm_accuracy_labels_training = \
@@ -57,10 +63,17 @@ def run_all_tests(test_results):
     #                      'score_training': rvm_accuracy_score_training, 'labels_training': rvm_accuracy_labels_training}
 
     print('Running MGC')
-    mgc_accuracy_score, mgc_accuracy_labels, mgc_accuracy_score_training, mgc_accuracy_labels_training \
+    mgc_result_class, mgc_accuracy_score, mgc_accuracy_labels, mgc_accuracy_score_training, mgc_accuracy_labels_training \
         = slopestabilityML.MGC.mgc_run(test_results, random_seed)
     ml_results['MGC'] = {'score': mgc_accuracy_score, 'labels': mgc_accuracy_labels,
                          'score_training': mgc_accuracy_score_training, 'labels_training': mgc_accuracy_labels_training}
+    ml_results_class['mgc'] = mgc_result_class
+
+    print('Asking committee for verdict')
+    committee_accuracy_score, committee_accuracy_labels, committee_accuracy_score_training, committee_accuracy_labels_training  \
+        = slopestabilityML.ask_committee(ml_results_class, test_results, random_seed=random_seed)
+    ml_results['com'] = {'score': committee_accuracy_score, 'labels': committee_accuracy_labels,
+                         'score_training': committee_accuracy_score_training, 'labels_training': committee_accuracy_labels_training}
 
     slopestabilityML.combine_results(ml_results)
     print('ML classification finished')