Commit aead35f8 authored by Feliks Kiszkurno's avatar Feliks Kiszkurno
Browse files

New way how folder structure is created.

All paths are now handled globally.
Added option to clip data.
Added option to reassign classes.
parent 16f9e927
......@@ -7,6 +7,7 @@ Created on Fri Jan 8 10:29:00 2021
"""
import slopestabilitytools.datamanagement
import slopestabilitytools
import slopestabilityML
import slostabcreatedata
import numpy as np
......@@ -15,13 +16,22 @@ import settings
settings.init()
# Config
create_new_data = True
create_new_data_only = True
create_new_data = False # set to True if you need to reassign the classes
create_new_data_only = False
reassign_classes = True; class_type = 'norm'
# Load existing data instead of creating new one.
if not create_new_data:
test_results = slopestabilitytools.datamanagement.import_tests()
if reassign_classes is True:
test_results = slopestabilitytools.reassign_classes(test_results, class_type)
# Check if folder structure for figures exists and create it if not
is_success = slopestabilitytools.folder_structure.create_folder_structure()
# Create new data
else:
# Prepare folder structure for output
......@@ -66,7 +76,7 @@ else:
'layers_pos': np.array([-4, -8])}
}
# tests_horizontal = {'hor_9': {'layer_n': 1, 'rho_values': [[1, 3], [2, 25]], 'layers_pos': np.array([-3])},}
# tests_horizontal = {'hor_11': {'layer_n': 1, 'rho_values': [[1, 10], [2, 12]], 'layers_pos': np.array([-4])}}
# Create models and invert them
test_results = {}
......@@ -82,11 +92,13 @@ else:
slopestabilitytools.plot_and_save(test_name, test_results[test_name], 'Test: ' + test_name, test_rho_max,
test_rho_min)
# Evaluate data with ML techniques
if not create_new_data_only:
print('Running ML stuff...')
ml_results = slopestabilityML.run_all_tests(test_results)
# Finish the script if ML classifiaction was not executed
elif create_new_data_only:
print('Done')
......@@ -16,13 +16,21 @@ def init():
# Normalization and classes
settings['norm_class'] = True # True to use normalized classes, False to use class_ids
settings['norm_class_num'] = 5 # Number of classes for normalized data
settings['norm'] = False # True to use normalized data, False to use raw data
settings['norm'] = True # True to use normalized data, False to use raw data
# Include sensitivity
settings['sen'] = False # True - include sensitivity, False - ignore sensitivity
# Include depth
settings['depth'] = True # True - include depth, False - ignore depth
# Clip data to max and min values from the input model
settings['clip'] = True # True - clip data, False - use unclipped data
# Paths
settings['results_folder'] = "results"
settings['results_folder'] = 'results'
settings['data_folder'] = settings['results_folder'] + '/data/'
settings['figures_folder'] = settings['results_folder'] + '/figures/'
# Plots
settings['plot_formats'] = ['png']
settings['plot_formats'] = ['png', 'pdf'] # list of formats to save plots as, supported formats: eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff
......@@ -11,12 +11,14 @@ import slopestabilityML.split_dataset
import slopestabilityML.run_classi
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
def adaboost_run(test_results, random_seed):
test_training, test_prediction = slopestabilityML.split_dataset(test_results.keys(), random_seed)
clf = AdaBoostClassifier(n_estimators=50, random_state=0)
clf = AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
n_estimators=20, random_state=0)
# Train classifier
accuracy_labels, accuracy_score, accuracy_labels_training, accuracy_score_training = \
......@@ -27,4 +29,3 @@ def adaboost_run(test_results, random_seed):
slopestabilityML.plot_results(accuracy_labels_training, accuracy_score_training, 'ADA_training')
return accuracy_score, accuracy_labels, accuracy_score_training, accuracy_labels_training
......@@ -9,6 +9,8 @@ Created on 17.01.2021
import matplotlib.pyplot as plt
from pathlib import Path
import slopestabilitytools
def plot_results(accuracy_labels, accuracy_score, clf_name):
......@@ -23,8 +25,9 @@ def plot_results(accuracy_labels, accuracy_score, clf_name):
plt.title(clf_name_title+' accuracy score')
print('plot script is executed')
fig.tight_layout()
slopestabilitytools.save_plot(fig, clf_name, '_accuracy', subfolder='ML')
# fig.savefig(Path('results/figures/ML/'+clf_name+'.eps'), bbox_inches="tight")
# fig.savefig(Path('results/figures/ML/'+clf_name+'.pdf'), bbox_inches="tight")
fig.savefig(Path('results/figures/ML/'+clf_name+'.png'), bbox_inches="tight")
# fig.savefig(Path('results/figures/ML/'+clf_name+'.png'), bbox_inches="tight")
return
......@@ -13,9 +13,9 @@ import pandas as pd
def preprocess_data(data_set):
if settings.settings['norm'] is True:
x_train = data_set.drop(['X', 'Y', 'Z', 'INM', 'RES', 'CLASS', 'CLASSN'], axis='columns')
x_train = data_set.drop(['X', 'Z', 'INM', 'INMN', 'RES', 'CLASS', 'CLASSN'], axis='columns')
else:
x_train = data_set.drop(['X', 'Y', 'Z', 'INM', 'RESN', 'CLASS', 'CLASSN'], axis='columns')
x_train = data_set.drop(['X', 'Z', 'INM', 'INMN', 'RESN', 'CLASS', 'CLASSN'], axis='columns')
if settings.settings['sen'] is False:
x_train = x_train.drop(['SEN'], axis='columns')
......@@ -25,4 +25,7 @@ def preprocess_data(data_set):
else:
y_train = pd.DataFrame(data_set['CLASS'])
if settings.settings['depth'] is False:
x_train = x_train.drop(['Y'], axis='columns')
return x_train, y_train
......@@ -19,6 +19,8 @@ from .set_diff import set_diff
from .normalize import normalize
from .assign_classes import assign_classes
from .def_classes import def_classes
from .save_plot import save_plot
from .reassign_classes import reassign_classes
from .folder_structure.create_folder_structure import create_folder_structure
from .folder_structure.create_folder_for_test import create_folder_for_test
......
......@@ -7,6 +7,8 @@ Created on 17.01.2021
"""
import slopestabilitytools.datamanagement.test_list
import settings
import pandas as pd
......@@ -17,7 +19,7 @@ def import_tests():
#print(test_names)
for test_name in test_names:
test_result_curr = pd.read_csv('results/results/' + test_name + '.csv', index_col=0)
test_result_curr = pd.read_csv(settings.settings['data_folder'] + '/' + test_name + '.csv', index_col=0)
test_results.update({test_name: test_result_curr})
return test_results
......@@ -8,12 +8,14 @@ Created on 17.01.2021
import os
import settings
# TODO Add more flexibility in the way, that the path is handled
def test_list(extension):
path = 'results/results'
path = settings.settings['data_folder']
file_list = os.listdir(path)
test_names = []
......
......@@ -8,7 +8,9 @@ Created on 15.01.2021
import os
from pathlib import Path
from .check_create_folder import check_create_folder
import settings
def create_folder_structure():
......@@ -16,41 +18,23 @@ def create_folder_structure():
is_success = True
# Folder for figures
folder_path = Path(os.getcwd()+'/results/')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/pdf')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/eps')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/png')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/pdf')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/eps')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/png')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/training/')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/training/pdf')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/training/eps')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/training/png')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/prediction/pdf')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/prediction/eps')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/figures/ML/prediction/png')
is_success = check_create_folder(folder_path)
for file_format in settings.settings['plot_formats']:
folder_path = Path(os.getcwd() + '/' + settings.settings['figures_folder'] + file_format)
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/' + settings.settings['figures_folder'] + 'ML/' + file_format)
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/' + settings.settings['figures_folder'] + 'ML/training/' + file_format)
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/' + settings.settings['figures_folder'] + 'ML/prediction/' + file_format)
is_success = check_create_folder(folder_path)
# Folder for results
folder_path = Path(os.getcwd()+'/results/')
is_success = check_create_folder(folder_path)
folder_path = Path(os.getcwd() + '/results/results/')
folder_path = Path(os.getcwd() + '/' + settings.settings['data_folder'])
is_success = check_create_folder(folder_path)
return is_success
......@@ -11,18 +11,21 @@ import numpy as np
def normalize(array):
array_max = 0
array_min = 0
array = np.array(array)
if array.ndim == 1:
array_min = np.min(array)
array_max = np.max(array)
elif array.ndim == 2:
array_min = np.amin(array)
array_max = np.amax(array)
else:
print('ERROR: incorrect dimensionality of the input array')
exit()
array_norm = array / array_max
array_norm = (array - array_min) / (array_max - array_min)
return array_norm
......@@ -77,7 +77,8 @@ def plot_and_save(test_name, test_result, plot_title, rho_max, rho_min):
cb[2].update_ticks()
fig.tight_layout()
fig.savefig(Path('results/figures/eps/{}_in_inv_diff.eps'.format(test_name)), bbox_inches="tight")
slopestabilitytools.save_plot(fig, test_name, '_in_inv_diff')
#fig.savefig(Path('results/figures/eps/{}_in_inv_diff.eps'.format(test_name)), bbox_inches="tight")
#fig.savefig(Path('results/figures/png/{}_in_inv_diff.png'.format(test_name)), bbox_inches="tight")
#fig.savefig(Path('results/figures/pdf/{}_in_inv_diff.pdf'.format(test_name)), bbox_inches="tight")
......@@ -96,7 +97,8 @@ def plot_and_save(test_name, test_result, plot_title, rho_max, rho_min):
cb_cov.update_ticks()
fig_cov.tight_layout()
fig_cov.savefig(Path('results/figures/eps/{}_cov.eps'.format(test_name)), bbox_inches="tight")
slopestabilitytools.save_plot(fig_cov, test_name, '_cov')
# fig_cov.savefig(Path('results/figures/eps/{}_cov.eps'.format(test_name)), bbox_inches="tight")
#fig_cov.savefig(Path('results/figures/png/{}_cov.png'.format(test_name)), bbox_inches="tight")
#fig_cov.savefig(Path('results/figures/pdf/{}_cov.pdf'.format(test_name)), bbox_inches="tight")
......
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on 30.03.2021
@author: Feliks Kiszkurno
"""
import slopestabilitytools
import settings
def reassign_classes(test_results, class_type): # It will use number of classes defined in settings
for test_name in test_results.keys():
test_data = test_results[test_name]
if class_type == 'norm':
data_array = test_data['RESN']
class_array = slopestabilitytools.assign_classes(data_array)
test_data['CLASSN'] = class_array
test_data.to_csv(settings.settings['data_folder'] + '/' + test_name + '.csv')
test_results[test_name] = test_data
elif class_type == 'raw':
# Actually it is not needed right now
print('Why? It is in CLASS column...')
else:
print('Incorrect type of classes! Classes has not been reassigned')
exit(0)
return test_results
......@@ -6,9 +6,18 @@ Created on 25.03.2021
@author: Feliks Kiszkurno
"""
def save_plot(figure_handler, test_name, figure_name):
figure_handler.savefig('results/figures/png/' + test_name + '_1_geometry'+'.png', bbox_inches="tight")
import os
from pathlib import Path
# fig_geometry.savefig('results/figures/pdf/' + test_name + '_1_geometry.pdf', bbox_inches="tight")
# fig_geometry.savefig('results/figures/eps/' + test_name + '_1_geometry.eps', bbox_inches="tight")
import settings
def save_plot(figure_handler, test_name, figure_name, *, subfolder=False): # keep in mind, that subfolder has to exist, it wont be created here
if isinstance(subfolder, str) is True:
subfolder = '/' + subfolder + '/'
for file_format in settings.settings['plot_formats']:
figure_handler.savefig(Path(os.getcwd() + '/' + settings.settings[
'figures_folder'] + subfolder + file_format + '/' + test_name + figure_name + '.' + file_format),
bbox_inches="tight")
......@@ -9,3 +9,4 @@ Created on 15.01.2021
from .create_data import create_data
from .test_parameters import test_parameters
from .clip_data import clip_data
\ No newline at end of file
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on 30.03.2021
@author: Feliks Kiszkurno
"""
import numpy as np
def clip_data(array, array_max, array_min):
ids_max = np.argwhere(array > array_max)
ids_min = np.argwhere(array < array_min)
array[ids_max] = array_max
array[ids_min] = array_min
return array
......@@ -15,10 +15,11 @@ import pygimli.meshtools as mt
import pygimli.physics.ert as ert
import slopestabilitytools
import slostabcreatedata
import settings
def create_data(test_name, test_config, max_depth):
from main import settings
world_boundary_v = [-9 * max_depth, 0] # [NW edge] relatively to the middle
world_boundary_h = [9 * max_depth, -9 * max_depth] # [SE edge]
......@@ -40,7 +41,8 @@ def create_data(test_name, test_config, max_depth):
ax_geometry = slopestabilitytools.set_labels(ax_geometry)
ax_geometry.set_title('1 Geometry of the model')
fig_geometry.tight_layout()
fig_geometry.savefig('results/figures/png/' + test_name + '_1_geometry.png', bbox_inches="tight")
slopestabilitytools.save_plot(fig_geometry, test_name, '_1_geometry')
# fig_geometry.savefig('results/figures/png/' + test_name + '_1_geometry.png', bbox_inches="tight")
#fig_geometry.savefig('results/figures/pdf/' + test_name + '_1_geometry.pdf', bbox_inches="tight")
#fig_geometry.savefig('results/figures/eps/' + test_name + '_1_geometry.eps', bbox_inches="tight")
......@@ -61,7 +63,9 @@ def create_data(test_name, test_config, max_depth):
ax_model = slopestabilitytools.set_labels(ax_model)
ax_model.set_title('2 Mesh and resistivity distribution')
fig_model.tight_layout()
fig_model.savefig('results/figures/png/' + test_name + '_2_meshdist.png', bbox_inches="tight")
plot_name = '_2_meshdist.png'
slopestabilitytools.save_plot(fig_model, test_name, '_2_meshdist')
#fig_model.savefig('results/figures/png/' + test_name + '_2_meshdist.png', bbox_inches="tight")
#fig_model.savefig('results/figures/pdf/' + test_name + '_2_meshdist.pdf', bbox_inches="tight")
#fig_model.savefig('results/figures/eps/' + test_name + '_2_meshdist.eps', bbox_inches="tight")
......@@ -82,34 +86,44 @@ def create_data(test_name, test_config, max_depth):
quality=34, zPower=0.4)
result_full = ert_manager.inv.model
# result_array = pg.utils.interperc(result_full, 5)
# result_lim = result_full.array()
# result_lim[np.where(result_array > max(resistivity_map[1]))] = float("NaN")
# result_lim[np.where(result_array < min(resistivity_map[1]))] = float("NaN") # min(resistivity_map[1])
# result_array = result_lim
result_array = result_full.array()
result_array_norm = slopestabilitytools.normalize(result_array)
# result_array_norm = slopestabilitytools.normalize(result_array)
fig_result, ax_result = plt.subplots(1)
pg.show(ert_manager.paraDomain, result_full, label=pg.unit('res'), showMesh=True, ax=ax_result)
ax_result = slopestabilitytools.set_labels(ax_result)
ax_result.set_title('3 Result')
fig_result.tight_layout()
fig_result.savefig('results/figures/png/' + test_name + '_3_result.png', bbox_inches="tight")
slopestabilitytools.save_plot(fig_result, test_name, '_3_result')
# fig_result.savefig('results/figures/png/' + test_name + '_3_result.png', bbox_inches="tight")
#fig_result.savefig('results/figures/pdf/' + test_name + '_3_result.pdf', bbox_inches="tight")
#fig_result.savefig('results/figures/eps/' + test_name + '_3_result.eps', bbox_inches="tight")
input_model2 = pg.interpolate(srcMesh=mesh, inVec=input_model, destPos=ert_manager.paraDomain.cellCenters())
input_model2_array = input_model2.array()
if settings.settings['clip'] is True:
array_max = np.max(input_model2_array)
array_min = np.min(input_model2_array)
#input_model2_array = slostabcreatedata.clip_data(input_model2_array, array_max, array_min)
result_array = slostabcreatedata.clip_data(result_array, array_max, array_min)
input_model2_array_norm = slopestabilitytools.normalize(input_model2_array)
fig_input, ax_input = plt.subplots(1)
pg.show(ert_manager.paraDomain, input_model2, label=pg.unit('res'), showMesh=True, ax=ax_input)
ax_input = slopestabilitytools.set_labels(ax_input)
ax_input.set_title('4 Model on inv mesh')
fig_input.tight_layout()
fig_input.savefig('results/figures/png/' + test_name + '_4_modelinv.png', bbox_inches="tight")
slopestabilitytools.save_plot(fig_input, test_name, '_4_modelinv')
#fig_input.savefig('results/figures/png/' + test_name + '_4_modelinv.png', bbox_inches="tight")
#fig_input.savefig('results/figures/pdf/' + test_name + '_4_modelinv.pdf', bbox_inches="tight")
#fig_input.savefig('results/figures/eps/' + test_name + '_4_modelinv.eps', bbox_inches="tight")
......@@ -171,6 +185,6 @@ def create_data(test_name, test_config, max_depth):
# test_results[test_name] = experiment_results
experiment_results.to_csv('results/results/' + test_name + '.csv')
experiment_results.to_csv(settings.settings['data_folder'] + '/' + test_name + '.csv')
return experiment_results, rho_max, rho_min
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment