To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit e121718c authored by holukas's avatar holukas
Browse files

v0.11.0 random forest update

parent 2488de40
......@@ -3,9 +3,11 @@
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/python" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/src/main/python/_current" />
<excludeFolder url="file://$MODULE_DIR$/src/main/python/_not_used" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="jdk" jdkName="Python 3.6 (AMP06)" jdkType="Python SDK" />
<orderEntry type="jdk" jdkName="Python 3.6 (AMP2)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TestRunnerService">
......
......@@ -3,5 +3,5 @@
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (AMP06)" project-jdk-type="Python SDK" />
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (AMP2)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
This diff is collapsed.
name: AMP06
name: AMP2
channels:
- defaults
dependencies:
......@@ -17,7 +17,6 @@ dependencies:
- python=3.6.9=h5500b2f_0
- python-dateutil=2.8.0=py36_0
- pytz=2019.2=py_0
- scikit-learn=0.21.3=py36h6288b17_0
- setuptools=41.2.0=py36_0
- sqlite=3.29.0=he774522_0
- vc=14.1=h0510ff6_4
......@@ -81,6 +80,7 @@ dependencies:
- pywin32-ctypes==0.2.0
- pyyaml==5.1.2
- pyzmq==18.1.0
- scikit-learn==0.22.1
- scipy==1.3.1
- seaborn==0.9.0
- sip==4.19.8
......@@ -94,5 +94,5 @@ dependencies:
- webencodings==0.5.1
- wrapt==1.11.2
- zipp==0.6.0
prefix: C:\Users\holukas\Anaconda3\envs\AMP06
prefix: C:\Users\holukas\Anaconda3\envs\AMP2
# This file may be used to create an environment using:
# $ conda create --name <env> --file <this file>
# platform: win-64
altgraph=0.16.1=pypi_0
astroid=2.3.1=pypi_0
astropy=3.2.1=pypi_0
atomicwrites=1.3.0=pypi_0
attrs=19.2.0=pypi_0
blas=1.0=mkl
bleach=3.1.0=pypi_0
certifi=2019.9.11=py36_0
colorama=0.4.1=pypi_0
confuse=1.0.0=pypi_0
cycler=0.10.0=pypi_0
decorator=4.4.0=pypi_0
defusedxml=0.6.0=pypi_0
entrypoints=0.3=pypi_0
fbs=0.8.4=pypi_0
future=0.17.1=pypi_0
htmlmin=0.1.12=pypi_0
icc_rt=2019.0.0=h0cc432a_1
importlib-metadata=0.23=pypi_0
intel-openmp=2019.4=245
ipython-genutils=0.2.0=pypi_0
isort=4.3.21=pypi_0
jinja2=2.10.1=pypi_0
joblib=0.13.2=py36_0
jsonschema=3.0.2=pypi_0
jupyter-client=5.3.3=pypi_0
jupyter-core=4.5.0=pypi_0
kiwisolver=1.1.0=pypi_0
lazy-object-proxy=1.4.2=pypi_0
llvmlite=0.29.0=pypi_0
macholib=1.11=pypi_0
markupsafe=1.1.1=pypi_0
matplotlib=3.1.1=pypi_0
mccabe=0.6.1=pypi_0
missingno=0.4.2=pypi_0
mistune=0.8.4=pypi_0
mkl=2019.4=245
mkl-service=2.3.0=py36hb782905_0
mkl_fft=1.0.14=py36h14836fe_0
mkl_random=1.1.0=py36h675688f_0
more-itertools=7.2.0=pypi_0
nbconvert=5.6.0=pypi_0
nbformat=4.4.0=pypi_0
numba=0.45.1=pypi_0
numpy=1.17.2=pypi_0
numpy-base=1.16.5=py36hc3f5095_0
packaging=19.2=pypi_0
pandas=0.25.1=py36ha925a31_0
pandas-profiling=2.3.0=pypi_0
pandocfilters=1.4.2=pypi_0
patsy=0.5.1=pypi_0
pefile=2019.4.18=pypi_0
phik=0.9.8=pypi_0
pip=19.2.3=py36_0
pluggy=0.13.0=pypi_0
py=1.8.0=pypi_0
pygments=2.4.2=pypi_0
pyinstaller=3.4=pypi_0
pylint=2.4.2=pypi_0
pyparsing=2.4.2=pypi_0
pyqt5=5.9.2=pypi_0
pyrsistent=0.15.4=pypi_0
pytest=5.2.0=pypi_0
pytest-pylint=0.14.1=pypi_0
python=3.6.9=h5500b2f_0
python-dateutil=2.8.0=py36_0
pytz=2019.2=py_0
pywin32=225=pypi_0
pywin32-ctypes=0.2.0=pypi_0
pyyaml=5.1.2=pypi_0
pyzmq=18.1.0=pypi_0
scikit-learn=0.21.3=py36h6288b17_0
scipy=1.3.1=pypi_0
seaborn=0.9.0=pypi_0
setuptools=41.2.0=py36_0
sip=4.19.8=pypi_0
six=1.12.0=pypi_0
sqlite=3.29.0=he774522_0
statsmodels=0.10.1=pypi_0
testpath=0.4.2=pypi_0
tornado=6.0.3=pypi_0
traitlets=4.3.2=pypi_0
typed-ast=1.4.0=pypi_0
vc=14.1=h0510ff6_4
vs2015_runtime=14.16.27012=hf0eaf9b_0
wcwidth=0.1.7=pypi_0
webencodings=0.5.1=pypi_0
wheel=0.33.6=py36_0
wincertstore=0.2=py36h7fe50ca_0
wrapt=1.11.2=pypi_0
zipp=0.6.0=pypi_0
altgraph==0.16.1
astroid==2.3.1
astropy==3.2.1
atomicwrites==1.3.0
attrs==19.2.0
bleach==3.1.0
certifi==2019.9.11
colorama==0.4.1
confuse==1.0.0
cycler==0.10.0
decorator==4.4.0
defusedxml==0.6.0
entrypoints==0.3
fbs==0.8.4
future==0.17.1
htmlmin==0.1.12
importlib-metadata==0.23
ipython-genutils==0.2.0
isort==4.3.21
Jinja2==2.10.1
joblib==0.13.2
jsonschema==3.0.2
jupyter-client==5.3.3
jupyter-core==4.5.0
kiwisolver==1.1.0
lazy-object-proxy==1.4.2
llvmlite==0.29.0
macholib==1.11
MarkupSafe==1.1.1
matplotlib==3.1.1
mccabe==0.6.1
missingno==0.4.2
mistune==0.8.4
mkl-fft==1.0.14
mkl-random==1.1.0
mkl-service==2.3.0
more-itertools==7.2.0
nbconvert==5.6.0
nbformat==4.4.0
numba==0.45.1
numpy==1.17.2
packaging==19.2
pandas==0.25.1
pandas-profiling==2.3.0
pandocfilters==1.4.2
patsy==0.5.1
pefile==2019.4.18
phik==0.9.8
pluggy==0.13.0
py==1.8.0
Pygments==2.4.2
PyInstaller==3.4
pylint==2.4.2
pyparsing==2.4.2
PyQt5==5.9.2
pyrsistent==0.15.4
pytest==5.2.0
pytest-pylint==0.14.1
python-dateutil==2.8.0
pytz==2019.2
pywin32==225
pywin32-ctypes==0.2.0
PyYAML==5.1.2
pyzmq==18.1.0
scikit-learn==0.22.1
scipy==1.3.1
seaborn==0.9.0
sip==4.19.8
six==1.12.0
statsmodels==0.10.1
testpath==0.4.2
tornado==6.0.3
traitlets==4.3.2
typed-ast==1.4.0
wcwidth==0.1.7
webencodings==0.5.1
wincertstore==0.2
wrapt==1.11.2
zipp==0.6.0
EVENT_START,EVENT_END,EVENT
[yyyy-mm-dd],[yyyy-mm-dd],[type]
2019-06-07,2019-06-07,F49
2019-07-11,2019-07-11,F102
2019-08-19,2019-08-24,G
2019-09-27,2019-09-27,F79
......@@ -10,7 +10,7 @@
from inout.SettingsOps import update_ampsettings_dict
# from inout.TimestampTypes import C1_FullDateTimeNoSec
from inout.TimestampTypes import C1_FullDateTimeWithSec
from inout import DataFunctions
class Settings(object):
......@@ -43,5 +43,6 @@ class Settings(object):
def get_file_settings_dict(self):
return self.file_settings_dict
def post_parse(df):
return df
import pandas as pd
from inout.SettingsOps import update_settings_dict
from inout.TimestampTypes import C1_FullDateTimeWithSec
from inout.TimestampTypes import C1_FullDateNoTime
class Settings(object):
def __init__(self):
self.file_settings_dict = self.file_settings()
def file_settings(self):
""" File settings are filled into the GUI """
_timestamp_info_dict = C1_FullDateNoTime.Settings().get_timestamp_info_dict()
settings_dict = {
'filetype': 'Events',
'file_ext': '*.events',
'parse_date': '1', # 0=Yes, 1=No
'index_col': '0',
'header_rows': '[0, 1]',
'freq': '-none-',
'na_values': '-9999',
'skip_rows': '[]',
'timestamp_type': _timestamp_info_dict['format_id'],
'date_and_time_colstr': _timestamp_info_dict['date_and_time_colstr'],
'timestamp_type_drp_ix': _timestamp_info_dict['dropdown_ix'],
'zipped': 'No',
'delimiter': ','}
settings_dict = update_settings_dict(settings_dict=settings_dict)
return settings_dict
def get_file_settings_dict(self):
return self.file_settings_dict
def post_parse(df):
# data_df[('TIMESTAMP', '[yyyy-mm-dd HH:MM:SS]')] = data_df.index
# CONVERSION of events df to standardized format as used by timeseries data
# Detect how many unique events are given in the Events File and
# use this information to generate a new DataFrame
unique_events = set(df.iloc[:, 0].tolist()) ## event description in second col
# Make sure the index is datetime
# data_df.index = pd.to_datetime(data_df.index)
_df = pd.DataFrame(index=df.index, columns=unique_events)
# _df.index = pd.to_datetime(_df.index)
# In the data_df generated from reading the Events File, the event type is given in rows
# In _df, this information was used to generate a new df with the event type as column names
# The iterrows method cycles through all rows in the data_df one by one, reads the event
# type from the row, and inserts a flag in the corresponding column in _df
for ix, row in df.iterrows():
_df.loc[ix, row] = 1
lst_for_event_units = []
for e in range(len(_df.columns)): ## generate entry for all cols in df
lst_for_event_units.append('[event]')
_df.columns = [_df.columns, lst_for_event_units]
data_df = _df.copy()
# data_df[('TIMESTAMP', '[yyyy-mm-dd HH:MM:SS]')] = data_df.index
return data_df
......@@ -51,7 +51,7 @@ Kept here so it can be found with search function.
# # Show random example data in plot on load
# example_data_df = DataFunctions.example_timeseries()
# pl_Overview.Call(fig=self.plt_Figure, focus_df=example_data_df, focus_col=('example_y', 'units'),
# pl_Overview.Call(fig=self.plt_Figure, focus_df=example_data_df, measured_col=('example_y', 'units'),
# ms=4, color='#44b2d3', data_df=example_data_df)
# lbl_history = qw.QLabel('Variable Name > Step 1 > Step 2')
......@@ -122,13 +122,11 @@ Kept here so it can be found with search function.
# k = model.params[X_independent_var]
# def resample(df, timeres, method):
# # TODO separate df for resampled data? but plot in same plot
# # original_col_order = df.columns
#
# df2 = df.copy()
# df2 = non_numeric_cols(df=df2, remove=True)
#
# # todo all of this needs to be shifted because timestamp span
# # add col that is used as group
# df2.loc[:, 'Group'] = df2.index.to_period(timeres).start_time
# grouped_df = df2.groupby(['Group'])
......@@ -151,3 +149,42 @@ Kept here so it can be found with search function.
# else:
# return df_non_numeric_cols
# TODO testing font size
# font = QFont()
# font.setFamily("Arial")
# font.setPointSize(9999)
# font.setBold(True)
# self.texteditor1 = QtGui.QLineEdit(self)
# font = self.lbl_FocusVariableName.font() # lineedit current font
# font.setPointSize(32) # change it's size
# self.lbl_FocusVariableName.setFont(font) # set font
# self.lbl_FocusVariableName.setFont(font)
# Init StretchBox, used where needed
# grd_Stretch = GUI_BuildingBlocks.StretchBox().get_frame()
# stretch_widget = qw.QWidget()
# stretch_widget.setProperty('labelClass', '_StretchWidget')
# self.grd_categories.addWidget(stretch_widget, 5, 0, 1, 1)
# self.grd_categories.setRowStretch(5, 1)
# def plot_focus(self, fig, plot_type):
# """
#
# Creates a new axis and plots the selected focus data.
# The newly created axis is stored in self.ax_main
#
# :param fig:
# :param type:
# :return: None
# """
#
# # self.focus_data_setting = 'Single' # todo
# # if self.focus_data_setting == 'Single':
# # # plot.clear_ax(ax_main=ax_main)
# # self.cw_ix = 0
# # elif self.focus_data_setting == 'Multiple':
# # self.cw_ix += 1
# # elif self.focus_data_setting == 'Add Panel': # todo
# # pass
import ast
from pathlib import Path
from PyQt5 import QtWidgets as qw
from gui import gui_building_blocks, gui_elements
from inout.DataFunctions import search_files
class SelectFileTypeSettings(gui_building_blocks.SelectFileTypeSettingsFile):
""" Settings window is created in gui_building_blocks. """
def __init__(self, dir_filetypes):
super().__init__()
self.settings_dict = {} ## initialize dict that stores settings_dict
# Dialog window
self.layout, self.filelist = self.gui()
# Search files and show them in list
self.found_files_dict = self.search_files(dir_filetypes=dir_filetypes)
# Make buttons in gui
self.add_files_to_list()
def get_filepath(self):
selected_file = self.filelist.selectedIndexes()
selected_file_name = selected_file[0].data()
selected_file_path = self.found_files_dict[selected_file_name]
return selected_file_path
@staticmethod
def read_settings_from_file(filepath):
settings_dict = {}
with open(filepath) as input_file:
for line in input_file: # cycle through all lines in settings settings_file_name
if line.startswith('#'):
pass
else:
if '=' in line: # identify lines that contain setting
line_id, line_setting = line.split('=')
line_id = line_id.strip()
line_setting = line_setting.strip()
settings_dict[line_id] = line_setting # store setting from settings_file_name in dict
settings_dict = SelectFileTypeSettings.check_integrity(settings_dict=settings_dict)
return settings_dict
@staticmethod
def check_integrity(settings_dict):
""" Convert to required types. """
# GENERAL
# -------
settings_dict['NAME'] = str(settings_dict['NAME'])
settings_dict['DESCRIPTION'] = str(settings_dict['DESCRIPTION'])
# FILE
# ----
settings_dict['FILE_EXTENSION'] = str(settings_dict['FILE_EXTENSION'])
settings_dict['FILE_COMPRESSION'] = str(settings_dict['FILE_COMPRESSION'])
# TIMESTAMP INDEX
# ---------------
settings_dict['TIMESTAMP_PARSE_DATES'] = str(settings_dict['TIMESTAMP_PARSE_DATES'])
settings_dict['TIMESTAMP_INDEX_COLUMN'] = int(settings_dict['TIMESTAMP_INDEX_COLUMN'])
settings_dict['TIMESTAMP_DESCRIPTION'] = str(settings_dict['TIMESTAMP_DESCRIPTION'])
settings_dict['TIMESTAMP_DATE_AND_TIME_IN_SEPARATE_COLUMNS'] = \
str(settings_dict['TIMESTAMP_DATE_AND_TIME_IN_SEPARATE_COLUMNS'])
if settings_dict['TIMESTAMP_DATE_AND_TIME_IN_SEPARATE_COLUMNS'] == 'yes':
settings_dict['TIMESTAMP_DATE_COLUMN_NAME'] = ast.literal_eval(settings_dict['TIMESTAMP_DATE_COLUMN_NAME'])
settings_dict['TIMESTAMP_TIME_COLUMN_NAME'] = ast.literal_eval(settings_dict['TIMESTAMP_TIME_COLUMN_NAME'])
# settings_dict['TIMESTAMP_DATE_FORMAT'] = str(settings_dict['TIMESTAMP_DATE_FORMAT'])
# settings_dict['TIMESTAMP_TIME_FORMAT'] = str(settings_dict['TIMESTAMP_TIME_FORMAT'])
else:
settings_dict['TIMESTAMP_DATETIME_FORMAT'] = str(settings_dict['TIMESTAMP_DATETIME_FORMAT'])
# DATA
# ----
settings_dict['DATA_HEADER_ROWS'] = ast.literal_eval(settings_dict['DATA_HEADER_ROWS'])
try:
settings_dict['DATA_NA_VALUES'] = int(settings_dict['DATA_NA_VALUES'])
except ValueError:
settings_dict['DATA_NA_VALUES'] = str(settings_dict['DATA_NA_VALUES'])
settings_dict['DATA_SKIP_ROWS'] = ast.literal_eval(settings_dict['DATA_SKIP_ROWS'])
settings_dict['DATA_FREQUENCY'] = str(settings_dict['DATA_FREQUENCY'])
settings_dict['DATA_DELIMITER'] = str(settings_dict['DATA_DELIMITER'])
# # todo required_keys could be detected from a "master" file
# required_keys = ['NAME', 'DESCRIPTION', 'FILE_EXTENSION', 'FILE_COMPRESSION', 'TIMESTAMP_PARSE_DATES',
# 'TIMESTAMP_INDEX_COLUMN', 'TIMESTAMP_DESCRIPTION', 'TIMESTAMP_DATETIME_FORMAT',
# 'TIMESTAMP_DATE_COLUMN', 'TIMESTAMP_TIME_COLUMN', 'DATA_HEADER_ROWS', 'DATA_NA_VALUES',
# 'DATA_SKIP_ROWS', 'DATA_FREQUENCY', 'DATA_DELIMITER']
# required_num_keys = len(required_keys)
#
# found_keys = list(settings_dict.keys())
#
# found_num_keys = 0
# for k in required_keys:
# if k in found_keys:
# found_num_keys += 1
# else:
# print(f"{k} was not found in filetype settings!")
# return -1
#
# if found_num_keys == required_num_keys:
# print(f"Found {found_num_keys} setting entries, required {required_num_keys}")
return settings_dict
def add_files_to_list(self):
for filename, filepath in self.found_files_dict.items():
# Info from .filetype file
temp_dict = self.read_settings_from_file(filepath=filepath)
info = '<h3>{}</h3></br></br>'.format(temp_dict['NAME'])
del temp_dict['NAME'] # Remove from dict, we already used it
info += '<br/><i>{}</i><br/>'.format(temp_dict['DESCRIPTION'])
del temp_dict['DESCRIPTION'] # Remove from dict, we already used it
info += '<br/><i>{}</i><br/><br/>'.format(temp_dict['TIMESTAMP_DESCRIPTION'])
del temp_dict['TIMESTAMP_DESCRIPTION'] # Remove from dict, we already used it
info += '<br/>'.join(['<b>%s</b>: %s' % (key, value) for (key, value) in temp_dict.items()])
# Item for list
item = qw.QListWidgetItem(f"{filename}")
item.setToolTip(info)
self.filelist.addItem(item) # add column name to list
def search_files(self, dir_filetypes):
# dir_filetypes = Path(dir_filetypes) / 'inout' / 'FileTypes'
found_files_dict = search_files(src_dir=dir_filetypes)
return found_files_dict
def gui(self):
# Constructs the gui for the settings_dict window
layout = self.setupUi(self, win_title='File Settings')
filelist = gui_elements.lyt_List(layout=layout)
filelist.setWordWrap(True)
filelist.setAlternatingRowColors(True)
layout.addWidget(filelist)
# Button box: OK and Cancel
self.button_box = qw.QDialogButtonBox()
self.button_box.setEnabled(True)
self.button_box.setStandardButtons(qw.QDialogButtonBox.Cancel | qw.QDialogButtonBox.Ok)
self.button_box.setObjectName("button_box")
self.button_box.accepted.connect(self.accept)
self.button_box.rejected.connect(self.reject)
layout.addWidget(self.button_box)
return layout, filelist
# # Header
# label = qw.QLabel('File Settings')
# label.setProperty('labelClass', 'lbl_Header2')
# self.layout.addWidget(label, 0, 0, 1, 1)
......@@ -9,10 +9,26 @@
from PyQt5 import QtWidgets as qw
from PyQt5.QtGui import QIcon
from gui import GUI_Elements
from gui import gui_elements
from pathlib import Path
import pathlib
class SelectFileTypeSettingsFile(qw.QDialog):
def setupUi(self, SelectFileTypeSettingsFile, win_title):
SelectFileTypeSettingsFile.setAccessibleName('SettingsWindow')
SelectFileTypeSettingsFile.setWindowTitle(win_title)
SelectFileTypeSettingsFile.setWindowIcon(QIcon('gui/Icons/python_icon.png'))
SelectFileTypeSettingsFile.resize(1000, 600)
SelectFileTypeSettingsFile.setContentsMargins(0, 0, 0, 0)
# Container
layout = qw.QVBoxLayout()
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
self.setLayout(layout)
return layout
class SettingsWindow(qw.QDialog):
def setupUi(self, SettingsWindow, win_title):
SettingsWindow.setAccessibleName('SettingsWindow')
......@@ -73,7 +89,7 @@ class CategoryOptions:
def add_header(self):
# Header text is changed by modboxes
header = GUI_Elements.grd_Label(lyt=self.lyt_CategoryOptions,
header = gui_elements.grd_Label(lyt=self.lyt_CategoryOptions,
txt='*Box Title*',
css_id='lbl_Header2',