To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 9af1d38d authored by holukas's avatar holukas
Browse files

updated MDS gap-filling

parent 102551ac
...@@ -8,9 +8,6 @@ ...@@ -8,9 +8,6 @@
<orderEntry type="jdk" jdkName="Python 3.6 (AMP06)" jdkType="Python SDK" /> <orderEntry type="jdk" jdkName="Python 3.6 (AMP06)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="sourceFolder" forTests="false" />
</component> </component>
<component name="PyDocumentationSettings">
<option name="myDocStringFormat" value="NumPy" />
</component>
<component name="TestRunnerService"> <component name="TestRunnerService">
<option name="projectConfiguration" value="pytest" /> <option name="projectConfiguration" value="pytest" />
<option name="PROJECT_TEST_RUNNER" value="pytest" /> <option name="PROJECT_TEST_RUNNER" value="pytest" />
......
This diff is collapsed.
...@@ -75,7 +75,7 @@ ...@@ -75,7 +75,7 @@
33. when importing data from file, renaming columns possible in GUI, and if name already exists then ask if it should be merged with existing 33. when importing data from file, renaming columns possible in GUI, and if name already exists then ask if it should be merged with existing
34. when importing data and data column already exists, define both should be merged or one prioritized over the other 34. when importing data and data column already exists, define both should be merged or one prioritized over the other
35. output some kind of stats overview page for all vars 35. output some kind of stats overview page for all vars
36. CHECK: whether other methods also need shifted timestamp column, e.g. I think in u* filtering, or resampling to e.g. daily average 36. CHECK: whether other methods also need shifted timestamp column, e.g. I think in u* filtering, or resampling to e.g. daily average (for ustar checked in 0.9.0)
37. **Modifications:** convert *complete* df to e.g. daily average, or also something like running median lines etc… 37. **Modifications:** convert *complete* df to e.g. daily average, or also something like running median lines etc…
38. hexbins per year etc option 38. hexbins per year etc option
39. Waffle chart: https://github.com/gyli/PyWaffle 39. Waffle chart: https://github.com/gyli/PyWaffle
......
...@@ -63,7 +63,7 @@ class Ui_MainWindow(object): ...@@ -63,7 +63,7 @@ class Ui_MainWindow(object):
f"{info['version']}") f"{info['version']}")
MainWindow.setWindowIcon(QIcon(self.ctx.icon_amp)) MainWindow.setWindowIcon(QIcon(self.ctx.icon_amp))
# MainWindow.setWindowIcon(QIcon('gui/Icons/python_icon.png')) # MainWindow.setWindowIcon(QIcon('gui/Icons/python_icon.png'))
MainWindow.resize(1500, 1000) MainWindow.resize(1920, 1080)
MainWindow.setContentsMargins(0, 0, 0, 0) MainWindow.setContentsMargins(0, 0, 0, 0)
# Init CentralWidget (level 1) # Init CentralWidget (level 1)
...@@ -162,6 +162,7 @@ class Ui_MainWindow(object): ...@@ -162,6 +162,7 @@ class Ui_MainWindow(object):
lyt_FocusAreaGrid = qw.QGridLayout() lyt_FocusAreaGrid = qw.QGridLayout()
# lyt_FocusAreaVert = qw.QVBoxLayout() # lyt_FocusAreaVert = qw.QVBoxLayout()
lyt_FocusAreaGrid.setContentsMargins(0, 0, 0, 0) lyt_FocusAreaGrid.setContentsMargins(0, 0, 0, 0)
lyt_FocusAreaGrid.setSpacing(5)
# Init StretchBox, used where needed # Init StretchBox, used where needed
# grd_Stretch = GUI_BuildingBlocks.StretchBox().get_frame() # grd_Stretch = GUI_BuildingBlocks.StretchBox().get_frame()
...@@ -170,9 +171,22 @@ class Ui_MainWindow(object): ...@@ -170,9 +171,22 @@ class Ui_MainWindow(object):
# HEADER /////////////////////////////////////////////////////////// # HEADER ///////////////////////////////////////////////////////////
# ================================================================== # ==================================================================
# # todo testing font size
# font = QFont()
# font.setFamily("Arial")
# font.setPointSize(9999)
# font.setBold(True)
self.lbl_FocusVariableName = qw.QLabel('*Please Select Variable*') self.lbl_FocusVariableName = qw.QLabel('*Please Select Variable*')
self.lbl_FocusVariableName.setProperty('labelClass', 'lbl_Header1') self.lbl_FocusVariableName.setProperty('labelClass', 'lbl_Header1')
# self.texteditor1 = QtGui.QLineEdit(self)
# font = self.lbl_FocusVariableName.font() # lineedit current font
# font.setPointSize(32) # change it's size
# self.lbl_FocusVariableName.setFont(font) # set font
# self.lbl_FocusVariableName.setFont(font)
# ================================================================== # ==================================================================
# PLOTS & STATS //////////////////////////////////////////////////// # PLOTS & STATS ////////////////////////////////////////////////////
# ================================================================== # ==================================================================
...@@ -233,6 +247,7 @@ class Ui_MainWindow(object): ...@@ -233,6 +247,7 @@ class Ui_MainWindow(object):
frm_Categories.setProperty('labelClass', 'frm_Categories') frm_Categories.setProperty('labelClass', 'frm_Categories')
self.grd_categories = qw.QGridLayout() self.grd_categories = qw.QGridLayout()
self.grd_categories.setContentsMargins(0, 0, 0, 0) self.grd_categories.setContentsMargins(0, 0, 0, 0)
self.grd_categories.setSpacing(5)
frm_Categories.setLayout(self.grd_categories) frm_Categories.setLayout(self.grd_categories)
# Init options stack # Init options stack
...@@ -314,23 +329,10 @@ class Ui_MainWindow(object): ...@@ -314,23 +329,10 @@ class Ui_MainWindow(object):
def add_modbox_Plots(self): def add_modbox_Plots(self):
# CATEGORY 0: PLOTS (ModBox) # CATEGORY 0: PLOTS (ModBox)
self.btn_pl_cat_Plots_showStack, \ self.modbox_plots = modbox_pl.AddControls(grd_Categories=self.grd_categories,
self.btn_pl_opt_Heatmap, \ stk_Options=self.stk_category_options,
self.btn_pl_ref_Heatmap_showInNewTab, \ stk_Refinements=self.stk_option_refinements,
self.pl_ref_Heatmap_stackTab, \ ctx=self.ctx)
self.btn_pl_opt_Scatter, \
self.pl_ref_Scatter_stackTab, \
self.btn_pl_ref_Scatter_showInNewTab, \
self.pl_ref_FXN_ThresholdPlots_stackTab, \
self.btn_pl_opt_FXN_ThresholdPlots, \
self.btn_pl_ref_FXN_ThresholdPlots_showInNewTab, \
self.pl_ref_Hexbins_stackTab, \
self.btn_pl_opt_Hexbins, \
self.btn_pl_ref_Hexbins_showInNewTab = \
modbox_pl.AddControls(grd_Categories=self.grd_categories,
stk_Options=self.stk_category_options,
stk_Refinements=self.stk_option_refinements,
ctx=self.ctx).get_handles()
def add_modbox_QualityControl(self): def add_modbox_QualityControl(self):
self.qc_ref_AmpQcFlags_stackTab, \ self.qc_ref_AmpQcFlags_stackTab, \
......
...@@ -24,6 +24,7 @@ def grd_Button(lyt, txt, css_id, row, col, rowspan, colspan): ...@@ -24,6 +24,7 @@ def grd_Button(lyt, txt, css_id, row, col, rowspan, colspan):
lyt.addWidget(button, row, col, rowspan, colspan) lyt.addWidget(button, row, col, rowspan, colspan)
return button return button
def grd_ButtonIcon(lyt, txt, css_id, row, col, rowspan, colspan, icon): def grd_ButtonIcon(lyt, txt, css_id, row, col, rowspan, colspan, icon):
button = qw.QPushButton(text=txt) button = qw.QPushButton(text=txt)
button.setProperty('labelClass', css_id) button.setProperty('labelClass', css_id)
...@@ -105,8 +106,12 @@ def grd_LabelLineditPair(txt, css_ids, layout, row, col, orientation): ...@@ -105,8 +106,12 @@ def grd_LabelLineditPair(txt, css_ids, layout, row, col, orientation):
layout.addWidget(lbl_txt, row, col, 1, 1) layout.addWidget(lbl_txt, row, col, 1, 1)
layout.addWidget(lne_val, row + 1, col, 1, 1) layout.addWidget(lne_val, row + 1, col, 1, 1)
layout.setColumnStretch(0, 0)
layout.setColumnStretch(1, 1)
return lne_val return lne_val
def grd_LabelDateEditPair(layout, row, col, css_ids, txt): def grd_LabelDateEditPair(layout, row, col, css_ids, txt):
lbl_txt = qw.QLabel(text=txt) lbl_txt = qw.QLabel(text=txt)
dte_date = qw.QDateEdit() dte_date = qw.QDateEdit()
...@@ -121,6 +126,7 @@ def grd_LabelDateEditPair(layout, row, col, css_ids, txt): ...@@ -121,6 +126,7 @@ def grd_LabelDateEditPair(layout, row, col, css_ids, txt):
# date_edit.calendarWidget().installEventFilter() # date_edit.calendarWidget().installEventFilter()
return dte_date return dte_date
def grd_LabelLineeditDropdownTriplet(txt, css_ids, layout, row, col, orientation): def grd_LabelLineeditDropdownTriplet(txt, css_ids, layout, row, col, orientation):
# Adds two labels to a grid lyt_CategoryOptions: one for the label text, one for the value. # Adds two labels to a grid lyt_CategoryOptions: one for the label text, one for the value.
# Returns the id for the value label so it can later be updated with real values. # Returns the id for the value label so it can later be updated with real values.
...@@ -146,6 +152,7 @@ def grd_LabelDropdownPair(txt, css_ids, layout, row, col, orientation): ...@@ -146,6 +152,7 @@ def grd_LabelDropdownPair(txt, css_ids, layout, row, col, orientation):
# Returns the id for the value label so it can later be updated with real values. # Returns the id for the value label so it can later be updated with real values.
lbl_txt = qw.QLabel(text=txt) lbl_txt = qw.QLabel(text=txt)
drp_options = qw.QComboBox() drp_options = qw.QComboBox()
drp_options.setMaxVisibleItems(30)
lbl_txt.setProperty('labelClass', css_ids[0]) lbl_txt.setProperty('labelClass', css_ids[0])
drp_options.setProperty('labelClass', css_ids[1]) drp_options.setProperty('labelClass', css_ids[1])
...@@ -197,6 +204,7 @@ class ComboBox_dropEnabled(qw.QComboBox): ...@@ -197,6 +204,7 @@ class ComboBox_dropEnabled(qw.QComboBox):
self.model_mime_type = 'application/x-qabstractitemmodeldatalist' ## format from variable list self.model_mime_type = 'application/x-qabstractitemmodeldatalist' ## format from variable list
super(ComboBox_dropEnabled, self).__init__() super(ComboBox_dropEnabled, self).__init__()
self.setAcceptDrops(True) self.setAcceptDrops(True)
self.setMaxVisibleItems(30)
def dragEnterEvent(self, event): def dragEnterEvent(self, event):
if event.mimeData().hasFormat(self.model_mime_type) or event.mimeData().hasFormat('text/plain'): if event.mimeData().hasFormat(self.model_mime_type) or event.mimeData().hasFormat('text/plain'):
......
import ast import ast
import datetime as dt import datetime as dt
import fnmatch
import zipfile as zf import zipfile as zf
from pathlib import Path from pathlib import Path
...@@ -22,12 +23,12 @@ from inout.FileTypes import IfDews_1T ...@@ -22,12 +23,12 @@ from inout.FileTypes import IfDews_1T
from inout.FileTypes import NABEL_1T from inout.FileTypes import NABEL_1T
from inout.FileTypes import TOA5_10T from inout.FileTypes import TOA5_10T
from inout.FileTypes import TOA5_1T from inout.FileTypes import TOA5_1T
from inout.GlobalVars import *
from inout.TimestampTypes import C1_FullDateTimeDayFirstNoSec from inout.TimestampTypes import C1_FullDateTimeDayFirstNoSec
from inout.TimestampTypes import C1_FullDateTimeNanosec from inout.TimestampTypes import C1_FullDateTimeNanosec
from inout.TimestampTypes import C1_FullDateTimeNoSec from inout.TimestampTypes import C1_FullDateTimeNoSec
from inout.TimestampTypes import C1_FullDateTimeOneNumber from inout.TimestampTypes import C1_FullDateTimeOneNumber
from inout.TimestampTypes import C1_FullDateTimeWithSec from inout.TimestampTypes import C1_FullDateTimeWithSec
from inout.TimestampTypes import C1_FullDateNoTime
from inout.TimestampTypes import C2_DateAndTime from inout.TimestampTypes import C2_DateAndTime
from inout.TimestampTypes import NoTimestamp from inout.TimestampTypes import NoTimestamp
...@@ -648,12 +649,15 @@ def sort_multiindex_columns_names(df, priority_vars): ...@@ -648,12 +649,15 @@ def sort_multiindex_columns_names(df, priority_vars):
# Put priority vars first # Put priority vars first
for ix, col in enumerate(cols_list): for ix, col in enumerate(cols_list):
if col[0] in priority_vars: if col[0] in priority_vars:
cols_list.insert(0, cols_list.pop(ix)) # removes from old location ix, puts to top of list cols_list.insert(0, cols_list.pop(ix)) # removes from old location ix, puts to top of list
# Custom vars are marked w/ point at beginning # Custom vars are marked w/ point at beginning
for ix, col in enumerate(cols_list): elif col[0].startswith('.'):
if col[0].startswith('.'): cols_list.insert(0, cols_list.pop(ix))
elif any(fnmatch.fnmatch(col[0], gf_id) for gf_id in GAPFILLED_GENERAL_SCALARS):
cols_list.insert(0, cols_list.pop(ix)) cols_list.insert(0, cols_list.pop(ix))
df = df[cols_list] # assign new (sorted) column order df = df[cols_list] # assign new (sorted) column order
...@@ -931,14 +935,30 @@ def resample_df(df, freq_str, agg_method, min_vals, orig_freq): ...@@ -931,14 +935,30 @@ def resample_df(df, freq_str, agg_method, min_vals, orig_freq):
return agg_df return agg_df
def df_between_two_dates(df, start_date, end_date): def df_between_two_dates(df, start_date, end_date, dropna, dropna_col):
""" Get data for the time window, greater than the start date and smaller than the end date.
:param df: DataFrame
:param start_date: datetime
:param end_date: datetime
:param dropna: bool
:param dropna_col: Column name in df that is checked for NaNs.
:return: Data between start_date and end_date.
:rtype: DataFrame
"""
mask = (df.index >= start_date) & (df.index <= end_date) mask = (df.index >= start_date) & (df.index <= end_date)
snippet_df = df.loc[mask] snippet_df = df.loc[mask]
if dropna:
# Remove rows in df where dropna_col is NaN.
filter_nan = snippet_df[dropna_col].isnull() # True if NaN
snippet_df = snippet_df.loc[~filter_nan] # Keep False (=values, i.e. not NaN)
# snippet_df = snippet_df.dropna()
return snippet_df return snippet_df
def insert_datetimerange(df, win_days, win_hours): def insert_datetimerange(df, win_days, win_hours):
# insert date ranges, varying colnames depending on settings """Insert datetime range that describes start and end of chosen day window."""
win_start_col = ('start_dt_{}d-{}h'.format(win_days, win_hours), '[datetime]') win_start_col = ('start_dt_{}d-{}h'.format(win_days, win_hours), '[datetime]')
win_end_col = ('end_dt_{}d-{}h'.format(win_days, win_hours), '[datetime]') win_end_col = ('end_dt_{}d-{}h'.format(win_days, win_hours), '[datetime]')
df[win_start_col] = df.index - pd.Timedelta(days=win_days) - pd.Timedelta(hours=win_hours) df[win_start_col] = df.index - pd.Timedelta(days=win_days) - pd.Timedelta(hours=win_hours)
...@@ -947,7 +967,7 @@ def insert_datetimerange(df, win_days, win_hours): ...@@ -947,7 +967,7 @@ def insert_datetimerange(df, win_days, win_hours):
def insert_timerange(df, win_hours): def insert_timerange(df, win_hours):
# insert date ranges, varying colnames depending on settings """Insert time range that describes start and end of chosen hour window."""
win_start_time_col = ('start_time_{}h'.format(win_hours), '[time]') win_start_time_col = ('start_time_{}h'.format(win_hours), '[time]')
win_end_time_col = ('end_time_{}h'.format(win_hours), '[time]') win_end_time_col = ('end_time_{}h'.format(win_hours), '[time]')
df[win_start_time_col] = df.index - pd.Timedelta(hours=win_hours) df[win_start_time_col] = df.index - pd.Timedelta(hours=win_hours)
...@@ -972,7 +992,6 @@ def find_nans_in_df_col(df, col): ...@@ -972,7 +992,6 @@ def find_nans_in_df_col(df, col):
temp = pd.isnull(df[col]).to_numpy().nonzero() temp = pd.isnull(df[col]).to_numpy().nonzero()
gaps_df = df.ix[temp] # contains only NEE NaNs, but the index is important to locate missing NEE values gaps_df = df.ix[temp] # contains only NEE NaNs, but the index is important to locate missing NEE values
gap_count = len(gaps_df[col]) gap_count = len(gaps_df[col])
return gaps_df, gap_count return gaps_df, gap_count
......
AIR_TEMPERATURE = ['*TA_*', '*air_temperature*'] AIR_TEMPERATURE_METEO = ['*TA_*']
AIR_TEMPERATURE_EDDYPRO = ['*air_temperature*']
AIR_TEMPERATURE = AIR_TEMPERATURE_METEO + AIR_TEMPERATURE_EDDYPRO
FLUXES_EDDYPRO = ['co2_flux', 'h2o_flux', 'LE', 'H', 'ET', 'n2o_flux', 'ch4_flux'] FLUXES_EDDYPRO = ['co2_flux', 'h2o_flux', 'LE', 'H', 'ET', 'n2o_flux', 'ch4_flux']
FLUXES_FLUXNET = ['NEE*', 'GPP*', 'RECO*'] FLUXES_FLUXNET = ['NEE*', 'GPP*', 'RECO*']
...@@ -13,6 +15,8 @@ FLUXES_GENERAL_N2O = ['*n2o_flux*'] ...@@ -13,6 +15,8 @@ FLUXES_GENERAL_N2O = ['*n2o_flux*']
FLUX_SCALARS_CO2_HIRES = ['co2_ppb*', 'CO2_ppm*', 'CO2_mmol_m-3*'] FLUX_SCALARS_CO2_HIRES = ['co2_ppb*', 'CO2_ppm*', 'CO2_mmol_m-3*']
GAPFILLED_GENERAL_SCALARS = ['*-f*']
MONIN_OBUKHOV_STABILITY = ['(z-d)/L'] MONIN_OBUKHOV_STABILITY = ['(z-d)/L']
# ============================================================================================== # ==============================================================================================
......
...@@ -106,8 +106,8 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo ...@@ -106,8 +106,8 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo
# ================================================================== # ==================================================================
# Load example file when script starts # Load example file when script starts
# self.initial_data, self.settings_dict = example_files.load(file='Amp_30T') # self.initial_data, self.settings_dict = example_files.load(file='Amp_30T')
# self.initial_data, self.settings_dict = example_files.load(file='AmpCSV_30T', ctx=self.ctx) self.initial_data, self.settings_dict = example_files.load(file='AmpCSV_30T', ctx=self.ctx)
self.initial_data, self.settings_dict = example_files.load(file='EddyProFullOutput_30T', ctx=self.ctx) # self.initial_data, self.settings_dict = example_files.load(file='EddyProFullOutput_30T', ctx=self.ctx)
# self.initial_data, self.settings_dict = example_files.load(file='FluxnetFullSet_30T', ctx=self.ctx) # self.initial_data, self.settings_dict = example_files.load(file='FluxnetFullSet_30T', ctx=self.ctx)
# self.initial_data, self.settings_dict = example_files.load(file='HiRes_20Hz', ctx=self.ctx) # self.initial_data, self.settings_dict = example_files.load(file='HiRes_20Hz', ctx=self.ctx)
# self.initial_data, self.settings_dict = example_files.load(file='IfDews_1T', ctx=self.ctx) # self.initial_data, self.settings_dict = example_files.load(file='IfDews_1T', ctx=self.ctx)
...@@ -158,7 +158,7 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo ...@@ -158,7 +158,7 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo
if self.load_initial_data: if self.load_initial_data:
self.select_source(action='load_initial_data') self.select_source(action='load_initial_data')
# self.plot_focus(fig=-9999, plot_type='OVERVIEW') # self.plot_focus(fig=-9999, plot_type='OVERVIEW')
# self.make_tab(make_tab='MDS') self.make_tab(make_tab='MDS')
# self.make_tab(make_tab='USTAR_DETECTION') # e.g., shortcut to ustar detection # self.make_tab(make_tab='USTAR_DETECTION') # e.g., shortcut to ustar detection
# self.make_tab(make_tab='AGGREGATOR') # self.make_tab(make_tab='AGGREGATOR')
# self.make_tab(make_tab='SCATTER') # self.make_tab(make_tab='SCATTER')
...@@ -284,7 +284,8 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo ...@@ -284,7 +284,8 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo
except ValueError: ## which means no overlapping column names except ValueError: ## which means no overlapping column names
self.data_df = self.data_df.join(incoming_data_df, how='outer') self.data_df = self.data_df.join(incoming_data_df, how='outer')
self.data_df = data_fn.sort_multiindex_columns_names(df=self.data_df, priority_vars=FLUXES) self.data_df = data_fn.sort_multiindex_columns_names(df=self.data_df,
priority_vars=FLUXES + GAPFILLED_GENERAL_SCALARS)
self.update_gui_lists() self.update_gui_lists()
def get_source_files_list(self, action): def get_source_files_list(self, action):
...@@ -392,9 +393,9 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo ...@@ -392,9 +393,9 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo
'Error': e, 'Error': e,
'Recommendation': 'Check if File Settings are correct for this file.'}, highlight=False) 'Recommendation': 'Check if File Settings are correct for this file.'}, highlight=False)
print("\n(!)SKIPPED FILES") # print("\n(!)SKIPPED FILES")
print(skipped_files_list) # print(skipped_files_list)
print("\n") # print("\n")
return all_data_df return all_data_df
...@@ -506,7 +507,7 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo ...@@ -506,7 +507,7 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo
# # OUTLIER DETECTION: Double-Differenced Time Series # # OUTLIER DETECTION: Double-Differenced Time Series
# if colname_tuple[0].endswith('daytime'): # if colname_tuple[0].endswith('daytime'):
# self.drp_od_ref_DoubleDiff_defineNighttimeDataAs.addItem(colname_pretty_string) # self.drp_od_ref_DoubleDiff_defineNighttimeDataAs.addItem(col_list_pretty)
# QUALITY CONTROL # QUALITY CONTROL
# -------------------------------------- # --------------------------------------
...@@ -808,7 +809,7 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo ...@@ -808,7 +809,7 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo
self.make_tab_ustar() self.make_tab_ustar()
elif make_tab == 'MDS': elif make_tab == 'MDS':
self.make_tab_mds() self.mds_make_tab()
elif make_tab == 'AGGREGATOR': elif make_tab == 'AGGREGATOR':
self.make_tab_aggregator() self.make_tab_aggregator()
...@@ -1017,68 +1018,48 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo ...@@ -1017,68 +1018,48 @@ class Amp(QtWidgets.QMainWindow, Ui_MainWindow, GUI_BuildingBlocks.SettingsWindo
break break
return tab_exists, nt return tab_exists, nt
def make_tab_mds(self): def mds_make_tab(self):
"""Make new tab and store current data_df in instance."""
tab_exists, tab_number = self.check_if_tab_exists(tab_name='MDS') tab_exists, tab_number = self.check_if_tab_exists(tab_name='MDS')
# In case the tab already exists, activate the existing tab, otherwise make new tab # In case the tab already exists, activate the existing tab, otherwise make new tab
if tab_exists: if tab_exists:
self.TabWidget.setCurrentIndex(tab_number) self.TabWidget.setCurrentIndex(tab_number)
# pass
else: else:
# Create tab # Create tab
self.drp_gf_MDS_flux, self.drp_gf_MDS_swin, self.drp_gf_MDS_ta, self.drp_gf_MDS_vpd, \ self.MDS_Tab = gf_MDS.MakeNewTab(TabWidget=self.TabWidget,
self.lne_gf_MDS_swin_sim, self.lne_gf_MDS_ta_sim, self.lne_gf_MDS_vpd_sim, \ col_dict_tuples=self.selected_file_colnames_tuples_dict,
self.drp_gf_MDS_templateReference, self.btn_gf_MDS_start_qual_A, self.btn_gf_MDS_start_qual_B, \ col_list_pretty=self.selected_file_colnames_pretty_strings_list,
self.btn_gf_MDS_start_qual_C, self.lne_gf_MDS_time_win_days, \ data_df=self.data_df)
self.lne_gf_MDS_time_win_hours, self.fig_gf_MDS, self.axes_dict_gf_MDS, \
self.gf_MDS_progress_bar, self.btn_gf_MDS_init = \ self.MDS_Tab.btn_init.clicked.connect(self.mds_init_data)
gf_MDS.MakeNewTab(TabWidget=self.TabWidget,
colname_pretty_string=self.selected_file_colnames_pretty_strings_list,
data_df=self.data_df).get_handles()
self.btn_gf_MDS_init.clicked.connect(self.mds_init_data)
def mds_init_data(self): def mds_init_data(self):
# Create object """Prepare data for MDS gap-filling."""
self.obj_mds = gf_MDS.StartMDSgapfilling(col_dict_tuples=self.selected_file_colnames_tuples_dict, self.obj_mds = gf_MDS.RunMDS(mds_tab=self.MDS_Tab)
data_df=self.data_df,
fig=self.fig_gf_MDS,
flux_data_col=self.drp_gf_MDS_flux.currentText(),
ta_data_col=self.drp_gf_MDS_ta.currentText(),
swin_data_col=self.drp_gf_MDS_swin.currentText(),
vpd_data_col=self.drp_gf_MDS_vpd.currentText(),
col_list_pretty=self.selected_file_colnames_pretty_strings_list,
axes_dict=self.axes_dict_gf_MDS,
button_qual_A=self.btn_gf_MDS_start_qual_A,
button_qual_B=self.btn_gf_MDS_start_qual_B,
button_qual_C=self.btn_gf_MDS_start_qual_C,
swin_sim_lim=float(self.lne_gf_MDS_swin_sim.text()),
ta_sim_lim=float(self.lne_gf_MDS_ta_sim.text()),
vpd_sim_lim=float(self.lne_gf_MDS_vpd_sim.text()),
time_win_days=int(self.lne_gf_MDS_time_win_days.text()),
time_win_hours=int(self.lne_gf_MDS_time_win_hours.text()),
progress_bar=self.gf_MDS_progress_bar)
# Start gap-filling when button is pressed # Start gap-filling when button is pressed
self.btn_gf_MDS_start_qual_A.clicked.connect(lambda: self.mds_get_results(which='quality_A')) self.obj_mds.btn_qual_A.clicked.connect(lambda: self.mds_get_results(which='quality_A'))
self.btn_gf_MDS_start_qual_B.clicked.connect(lambda: self.mds_get_results(which='quality_B')) self.obj_mds.btn_qual_B.clicked.connect(lambda: self.mds_get_results(which='quality_B'))
self.btn_gf_MDS_start_qual_C.clicked.connect(lambda: self.mds_get_results(which='quality_C')) self.obj_mds.btn_qual_C.clicked.connect(lambda: self.mds_get_results(which='quality_C'))
self.obj_mds.btn_calc_unc.clicked.connect(lambda: self.mds_get_results(which='uncertainty'))
self.update_gui_lists() self.update_gui_lists()
def mds_get_results(self, which): def mds_get_results(self, which):
""" Perform gap-filling and put gap-filled values into the main df """Perform gap-filling and put gap-filled values into the main df."""
"""
if which == 'quality_A': if which == 'quality_A':
self.obj_mds.gapfilling_quality_A() self.obj_mds.gapfilling_quality_A()
elif which == 'quality_B': elif which == 'quality_B':
self.obj_mds.gapfilling_quality_B() self.obj_mds.gapfilling_quality_B()
elif which == 'quality_C': elif which == 'quality_C':
self.obj_mds.gapfilling_quality_C() self.obj_mds.gapfilling_quality_C()
elif which == 'uncertainty':
self.obj_mds.calculate_uncertainty()
self.data_df = self.obj_mds.get_filled() self.data_df = self.obj_mds.get_filled(add_to_df=self.data_df) # Add results to main data_df
self.data_df = data_fn.sort_multiindex_columns_names(df=self.data_df,
self.data_df = data_fn.sort_multiindex_columns_names(df=self.data_df, priority_vars=FLUXES) priority_vars=FLUXES + GAPFILLED_GENERAL_SCALARS)
self.update_gui_lists() self.update_gui_lists()
def make_tab_ustar(self): def make_tab_ustar(self):
...@@ -1548,7 +1529,9 @@ class AppContext(ApplicationContext): # 1. Subclass ApplicationContext ...@@ -1548,7 +1529,9 @@ class AppContext(ApplicationContext): # 1. Subclass ApplicationContext
# Example files # Example files
@cached_property @cached_property
def file_AmpCSV_30T(self): def file_AmpCSV_30T(self):
return self.get_resource('example_files/AmpCSV_30T.csv') # Returns absolute path return self.get_resource(
'example_files/CH-OE2_Dataset_AMP-20191025-145003_Original-30T.csv') # Returns absolute path
# return self.get_resource('example_files/AmpCSV_30T.csv') # Returns absolute path
@cached_property @cached_property
def file_EddyProFullOutput_30T(self): def file_EddyProFullOutput_30T(self):
......
...@@ -584,7 +584,7 @@ class MakeUstarAnalysis: ...@@ -584,7 +584,7 @@ class MakeUstarAnalysis:
bts_results_sorted = bts_results bts_results_sorted = bts_results
overall_ustar_threshold = bts_results_sorted[0] overall_ustar_threshold = bts_results_sorted[0]
self.data_df=self.make_insert_qc_flag(overall_ustar_threshold=overall_ustar_threshold, df=self.data_df) self.data_df = self.make_insert_qc_flag(overall_ustar_threshold=overall_ustar_threshold, df=self.data_df)
GUI_Elements.btn_txt_live_update(btn=self.button, txt=self.button_orig_text, perc=-9999) GUI_Elements.btn_txt_live_update(btn=self.button, txt=self.button_orig_text, perc=-9999)
...@@ -1055,7 +1055,7 @@ class MakeUstarAnalysis: ...@@ -1055,7 +1055,7 @@ class MakeUstarAnalysis:
ax.plot_date(x=class_group.index, y=class_group[data_col], c=class_colors_dict[class_key], alpha=0.5, ax.plot_date(x=class_group.index, y=class_group[data_col], c=class_colors_dict[class_key], alpha=0.5,
markeredgecolor='none', marker='o') markeredgecolor='none', marker='o')
Format(ax=ax, label_color='#eab839', xlabel='-', ylabel='-', ylabel_units='-', fontsize='large') Format(ax=ax, txt_xlabel='-', txt_ylabel='-', txt_ylabel_units='-', label_color='#eab839', fontsize='large')