To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit c6809a32 authored by holukas's avatar holukas
Browse files

v0.12.0 multi-panel plots and middle timestamp

parent e121718c
This diff is collapsed.
......@@ -40,18 +40,18 @@
### Then
1. add site IDs or similar to stats and output files etc
2. **I/O:** solve full timestamp issue with missing or available seconds, auto-check which timestamp format works, starting with the most likely format
3. by default, all vars should be listed in all dropdowns, and the best guess based on global vars should be selected by default, or ix 0 if no global var found
4. generate “infocards” e.g. during Pipelines
5. BUG rejectval_col.sum is off by 2 in EP statistical raw data flags info text in plot
6. Amp QC flags analysis board / tab
7. install miniconda on Windows Subsystem for Linux to generate Linux executables
8. **HiRes Tests:** stationarity test after Mahrt 1998
9. **Analyses:NetGHGBudget:** refine calculations
10. **Analyses:NetGHGBudget:** better unit conversion, detect from units; GWP60
11. **Analyses:NetGHGBudget:** include albedo effect
12. automatic GHG budgets CO2 + N2O + CH4
1. add site IDs or similar to stats and output files etc **X**
2. **I/O:** solve full timestamp issue with missing or available seconds, auto-check which timestamp format works, starting with the most likely format **X**
3. by default, all vars should be listed in all dropdowns, and the best guess based on global vars should be selected by default, or ix 0 if no global var found **X**
4. generate “infocards” e.g. during Pipelines **X**
5. BUG rejectval_col.sum is off by 2 in EP statistical raw data flags info text in plot **X**
6. Amp QC flags analysis board / tab **X**
7. install miniconda on Windows Subsystem for Linux to generate Linux executables **X**
8. HiRes Tests: stationarity test after Mahrt 1998 **X**
9. Analyses:NetGHGBudget: refine calculations
10. Analyses:NetGHGBudget: better unit conversion, detect from units; GWP60 **X**
11. Analyses:NetGHGBudget: include albedo effect **X**
12. automatic GHG budgets CO2 + N2O + CH4 **X**
13. **Analyses:Aggregator:** expand functionality; analyses for extreme events e.g. high temperatures and check potential impacts, e.g. w/ significances, use results for auto-events generation? compare e.g. same month of other years to current year etc
14. **Analyses:Aggregator:** make it like NBA season finder with more filter options
15. plot multiple variables (below?) each other, maybe in some sort of plot generator (drag&drop? multi-select from list?)
......
......@@ -188,3 +188,202 @@ Kept here so it can be found with search function.
# # elif self.focus_data_setting == 'Add Panel': # todo
# # pass
# heatmap_df = pd.DataFrame(index=plot_df['TIMESTAMP_shifted'],
# columns=['z'],
# data=plot_df[self.focus_col].values)
# def mlr_ols_with_ln(site, ax_main, dataframe, column_names, color, plot_legend, ax_legend,
# main_plot, plot_mgmt, mgmt_data, main_ticks, win_size, min_days, win_step, use_perc,
# color_snow, color_fert, color_cut, color_significant_r2, color_r2, color_regression_period,
# color_growing_season):
# # log transform
# dataframe['DEPENDENT_SHIFTED'] = dataframe[column_names[0]] + abs(dataframe[column_names[0]].min()) + 1
# dataframe['DEPENDENT_LN'] = np.log(dataframe['DEPENDENT_SHIFTED'])
# subset = pd.DataFrame(index=dataframe.index)
# subset['DEPENDENT_LN'] = dataframe['DEPENDENT_LN']
# for x in range(1, len(column_names)):
# subset[column_names[x]] = dataframe[column_names[x]]
# subset[subset['DEPENDENT_LN'] > subset['DEPENDENT_LN'].quantile(q=0.99)] = np.nan
# # dataframe = dataframe.dropna(subset=['DEPENDENT_LN'])
# formula_string = "DEPENDENT_LN ~ " + column_names[1] # as function of first independent variable
# more_than_one_regressor = False
# for x in range(2, len(column_names)):
# formula_string += " + " + column_names[x]
# more_than_one_regressor = True
# results = smf.ols(formula_string, data=subset).fit() # -1 removes the intercept
# print(results.summary())
# # predicted = results.predict()
# coef_ls = []
# r2 = []
# pvalues = []
# this_from = []
# this_to = []
# this_middle = []
# best_r2 = 0
#
# for i in range(0, len(subset) - win_size, win_step):
#
# # print(len(subset.ix[i:i + win_size]))
# if len(subset.ix[i:i + win_size].dropna()) > min_days: # (win_size * 0.3): # min values
# results = smf.ols(formula_string, data=subset.ix[i:i + win_size]).fit()
# this_from.append(subset.index[i])
# this_to.append(subset.index[i + win_size])
# this_middle.append(subset.index[int(i + (win_size / 2))]) # index needs to be integer
# this_params = results.params
# this_r2 = results.rsquared
# coef_ls.append(this_params)
# r2.append(this_r2)
# if more_than_one_regressor == True:
# for p in results.pvalues:
# if p < 0.05:
# significant = True
# else:
# significant = False
#
# if significant == True:
# pvalues.append(1)
# else:
# pvalues.append(0)
# # print("from " + str(subset.index[i]) + " to " + str(subset.index[i + win_size]) + ": r2 = " + str(# this_r2) + "(" + str(len(subset.ix[i:i + win_size].dropna())) + " days)")
# if this_r2 > best_r2: # plot best fit in chosen window, plotted is only ONE parameter
# best_r2 = this_r2
# best_params = this_params
# best_x = subset[column_names[1]][i:i + win_size]
# best_y = subset['DEPENDENT_LN'][i:i + win_size]
# best_from = subset.index[i]
# best_to = subset.index[i + win_size]
# best_middle_day = subset.index[int(i + (win_size / 2))]
# best_days = str(len(subset.ix[i:i + win_size].dropna()))
# print(" [NEW BEST R2] from " + str(subset.index[i]) + " to " + str(
# subset.index[i + win_size]) + ": r2 = " + str(
# this_r2) + "(" + str(len(subset.ix[i:i + win_size].dropna())) + " days)")
# # else:
# # print(" X from " + str(subset.index[i]) + " to " + str(subset.index[i + win_size]) + ": r2 = not enough days (" + str(
# # len(subset.ix[i:i + win_size].dropna())) + " days)")
#
# # put our results in a dataframe, then we can use .dropna() and fill up empty dates with nan
# this_middle_r2 = pd.DataFrame({'r2': r2}, index=pd.to_datetime(this_middle))
# if more_than_one_regressor:
# this_middle_r2['significant'] = pvalues
# this_middle_r2 = this_middle_r2.dropna()
# filled_date_range = pd.date_range(this_middle_r2.index[0], this_middle_r2.index[-1],
# freq='D') # generate continuous date range and re-index data
# this_middle_r2 = this_middle_r2.reindex(filled_date_range,
# fill_value=np.nan) # apply new continuous index to dataframe1
# print("length r2: " + str(len(r2)))
#
# if main_plot == 1:
# # significant
# ax_main.scatter(this_middle_r2.index[this_middle_r2['significant'] == 1],
# this_middle_r2['r2'][this_middle_r2['significant'] == 1], linewidths=1, s=50, zorder=98,
# c=color_significant_r2, alpha=1, facecolors=color_significant_r2,
# edgecolors=color_significant_r2, label="significant")
#
# # not significant
# ax_main.scatter(this_middle_r2.index[this_middle_r2['significant'] == 0],
# this_middle_r2['r2'][this_middle_r2['significant'] == 0], linewidths=1, s=50, zorder=97,
# c=color_r2, alpha=1, facecolors=color_r2, edgecolors='k', label="not significant")
#
# span = int(win_size / 2)
#
# ax_main.errorbar(this_middle_r2.index, this_middle_r2['r2'], alpha=0.2, xerr=span, capsize=0, ls='none',
# color=color_regression_period, elinewidth=4, label="regression period", zorder=96)
#
# ax_main.text(0.01, 0.84, site, transform=ax_main.transAxes, fontsize=18, ha='left', zorder=99)
#
# # set date format on axis
# import matplotlib.dates as dates
# ax_main.xaxis.set_major_locator(dates.MonthLocator(interval=main_ticks))
# ax_main.xaxis.set_major_formatter(dates.DateFormatter('%m/%Y'))
# ax_main.xaxis.set_minor_locator(dates.MonthLocator(interval=1))
#
# if plot_mgmt:
# plot_mgmt_snow(management_data=mgmt_data, ax=ax_main,
# color_snow=color_snow,
# color_fert=color_fert,
# color_cut=color_cut,
# color_growing_season=color_growing_season)
# # color_cut=sns.xkcd_rgb["steel"]
#
# if plot_legend:
# # sort legend entries manually
# handles, labels = ax_main.get_legend_handles_labels()
# handles = [handles[4], handles[5], handles[3], handles[2], handles[0], handles[1], handles[6]]
# labels = [labels[4], labels[5], labels[3], labels[2], labels[0], labels[1], labels[6]]
# ax_main.legend(handles, labels, loc='upper center', bbox_to_anchor=(0.5, 1.4), ncol=4, prop={'size': 16})
# # ax_main.legend(loc='upper center', bbox_to_anchor=(0.5, 1.4), ncol=3, prop={'size': 16})
# # Get shifted timestamp info, required for correct e.g. month assignment of data
# _plot_df_shifted = DataFunctions.shift_timestamp(df=plot_df, freq=self.freq)
# plot_df['TIMESTAMP_shifted'] = _plot_df_shifted.index
# plot_df['HOUR'] = _plot_df_shifted.index.hour
# # Needs shifted timestamp
# mean_df = pd.DataFrame(index=self.plot_df['TIMESTAMP_shifted'],
# columns=[self.focus_col],
# data=self.plot_df[self.focus_col].values)
# data_df = insert_shifted_index_col(df=data_df, minutes=True, amount=30)
# _df = shift_timestamp(df=_df, freq=orig_freq)
# resampled_df = _df.resample(freq_str, label='left', closed='left')
# filter = (resampled_df.index.year == year) & (resampled_df.index.month == month)
# filter = \
# (_resampled_df_shifted.index.year == year) & (_resampled_df_shifted.index.month == month) \
# if use_shifted else \
# (resampled_df.index.year == year) & (resampled_df.index.month == month)
# if use_shifted:
# filter = (_resampled_df_shifted.index.year == year)
# else:
# filter = (resampled_df.index.year == year)
# tab_exists, tab_number = self.check_if_tab_exists(tab_name='SCATTER PLOT')
# # In case the tab already exists, activate the existing tab, otherwise make new tab
# if tab_exists:
# self.TabWidget.setCurrentIndex(tab_number)
# # pass
# else:
# q = qw.QListWidgetItem(self.col_list_pretty[ix])
# self.lst_Features.addItem(q) # add column name to list
# background_color = QColor()
# background_color.setNamedColor('#f44336')
# q.setBackground(background_color)
# q.setBackground(QtCore.Qt.blue) # works but is overwritten by css file
# todo consider label / closed --> left or right?
# # Timestamp always needs to include AND show the *end time* of row data, therefore
# # the args label='left' and closed='left'.
# # closed='right' means e.g. ( 3:00, 6:00 ] or 3:00 < x <= 6:00
# if to_freq == 'H':
# # When resampling to hourly, show the end point of the resampled interval
# # e.g. averaging mean to 1 hour from half-hourly
# # 2019-08-28 23:30 and 2019-08-29 00:00 --> output as 2019-08-29 00:00
# label = 'right'
# closed = 'right'
#
# elif (to_freq == 'D') | (to_freq == 'M') | (to_freq == 'Y'):
# # e.g. averaging mean to 1 daily average from half-hourly
# # 2019-08-26 00:30 and 2019-08-27 00:00 --> output as 2019-08-26
# # Note that 2019-08-26 00:00 will not be included due to closed='right'.
# label = 'right'
# closed = 'right'
# use_shifted, _resampled_df_shifted = DataFunctions.check_shift_need(resampled_df=resampled_df,
# freq_orig=self.orig_freq,
# freq_to=self.to_freq)
import ast
from pathlib import Path
from help import infoboxes
from PyQt5 import QtWidgets as qw
from gui import gui_building_blocks, gui_elements
......@@ -135,23 +134,26 @@ class SelectFileTypeSettings(gui_building_blocks.SelectFileTypeSettingsFile):
item.setToolTip(info)
self.filelist.addItem(item) # add column name to list
def search_files(self, dir_filetypes):
# dir_filetypes = Path(dir_filetypes) / 'inout' / 'FileTypes'
found_files_dict = search_files(src_dir=dir_filetypes)
return found_files_dict
def gui(self):
# Constructs the gui for the settings_dict window
"""
Construct gui for filetypes window
"""
layout = self.setupUi(self, win_title='File Settings')
left_layout, right_layout = self.setupUi(self, win_title='File Settings')
filelist = gui_elements.lyt_List(layout=layout)
gui_elements.add_label_to_layout(txt='Load File Type Settings', css_id='lbl_Header2', layout=left_layout)
gui_elements.add_infobox_to_layout(txt=infoboxes.filetype_dialog, css_id='infobox', layout=right_layout)
# List of files
filelist = gui_elements.add_list_to_layout(layout=left_layout)
filelist.setWordWrap(True)
filelist.setAlternatingRowColors(True)
layout.addWidget(filelist)
# Button box: OK and Cancel
self.button_box = qw.QDialogButtonBox()
self.button_box.setEnabled(True)
......@@ -160,9 +162,11 @@ class SelectFileTypeSettings(gui_building_blocks.SelectFileTypeSettingsFile):
self.button_box.accepted.connect(self.accept)
self.button_box.rejected.connect(self.reject)
layout.addWidget(self.button_box)
# Assemble
left_layout.addWidget(filelist)
left_layout.addWidget(self.button_box)
return layout, filelist
return left_layout, filelist
# # Header
# label = qw.QLabel('File Settings')
......
......@@ -160,7 +160,7 @@ class Ui_MainWindow(object):
# Search box for variable list
self.lne_FilterVarList = gui_elements.lyt_Lineedit(css_id='search_box', lyt=lyt_VariableList)
self.lst_Variables = gui_elements.lyt_List(layout=lyt_VariableList)
self.lst_Variables = gui_elements.add_list_to_layout(layout=lyt_VariableList)
self.lst_Variables.setWordWrap(True)
self.lst_Variables.setAlternatingRowColors(True)
# self.lst_Variables.setSelectionMode(qw.QListWidget.MultiSelection)
......
......@@ -10,8 +10,7 @@ from PyQt5 import QtWidgets as qw
from PyQt5.QtGui import QIcon
from gui import gui_elements
from pathlib import Path
import pathlib
class SelectFileTypeSettingsFile(qw.QDialog):
def setupUi(self, SelectFileTypeSettingsFile, win_title):
......@@ -19,48 +18,63 @@ class SelectFileTypeSettingsFile(qw.QDialog):
SelectFileTypeSettingsFile.setWindowTitle(win_title)
SelectFileTypeSettingsFile.setWindowIcon(QIcon('gui/Icons/python_icon.png'))
SelectFileTypeSettingsFile.resize(1000, 600)
SelectFileTypeSettingsFile.resize(1200, 800)
SelectFileTypeSettingsFile.setContentsMargins(0, 0, 0, 0)
# Container
layout = qw.QVBoxLayout()
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
self.setLayout(layout)
return layout
class SettingsWindow(qw.QDialog):
def setupUi(self, SettingsWindow, win_title):
SettingsWindow.setAccessibleName('SettingsWindow')
# # Load external CSS file
# dir_start = pathlib.Path(__file__) # Auto-detect where template_main.py is located.
# dir_css = dir_start.parents[3] / 'main' / 'resources' / 'base' # One folder up
# # path_gui = Path('gui')
# with open(dir_css / 'gui_light.css', "r") as fh:
# SettingsWindow.setStyleSheet(fh.read())
# # with open(path_gui / 'gui_light.css', "r") as fh:
# # SettingsWindow.setStyleSheet(fh.read())
SettingsWindow.setWindowTitle(win_title)
SettingsWindow.setWindowIcon(QIcon('gui/Icons/python_icon.png'))
SettingsWindow.resize(1000, 600)
SettingsWindow.setContentsMargins(0, 0, 0, 0)
# CONTAINER (lyt_CategoryOptions)
layout = qw.QGridLayout()
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
self.setLayout(layout)
return layout
# Container layout
container_layout = qw.QHBoxLayout()
container_layout.setContentsMargins(0, 0, 0, 0)
container_layout.setSpacing(0)
self.setLayout(container_layout)
# Left layout
layout_left = qw.QVBoxLayout()
layout_left.setContentsMargins(0, 0, 0, 0)
layout_left.setSpacing(0)
# Right layout
layout_right = qw.QVBoxLayout()
layout_right.setContentsMargins(0, 0, 0, 0)
layout_right.setSpacing(0)
# return SettingsWindow
# self.CentralWidget = qw.QWidget(SettingsWindow)
# self.CentralWidget.setObjectName("centralwidget")
# self.CentralWidget.setAccessibleName('SettingsWindow')
# self.CentralWidget.setContentsMargins(0, 0, 0, 0)
# SettingsWindow.setCentralWidget(self.CentralWidget)
# Assemble
container_layout.addLayout(layout_left, stretch=1)
container_layout.addLayout(layout_right, stretch=2)
return layout_left, layout_right
# class SettingsWindow(qw.QDialog):
# def setupUi(self, SettingsWindow, win_title):
# SettingsWindow.setAccessibleName('SettingsWindow')
#
# # # Load external CSS file
# # dir_start = pathlib.Path(__file__) # Auto-detect where template_main.py is located.
# # dir_css = dir_start.parents[3] / 'main' / 'resources' / 'base' # One folder up
# # # path_gui = Path('gui')
# # with open(dir_css / 'gui_light.css', "r") as fh:
# # SettingsWindow.setStyleSheet(fh.read())
# # # with open(path_gui / 'gui_light.css', "r") as fh:
# # # SettingsWindow.setStyleSheet(fh.read())
#
# SettingsWindow.setWindowTitle(win_title)
# SettingsWindow.setWindowIcon(QIcon('gui/Icons/python_icon.png'))
# SettingsWindow.resize(1000, 600)
# SettingsWindow.setContentsMargins(0, 0, 0, 0)
#
# # CONTAINER (lyt_CategoryOptions)
# layout = qw.QGridLayout()
# layout.setContentsMargins(0, 0, 0, 0)
# layout.setSpacing(0)
# self.setLayout(layout)
# return layout
#
# # return SettingsWindow
# # self.CentralWidget = qw.QWidget(SettingsWindow)
# # self.CentralWidget.setObjectName("centralwidget")
# # self.CentralWidget.setAccessibleName('SettingsWindow')
# # self.CentralWidget.setContentsMargins(0, 0, 0, 0)
# # SettingsWindow.setCentralWidget(self.CentralWidget)
class CategoryOptions:
......@@ -114,6 +128,7 @@ class StretchBox:
layout.setColumnStretch(1, 1)
return frame, layout
class StretchWidget:
def __init__(self):
self.widget = qw.QWidget()
......@@ -122,6 +137,7 @@ class StretchWidget:
def get_widget(self):
return self.widget
class QCustomTabWidget(qw.QTabWidget):
def __init__(self, parent=None):
super(QCustomTabWidget, self).__init__(parent)
......
......@@ -70,13 +70,24 @@ def add_button_to_layout(layout, txt, css_id):
return button
def lyt_Label(txt, css_id, lyt):
def add_label_to_layout(txt, css_id, layout):
lbl = qw.QLabel(text=txt)
lbl.setProperty('labelClass', css_id)
lyt.addWidget(lbl, stretch=0)
layout.addWidget(lbl, stretch=0)
return lbl
def add_infobox_to_layout(txt, css_id, layout):
infobox = qw.QTextEdit()
# infobox.setAcceptRichText(True)
infobox.setReadOnly(True)
infobox.setProperty('labelClass', css_id)
infobox.setText(txt)
# infobox.setHtml(txt)
layout.addWidget(infobox, stretch=0)
return infobox
def lyt_Lineedit(css_id, lyt):
lne_val = qw.QLineEdit()
lne_val.setProperty('labelClass', css_id)
......@@ -168,7 +179,7 @@ def grd_LabelLineeditDropdownTriplet(txt, css_ids, layout, row, col, orientation
return lne_duration, drp_freq
def grd_LabelDropdownPair(txt, css_ids, layout, row, col, orientation):
def grd_LabelDropdownPair(txt, css_ids, layout, row, col, orientation, colspan=1):
# Adds two labels to a grid lyt_CategoryOptions: one for the label text, one for the value.
# Returns the id for the value label so it can later be updated with real values.
lbl_txt = qw.QLabel(text=txt)
......@@ -184,7 +195,7 @@ def grd_LabelDropdownPair(txt, css_ids, layout, row, col, orientation):
# drp_opt_default_epqc.setAlignment(qtc.Qt.AlignLeft | qtc.Qt.AlignVCenter)
layout.addWidget(lbl_txt, row, col, 1, 1)
layout.addWidget(drp_options, row, col + 1, 1, 1)
layout.addWidget(drp_options, row, col + 1, 1, colspan)
# elif orientation == 'vert':
# lbl_txt.setAlignment(qtc.Qt.AlignLeft | qtc.Qt.AlignVCenter)
......@@ -275,17 +286,7 @@ def grd_LabelDropdownPairRowSpan(txt, css_ids, lyt, row, col, orientation, rowsp
return drp_options
def lyt_List(layout):
# list = ListDragEnabled()
list = qw.QListWidget()
list.setDragEnabled(True)
list.setObjectName("list")
list.addItem("-empty-")
layout.addWidget(list)
return list
def add_list_to_grid(grid):
def add_list_to_layout(layout):
# list = ListDragEnabled()
list = qw.QListWidget()
list.setDragEnabled(True)
......@@ -334,10 +335,10 @@ def add_header_to_grid_top(layout, txt):
return None
def add_header_in_grid_row(layout, header, row):
def add_header_in_grid_row(layout, header, row, css_id='lbl_Header2'):
_header = grd_Label(lyt=layout,
txt=header,
css_id='lbl_Header2',
css_id=css_id,
row=row, col=0, rowspan=1, colspan=1)
return None
......
filetype_dialog = """
NOTE
Whenever Amp imports data, it converts the timestamp to the center time of the recorded time period. The entry TIMESTAMP_SHOWS_START_MIDDLE_OR_END_OF_RECORD in the .filetype files defines if the timestamp of the original data marks the start time or the end time of the recorded values. Amp uses this info to convert the timestamp to the center time.
Example
(a) TIMESTAMP_SHOWS_START_MIDDLE_OR_END_OF_RECORD = end
The timestamp 2020-01-22 14:30 is converted to 2020-01-22 14:15
(b) TIMESTAMP_SHOWS_START_MIDDLE_OR_END_OF_RECORD = start
The timestamp 2020-01-22 14:30 is converted to 2020-01-22 14:45
"""
......@@ -123,7 +123,7 @@ def parse_csv_file(filepath, settings_dict):
index_col=None,
dtype=None)
standardize_index(df=data_df)
standardize_index(df=data_df, settings_dict=settings_dict)
# Check if there is only one header row, if yes, then add second row (standardization)
if len(settings_dict['DATA_HEADER_ROWS']) == 1:
......@@ -148,8 +148,6 @@ def read_selected_events_file(index_data_df, filepath, settings_dict):
data is used to convert the info from the Events file into a usable format
that can be merged with the available data.
"""
# todo needs shifted timestamp
# todo consider compression also for these files?
# Default Event columns
event_start_col = ('EVENT_START', '[yyyy-mm-dd]')
......@@ -261,9 +259,9 @@ def read_selected_data_file(filepath, settings_dict):
# os.remove(del_filepath)
# shutil.rmtree(dir_temp_unzipped)
# START END TIMESTAMP
# ------------------- todo start end timestamp col
# data_df = insert_shifted_index_col(df=data_df, minutes=True, amount=30)
# START END TIMESTAMP # todo start end timestamp col?
# -------------------
pass
return data_df
......@@ -278,36 +276,22 @@ def downsample_data(df, freq, max_freq):
return df, new_freq
def standardize_index(df):
def standardize_index(df, settings_dict):
# Index name is now the same for all filetypes w/ timestamp in data
df.set_index([('index', '[parsed]')], inplace=True)
df.index.name = ('TIMESTAMP', '[yyyy-mm-dd HH:MM:SS]')
# Make sure the index is datetime
df.index = pd.to_datetime(df.index)
return df
def insert_shifted_index_col(df, amount=0, seconds=False, minutes=False, hours=False,
shifted_col_shows='start'):
"""Copy index column to new column, shift time in new column by Timedelta.
This is necessary b/c sometimes the timestamp shows the end, sometimes the
start of the measurements. To avoid misunderstandings, the default timestamp
is kept as the index (end of measurement period), and the shifted new column
shows the start of the measurement period.
"""
if shifted_col_shows == 'start':
timestamp_from_col = ('TIMESTAMP_START', '[yyyy-mm-dd HH:MM:SS]')
df.insert(0, timestamp_from_col, df.index)
if hours:
df[timestamp_from_col] = df[timestamp_from_col] - pd.Timedelta(hours=amount)
if minutes:
df[timestamp_from_col] = df[timestamp_from_col] - pd.Timedelta(minutes=amount)
if seconds:
df[timestamp_from_col] = df[timestamp_from_col] - pd.Timedelta(seconds=amount)
# TODO implement if data timestamp shows start
# Shift timestamp by half-frequency, if needed
if settings_dict['TIMESTAMP_SHOWS_START_MIDDLE_OR_END_OF_RECORD'] == 'middle':
pass
else:
timedelta = pd.to_timedelta(settings_dict['DATA_FREQUENCY']) / 2
if settings_dict['TIMESTAMP_SHOWS_START_MIDDLE_OR_END_OF_RECORD'] == 'end':
df.index = df.index - pd.Timedelta(timedelta)
elif settings_dict['TIMESTAMP_SHOWS_START_MIDDLE_OR_END_OF_RECORD'] == 'start':
df.index = df.index + pd.Timedelta(timedelta)