Commit 364edfdc authored by holukas's avatar holukas
Browse files

Initial commit

parents
# Custom
_archive/
_raw_photoshop/
_example_data/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
This diff is collapsed.
name: FQC1
channels:
- defaults
dependencies:
- blas=1.0=mkl
- ca-certificates=2020.1.1=0
- certifi=2020.4.5.1=py36_0
- cycler=0.10.0=py36h009560c_0
- freetype=2.9.1=ha9979f8_1
- icc_rt=2019.0.0=h0cc432a_1
- icu=58.2=ha925a31_3
- intel-openmp=2020.1=216
- jpeg=9b=hb83a4c4_2
- kiwisolver=1.2.0=py36h74a9793_0
- libpng=1.6.37=h2a8f88b_0
- matplotlib=3.1.3=py36_0
- matplotlib-base=3.1.3=py36h64f37c6_0
- mkl=2020.1=216
- mkl-service=2.3.0=py36hb782905_0
- mkl_fft=1.0.15=py36h14836fe_0
- mkl_random=1.1.0=py36h675688f_0
- numpy=1.18.1=py36h93ca92e_0
- numpy-base=1.18.1=py36hc3f5095_1
- openssl=1.1.1g=he774522_0
- pandas=1.0.3=py36h47e9c7a_0
- pip=20.0.2=py36_3
- pyparsing=2.4.7=py_0
- pyqt=5.9.2=py36h6538335_2
- python=3.6.10=h9f7ef89_2
- python-dateutil=2.8.1=py_0
- pytz=2020.1=py_0
- qt=5.9.7=vc14h73c81de_0
- setuptools=46.4.0=py36_0
- sip=4.19.8=py36h6538335_0
- six=1.14.0=py36_0
- sqlite=3.31.1=h2a8f88b_1
- tornado=6.0.4=py36he774522_1
- vc=14.1=h0510ff6_4
- vs2015_runtime=14.16.27012=hf0eaf9b_2
- wheel=0.34.2=py36_0
- wincertstore=0.2=py36h7fe50ca_0
- zlib=1.2.11=h62dcd97_4
prefix: C:\Users\holukas\Anaconda3\envs\FQC1
This diff is collapsed.
from tkinter import *
import webbrowser
def main_window(site_list, IRGA_list):
def link_callback(event):
webbrowser.open_new(r"http://envsupport.licor.com/help/EddyPro3/Content/Topics/Despiking_Raw_Stat_Screening.htm")
master = Tk()
master.wm_title("FQC - ETH Flux Quality Control - QC_FINAL_FLAG")
site_to_process = StringVar()
select_IRGA = StringVar()
eddypro_qc = IntVar()
minimum_threshold = IntVar()
maximum_threshold = IntVar()
window_dirtiness = IntVar()
spikes_hf = IntVar()
amplitude_resolution_hf = IntVar()
drop_out_hf = IntVar()
absolute_limits_hf = IntVar()
skewness_kurtosis_hf = IntVar()
discontinuities_hf = IntVar()
timelag_hf = IntVar()
timelag_sf = IntVar()
attack_angle_hf = IntVar()
non_steady_wind_hf = IntVar()
agc_threshold = IntVar()
agc_setting = StringVar()
flag_H = IntVar()
fill_gaps = IntVar()
ustar_hf = IntVar()
# set default values
site_to_process.set('Please select site ...')
select_IRGA.set('Please select used IRGA ...')
eddypro_qc.set(1)
minimum_threshold.set(1)
maximum_threshold.set(1)
window_dirtiness.set(1)
spikes_hf.set(1)
amplitude_resolution_hf.set(0)
drop_out_hf.set(1)
absolute_limits_hf.set(1)
skewness_kurtosis_hf.set(0)
discontinuities_hf.set(0)
timelag_hf.set(0)
timelag_sf.set(0)
attack_angle_hf.set(0)
non_steady_wind_hf.set(0)
agc_threshold.set(80)
agc_setting.set('flag AGC values above')
flag_H.set(1)
fill_gaps.set(1)
ustar_hf.set(0)
master.geometry('1500x706+150+200')
Label(master, justify=RIGHT, text="CHOOSE SITE")\
.grid(column=0, row=0, sticky=W)
OptionMenu(master, site_to_process, *site_list)\
.grid(column=1, row=0, sticky=W)
Label(master, justify=RIGHT, text="CHOOSE IRGA")\
.grid(column=0, row=1, sticky=W)
OptionMenu(master, select_IRGA, *IRGA_list)\
.grid(column=1, row=1, sticky=W)
Label(master, text="Please select quality tests:")\
.grid(column=0, row=2, sticky=W)
Checkbutton(master, text="STEADY STATE & TURBULENCE", variable=eddypro_qc)\
.grid(column=1, row=2, sticky=W)
Label(master, text="(eddypro_qc) SSITC: steady state test and developed turbulent conditions test (see Foken et al., 2004; Foken and Wichura, 1996; Goeckede et al., 2008)")\
.grid(column=2, row=2, sticky=W)
Checkbutton(master, text="MINIMUM FLUX", variable=minimum_threshold)\
.grid(column=1, row=3, sticky=W)
Label(master, text="(minimum_threshold) acceptable minimum flux values")\
.grid(column=2, row=3, sticky=W)
Checkbutton(master, text="MAXIMUM FLUX", variable=maximum_threshold)\
.grid(column=1, row=4, sticky=W)
Label(master, text="(maximum_threshold) acceptable maximum flux values")\
.grid(column=2, row=4, sticky=W)
Checkbutton(master, text="AGC", variable=window_dirtiness)\
.grid(column=1, row=5, sticky=W)
Label(master, text="(raw data screening) (window_dirtiness) automatic gain control threshold, signal strength")\
.grid(column=2, row=5, sticky=W)
agc_setting_list = ('flag AGC values above', 'flag AGC values below')
OptionMenu(master, agc_setting, *agc_setting_list)\
.grid(column=2, row=5, sticky=E)
Entry(master, textvariable=agc_threshold)\
.grid(column=3, row=5, sticky=W)
Checkbutton(master, text="SPIKE TEST", variable=spikes_hf)\
.grid(column=1, row=6, sticky=W)
Label(master, text="(raw data screening) (spikes_hf)")\
.grid(column=2, row=6, sticky=W)
Checkbutton(master, text="AMPLITUDE RESOLUTION", variable=amplitude_resolution_hf)\
.grid(column=1, row=7, sticky=W)
Label(master, text="(raw data screening) (amplitude_resolution_hf)")\
.grid(column=2, row=7, sticky=W)
Checkbutton(master, text="DROP-OUTS", variable=drop_out_hf)\
.grid(column=1, row=8, sticky=W)
Label(master, text="(raw data screening) (drop_out_hf)")\
.grid(column=2, row=8, sticky=W)
Checkbutton(master, text="ABSOLUTE LIMITS", variable=absolute_limits_hf)\
.grid(column=1, row=9, sticky=W)
Label(master, text="(raw data screening) (absolute_limits_hf) for input data wind, sonic temperature, concentration values")\
.grid(column=2, row=9, sticky=W)
Checkbutton(master, text="SKEWNESS & KURTOSIS", variable=skewness_kurtosis_hf)\
.grid(column=1, row=10, sticky=W)
Label(master, text="(raw data screening) (skewness_kurtosis_hf)")\
.grid(column=2, row=10, sticky=W)
Checkbutton(master, text="DISCONTINUITIES", variable=discontinuities_hf)\
.grid(column=1, row=11, sticky=W)
Label(master, text="(raw data screening) (discontinuities_hf) automatic gain control threshold, signal strength")\
.grid(column=2, row=11, sticky=W)
Checkbutton(master, text="TIME LAGS HARD FLAG", variable=timelag_hf)\
.grid(column=1, row=12, sticky=W)
Label(master, text="(timelag_hf)")\
.grid(column=2, row=12, sticky=W)
Checkbutton(master, text="TIME LAGS SOFT FLAG", variable=timelag_sf)\
.grid(column=1, row=13, sticky=W)
Label(master, text="(timelag_sf) automatic gain control threshold, signal strength")\
.grid(column=2, row=13, sticky=W)
Checkbutton(master, text="ANGLE OF ATTACK", variable=attack_angle_hf)\
.grid(column=1, row=14, sticky=W)
Label(master, text="(raw data screening) (attack_angle_hf)")\
.grid(column=2, row=14, sticky=W)
Checkbutton(master, text="STEADINESS OF HORIZONTAL WIND", variable=non_steady_wind_hf)\
.grid(column=1, row=15, sticky=W)
Label(master, text="(raw data screening) (non_steady_wind_hf)")\
.grid(column=2, row=15, sticky=W)
# Checkbutton(master, text="SENSIBLE HEAT FLAG", variable=flag_H).grid(column=1, row=16, sticky=W)
# Label(master, text="(flag_H) set flag for H to 2 when all CO2, H2O and LE fluxes are flagged with 2").grid(column=2, row=16, sticky=W)
Checkbutton(master, text="FILL GAPS", variable=fill_gaps)\
.grid(column=1, row=17, sticky=W) # todo
Label(master, text="fill missing values between start and end date with -9999")\
.grid(column=2, row=17, sticky=W)
Checkbutton(master, text="USTAR FILTER", variable=ustar_hf)\
.grid(column=1, row=18, sticky=W)
Label(master, text="set flag to 2 if u* is below site-specific limit")\
.grid(column=2, row=18, sticky=W)
lbl_google_link = Label(master, text="Help", fg="Blue", cursor="hand2")
lbl_google_link\
.grid(column=2, row=99, sticky=E)
lbl_google_link.bind("<Button-1>", link_callback)
Button(master, text='Run', command=master.quit)\
.grid(column=1, row=99, sticky=W, padx=10, pady=10)
# Button(master, text='Exit', command=master.quit).grid(column=2, row=99, sticky=W, pady=4)
master.mainloop()
master.destroy()
site_to_process = site_to_process.get()
select_IRGA = select_IRGA.get()
eddypro_qc = eddypro_qc.get()
minimum_threshold = minimum_threshold.get()
maximum_threshold = maximum_threshold.get()
window_dirtiness = window_dirtiness.get()
spikes_hf = spikes_hf.get()
amplitude_resolution_hf = amplitude_resolution_hf.get()
drop_out_hf = drop_out_hf.get()
absolute_limits_hf = absolute_limits_hf.get()
skewness_kurtosis_hf = skewness_kurtosis_hf.get()
discontinuities_hf = discontinuities_hf.get()
timelag_hf = timelag_hf.get()
timelag_sf = timelag_sf.get()
attack_angle_hf = attack_angle_hf.get()
non_steady_wind_hf = non_steady_wind_hf.get()
agc_threshold = agc_threshold.get()
agc_setting = agc_setting.get()
# flag_H = flag_H.get()
fill_gaps = fill_gaps.get()
ustar_hf = ustar_hf.get()
flag_list_of_all_selected = [] # list that will contain the names of all selected flags
# STANDARD FLAGS ----------------------------------------
if eddypro_qc == 1:
flag_list_of_all_selected.append('eddypro_qc')
if minimum_threshold == 1:
flag_list_of_all_selected.append('minimum_threshold')
if maximum_threshold == 1:
flag_list_of_all_selected.append('maximum_threshold')
if window_dirtiness == 1:
flag_list_of_all_selected.append('window_dirtiness')
if ustar_hf == 1:
flag_list_of_all_selected.append('ustar')
# STATISTICAL FLAGS -------------------------------------
flag_list_9_digits = []
if spikes_hf == 1:
flag_list_9_digits.append('spikes_hf')
flag_list_of_all_selected.append('spikes_hf')
if amplitude_resolution_hf == 1:
flag_list_9_digits.append('amplitude_resolution_hf')
flag_list_of_all_selected.append('amplitude_resolution_hf')
if drop_out_hf == 1:
flag_list_9_digits.append('drop_out_hf')
flag_list_of_all_selected.append('drop_out_hf')
if absolute_limits_hf == 1:
flag_list_9_digits.append('absolute_limits_hf')
flag_list_of_all_selected.append('absolute_limits_hf')
if skewness_kurtosis_hf == 1:
flag_list_9_digits.append('skewness_kurtosis_hf')
flag_list_of_all_selected.append('skewness_kurtosis_hf')
if discontinuities_hf == 1:
flag_list_9_digits.append('discontinuities_hf')
flag_list_of_all_selected.append('discontinuities_hf')
flag_list_5_digits = []
if timelag_hf == 1:
flag_list_5_digits.append('timelag_hf')
flag_list_of_all_selected.append('timelag_hf')
if timelag_sf == 1:
flag_list_5_digits.append('timelag_sf')
flag_list_of_all_selected.append('timelag_sf')
flag_list_2_digits = []
if attack_angle_hf == 1:
flag_list_2_digits.append('attack_angle_hf')
flag_list_of_all_selected.append('attack_angle_hf')
if non_steady_wind_hf == 1:
flag_list_2_digits.append('non_steady_wind_hf')
flag_list_of_all_selected.append('non_steady_wind_hf')
# (EddyPro qc_ is searched automatically)
return eddypro_qc, minimum_threshold, maximum_threshold, window_dirtiness, agc_setting, agc_threshold, ustar_hf,\
site_to_process, flag_list_9_digits, flag_list_5_digits, flag_list_2_digits, select_IRGA
import matplotlib as mpl
# mpl.use('Agg') # when calling savefig() instead of show() the pop-up window won’t appear
import os
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from matplotlib import dates
from matplotlib.ticker import MultipleLocator
import datetime as dt
# from pandas.stats.moments import *
import pandas
import sys
def plot_diurnal_cycles(data_name, data_values, data_units, destination_folder):
# assemble PNG output
print("Now plotting daily average cycles...")
print("working on: " + data_name)
plot_name = data_name
# plot_name = plot_name.replace('*', 'star')
# plot_name = plot_name.replace('/', '_over_')
y = data_values
fig = plt.figure(figsize=(11.69, 6.58), dpi=150)
plt.title(plot_name)
y = y.replace(-9999, np.nan) # general missing value
y = y.replace(-6999, np.nan) # special missing value used for some scalars, e.g. ch4
y = y.dropna()
if not y.empty:
heading_size = 12
subheading_size = 10
label_size = 7
text_size = 7
# let's see which months we have in our time series
y_found_months = y.index.month
y_found_months = np.unique(y_found_months)
print("...found data for months: " + str(y_found_months))
ax = plt.subplot2grid((4, 3), (0, 0))
# not let's cycle through all found months
plot_counter = 0
for month_cycler in range(0, len(y_found_months)):
print("...calculating month " + str(month_cycler + 1))
plot_counter += 1
y_one_month = y[y.index.month == y_found_months[month_cycler]] # loads y data only from current month
if 1 <= month_cycler + 1 <= 3:
plot_row = 0
plot_column = month_cycler
elif 4 <= month_cycler + 1 <= 6:
plot_row = 1
plot_column = month_cycler - 3
elif 7 <= month_cycler + 1 <= 9:
plot_row = 2
plot_column = month_cycler - 6
elif 10 <= month_cycler + 1 <= 12:
plot_row = 3
plot_column = month_cycler - 9
else:
input("Month not valid. Please check file. Stopping program. Sorry.")
sys.exit()
# hourly average
ax1 = plt.subplot2grid((4, 3), (plot_row, plot_column), colspan=1, rowspan=1)
hourly_avg = y_one_month.groupby(y_one_month.index.hour).mean()
hourly_std = y_one_month.groupby(y_one_month.index.hour).std()
hourly_avg.plot()
plt.fill_between(hourly_avg.index, hourly_avg - hourly_std, hourly_avg + hourly_std, color='k',
alpha=0.1)
ax1.set_xlabel("hour", size=label_size)
ax1.set_ylabel(data_units, size=label_size)
ax1.text(0.06, 0.85, str(y_found_months[month_cycler]), horizontalalignment='center',
verticalalignment='center', transform=ax1.transAxes, backgroundcolor='#ffc532',
size=subheading_size)
if plot_counter == 1:
ax1.text(0.05, 1.1, data_name, horizontalalignment='left',
verticalalignment='baseline', transform=ax1.transAxes, backgroundcolor='#b1d32f',
size=heading_size, )
ax1.tick_params(axis='both', labelsize=label_size)
plt.setp(ax1.xaxis.get_majorticklabels(), rotation=0)
plt.axhline(0, color='black', alpha=0.8)
plt.xlim(0, 23)
if hourly_avg.min() < 0:
factor = 1.2
else:
factor = 0.8
plt.ylim(hourly_avg.min() * factor, hourly_avg.max() * 1.2)
majorLocator = MultipleLocator(2)
ax1.xaxis.set_major_locator(majorLocator)
# # text info output
# fig.text(0.8, 0.05, y.describe(), size=text_size, backgroundcolor='#CCCCCC')
# if quality_controlled == 1:
# plt.figtext(0.84, 0.97, "quality controlled using " + qc_string, color='red', size=text_size)
# plt.show()
# plt.tight_layout()
# fig.subplots_adjust(hspace=0.1)
print("...saving plots to PNG image...")
plot_name = os.path.join(destination_folder, plot_name)
plt.savefig(plot_name + '_daily_average_cycles.png', dpi=150)
plt.close()
else:
print("Data for " + data_name + " is empty --> no plot")
\ No newline at end of file
# release notes:
# 2020-05-24: version 2.1.0
# * created conda environment
# * changed .scatter plotting (did not work anymore) to .plot_date
# 2019-06-02: version 2.0.0
# * implemented the new *_fluxnet_* files that replace the *_ghg-europe_* files in EddyPro 7
# * changed output folder to better match the folder naming convention: OUT_QC-YYYYMMDD-HHMMSS
# 2019-03-04: version 1.04
# * signal strength / status byte / window dirtiness is now also recognized by the string "signal_strength"
# 2019-02-25: version 1.03
# * ignoring empty row for merged files is no longer necessary, pandas now outputs without that empty row
# 2019-02-25: version 1.02
# * date format of ghg-europe file is now in separate variable dateformat_ghg_europe_file
# 2019-02-24:
# * now called version 1.01
# * changed output ID to different format, e.g. QC-20190224-1725
# * now_string is now called id_string
# * added id_string to output csv files
# 2017-03-07: changed "mangle_dupe_cols" to "True" in line
# GHG_units = pd.read_csv(GHG_found_file, skiprows=GHG_skiprows_units, nrows=1, mangle_dupe_cols=True)
# 2017-01-31: rewrote the logic behind finding the correct AGC variable
# added error message if no site or IRGA was selected
# 2016-07-13: removed bug that wrote the wrong units in diurnal cycle plots
# 2015-09-22: flag in ghg-europe file is not changed, raw data screening sets flagged values to -9999
# SSITC flag in ghg-europe must not be changed for database upload
# instead values that fail the raw data screening are set to -9999 (in accordance w/ database submission guidelines)
# 2015-08-24: update to version 0.2: added support for EddyPro 6 output files
# removed H-flag, not necessary anymore, bug in EP was removed in version 6.0.0
# the overall QC flag is not output in a separate column in the EP6 GHG-Europe output file
# 2015-05-06: the flag for originally missing values is now left at -9999 instead of setting it to 2
# from EddyPro 5.2 ghg-europe file contains "ISOdate" as date column, in addition missing dates are not filled anymore
# please specify working directory
# working_directory = r'M:\Dropbox\luhk_work\programming\python\FQC_FluxQualityControl'
import os
# os.chdir(working_directory)
# automatic detection of working directory, test if running on RDS
# automatically detect folder where main.py resides
# currently not working on RDS, but works on local computer
abspath = os.path.abspath(__file__)
working_directory = os.path.dirname(abspath)
os.chdir(working_directory)
from func import please_quality_control
please_quality_control(working_directory)
certifi==2020.4.5.1
cycler==0.10.0
kiwisolver==1.2.0
matplotlib==3.1.3
mkl-fft==1.0.15
mkl-random==1.1.0
mkl-service==2.3.0
numpy==1.18.1
pandas==1.0.3
pyparsing==2.4.7
python-dateutil==2.8.1
pytz==2020.1
six==1.14.0
tornado==6.0.4
wincertstore==0.2
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment