Commit 896f03ac authored by holukas's avatar holukas
Browse files

Smaller changes

parent 2ea75e41
......@@ -2,9 +2,16 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS100-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/file.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/file.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/vis.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/vis.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -43,7 +50,7 @@
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="restartRequiresConfirmation" value="false" />
......@@ -51,9 +58,9 @@
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\sonic_anemometers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\laser_gas_analyzers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\infrared_gas_analyzers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\sonic_anemometers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\header" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\ops" />
</key>
......@@ -280,7 +287,8 @@
<workItem from="1607436447683" duration="6112000" />
<workItem from="1607504715733" duration="6609000" />
<workItem from="1607594405028" duration="3277000" />
<workItem from="1607607175897" duration="2744000" />
<workItem from="1607607175897" duration="3170000" />
<workItem from="1607615147949" duration="11076000" />
</task>
<servers />
</component>
......@@ -313,7 +321,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607597406593" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607641243487" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component>
</project>
\ No newline at end of file
......@@ -98,7 +98,7 @@ class Bico(qtw.QMainWindow, Ui_MainWindow):
# Plot stats collection from file
if self.settings_dict['plot_ts_agg'] == '1':
bin_found_files_dict = file.SearchAll.search_all(dir=self.settings_dict['dir_out_run_plots_agg'],
file_id='__STATS__*.csv',
file_id='stats_*.csv',
logger=self.logger)
for fid, filepath in bin_found_files_dict.items():
df = pd.read_csv(filepath,
......
......@@ -168,7 +168,7 @@ class Ui_MainWindow(object):
'CH-INO', 'CH-LAE', 'CH-LAS', 'CH-OE2'])
# Data Blocks
sonic_anemometers = ['HS50-A', 'R350-A', '-None-']
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA75-A', 'LGR-A', 'QCL-C', '-None-']
header_instr_data_blocks = qtw.QLabel('Data Blocks')
......
......@@ -145,6 +145,7 @@ class SearchAll():
self.logger.info(f"{suffix} +++ {len(valid_files_dict)} files were larger than the selected minimum"
f" filesize of {min_filesize_lim}, keeping files: {list(valid_files_dict.keys())}")
self.logger.info(f"{suffix} --- {len(_invalid_files_dict)} files were smaller than the selected minimum"
f" filesize of {min_filesize_lim} and will not be used: {list(_invalid_files_dict.keys())}")
self.logger.info(f"{suffix} ============================")
return valid_files_dict
......
......@@ -190,23 +190,27 @@ def high_res_histogram(df, outfile, outdir, logger):
col_idx += 1
ax = axes[col_idx]
# ax.plot(dblock_df[col].index, dblock_df[col], 'r,', alpha=0.5, c='#5f87ae')
# import seaborn as sns
# sns.set_context("paper", font_scale=1.1, rc={"lines.linewidth": 1})
# h = sns.histplot(dblock_df[col], ax=ax, kde=False)
ax.hist(dblock_df[col], bins=20)
if not dblock_df[col].dropna().empty:
ax.hist(dblock_df[col], bins=20)
_default_format(ax=ax, width=1, length=2, txt_xlabel="", txt_ylabel="Counts", fontsize=7)
ax.text(0.01, 0.96, f"{col[0]} {col[1]} {col[2]}", transform=ax.transAxes, horizontalalignment='left',
**text_args)
txt_info = f"mean: {dblock_df[col].mean():.3f} vals: {dblock_df[col].count():.0f}\n" \
f"min: {dblock_df[col].min():.3f} max:{dblock_df[col].max():.3f}"
ax.text(0.99, 0.96, txt_info, transform=ax.transAxes, horizontalalignment='right', **text_args)
for tick in ax.xaxis.get_major_ticks():
tick.label.set_fontsize(6)
for tick in ax.yaxis.get_major_ticks():
tick.label.set_fontsize(6)
else:
# If data for col is empty
ax.set_facecolor('xkcd:salmon')
ax.text(0.5, 0.5,
f"(!)WARNING: variable {col} is empty and was therefore not plotted",
horizontalalignment='center', verticalalignment='center', transform=ax.transAxes,
size=14, color='white', backgroundcolor='red')
_default_format(ax=ax, width=1, length=2, txt_xlabel="", txt_ylabel="Counts", fontsize=7)
ax.text(0.01, 0.96, f"{col[0]} {col[1]} {col[2]}", transform=ax.transAxes, horizontalalignment='left',
**text_args)
txt_info = f"mean: {dblock_df[col].mean():.3f} vals: {dblock_df[col].count():.0f}\n" \
f"min: {dblock_df[col].min():.3f} max:{dblock_df[col].max():.3f}"
ax.text(0.99, 0.96, txt_info, transform=ax.transAxes, horizontalalignment='right', **text_args)
for tick in ax.xaxis.get_major_ticks():
tick.label.set_fontsize(6)
for tick in ax.yaxis.get_major_ticks():
tick.label.set_fontsize(6)
dblock_outfile = outdir / f"{outfile}_hires_histogram_{dblock}"
fig.savefig(f"{dblock_outfile}.png", format='png', bbox_inches='tight', facecolor='w',
......
run_id=BICO-20201209-114346
run_id=BICO-20201211-000044
# INSTRUMENTS
# ===========
# Site
site=CH-AES
site=CH-DAV
# Data Blocks
header=WECOM3
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=LGR-A
instrument_1=HS50-A
instrument_2=IRGA72-A
instrument_3=QCL-C
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/12/20_HS_LI72
# Time Range
start_date=2019-12-31 00:00
start_date=2020-01-07 13:00
end_date=2020-12-31 00:00
# File Settings
......@@ -27,7 +27,7 @@ filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=1
row_limit=100
row_limit=10000
# Special
select_random_files=0
......@@ -35,17 +35,17 @@ select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044
dir_out_run_log=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\raw_data_csv
file_compression=gzip
plot_file_availability=1
plot_ts_hires=0
plot_ts_hires=1
plot_histogram_hires=0
plot_ts_agg=1
......
run_id=BICO-20201209-114346
run_id=BICO-20201211-000044
# INSTRUMENTS
# ===========
# Site
site=CH-AES
site=CH-DAV
# Data Blocks
header=WECOM3
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=LGR-A
instrument_1=HS50-B
instrument_2=IRGA72-A
instrument_3=QCL-C
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/12/20_HS_LI72
# Time Range
start_date=2019-12-31 00:00
start_date=2020-01-07 13:00
end_date=2020-12-31 00:00
# File Settings
......@@ -27,7 +27,7 @@ filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=1
row_limit=100
row_limit=10000
# Special
select_random_files=0
......@@ -35,17 +35,17 @@ select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044
dir_out_run_log=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\raw_data_csv
file_compression=None
file_compression=gzip
plot_file_availability=1
plot_ts_hires=0
plot_ts_hires=1
plot_histogram_hires=0
plot_ts_agg=1
......
__version__ = "0.0.2"
__version__ = "0.0.4"
__date__ = "10 Dec 2020"
__link_source_code__ = "https://gitlab.ethz.ch/holukas/bico"
__link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases"
......
# =========================
# HS-100 - SONIC ANEMOMETER
# =========================
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'datablock': 'HS50-A'}
STAA == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS50-A'}
STAD == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS50-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS50-A'}
# NOTES
# =====
# (!) HS100-A HAS THE SAME BINARY FORMAT AS HS50-A
# (!) THE HS-100 HAS THE SAME BINARY FORMAT AS THE HS-50
#
# 'gain_on_signal' ... gain that was applied to the raw data signal, to get to units *divide* by this gain
# 'offset_on_signal' ... offset that was added to the raw data signal, to get to units *subtract* this offset
# 'apply_gain' ... gain that is applied during conversion, e.g. to convert to different units if needed
# 'add_offset' ... offset that is added during conversion, e.g. to convert to different units if needed
#
# BICO ID: HS50-A
# Old ID in FCT: hs-50_extended
# Old data block in FCT: data_block_sonic_hs_50_extended
#
# FCT FluxCalcTool source code:
# https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
#
# Extended data logging to comply w/ ICOS requirements.
# Installed in CH-DAV in 2017-07.
#
# from Werner Eugster email 2017-07-10:
# the former 4 data bytes containing the two inclinometer angles are now used differently
# * first byte is the StaA byte from the sonic
# * second byte is the StaD byte from the sonic
# * third and fourth byte are still a short integer as before, but now
# it contains the inclinometer X angle if the record number is an odd number,
# or the inclinometer Y angle if the record number is an even number
#
# https://docs.python.org/3.1/library/struct.html#format-characters
# B...unsigned char, integer, 1 Byte
# h...short integer, 2 Bytes
# >...big-endian, MSB most-significant Byte at lowest address
# Big-endian systems store the most significant byte of a word in the smallest address
# 2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements
#
# # https://docs.python.org/3/library/struct.html
# """
\ No newline at end of file
# ========================
# HS-50 - SONIC ANEMOMETER
# ========================
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-B'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-B'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-B'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'datablock': 'HS50-B'}
INC_X == {'order': 5, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS50-B'}
INC_Y == {'order': 6, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS50-B'}
# NOTES
# =====
#
# 'gain_on_signal' ... gain that was applied to the raw data signal, to get to units *divide* by this gain
# 'offset_on_signal' ... offset that was added to the raw data signal, to get to units *subtract* this offset
# 'apply_gain' ... gain that is applied during conversion, e.g. to convert to different units if needed
# 'add_offset' ... offset that is added during conversion, e.g. to convert to different units if needed
#
# BICO ID: HS50-B
# Old ID in FCT: hs_50
# Old data block in FCT: data_block_sonic_hs_50
#
# FCT FluxCalcTool source code:
# https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
#
# https://docs.python.org/3.1/library/struct.html#format-characters
# B...unsigned char, integer, 1 Byte
# h...short integer, 2 Bytes
# >...big-endian, MSB most-significant Byte at lowest address
# Big-endian systems store the most significant byte of a word in the smallest address
#
# # https://docs.python.org/3/library/struct.html
# """
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment