To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit fa3c297d authored by holukas's avatar holukas
Browse files

Checks and documentation

parent 896f03ac
......@@ -2,6 +2,7 @@
dist/
_archive/
_raw_photoshop/
_tests/
example/
# Byte-compiled / optimized / DLL files
......
......@@ -2,16 +2,26 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS100-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-B.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/file.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/file.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/vis.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/vis.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/IRGA72-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7500/IRGA75-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LOSGATOS_LGR/LGR-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/AERODYNE_QCL/QCL-C.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-100/HS100-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-A.dblock" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_R3-50/R350-A.dblock" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -50,7 +60,7 @@
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="restartRequiresConfirmation" value="false" />
......@@ -58,9 +68,9 @@
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\infrared_gas_analyzers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\sonic_anemometers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\laser_gas_analyzers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\infrared_gas_analyzers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\header" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\ops" />
</key>
......@@ -288,7 +298,8 @@
<workItem from="1607504715733" duration="6609000" />
<workItem from="1607594405028" duration="3277000" />
<workItem from="1607607175897" duration="3170000" />
<workItem from="1607615147949" duration="11076000" />
<workItem from="1607615147949" duration="16371000" />
<workItem from="1607676418125" duration="16535000" />
</task>
<servers />
</component>
......@@ -300,8 +311,8 @@
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>73</line>
<option name="timeStamp" value="2" />
<line>320</line>
<option name="timeStamp" value="6" />
</line-breakpoint>
</breakpoints>
<default-breakpoints>
......@@ -321,7 +332,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607641243487" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607694745154" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component>
</project>
\ No newline at end of file
......@@ -6,8 +6,8 @@ from shutil import copyfile
import pandas as pd
from PyQt5 import QtCore as qtc
from PyQt5 import QtWidgets as qtw
from PyQt5 import QtGui as qtg
from PyQt5 import QtWidgets as qtw
import ops.logger
import ops.setup
......@@ -358,6 +358,11 @@ class Bico(qtw.QMainWindow, Ui_MainWindow):
csv_filename = f"{self.settings_dict['site']}_{csv_filedate}"
# Export file CSV
file.export_raw_data_csv(df=df, outfile=csv_filename, logger=logger,
outdir=self.settings_dict['dir_out_run_raw_data_csv'],
compression=self.settings_dict['file_compression'])
# Plot high-resolution data
if self.settings_dict['plot_ts_hires'] == '1':
vis.high_res_ts(df=df.copy(), outfile=csv_filename,
......@@ -366,11 +371,6 @@ class Bico(qtw.QMainWindow, Ui_MainWindow):
vis.high_res_histogram(df=df.copy(), outfile=csv_filename,
outdir=self.settings_dict['dir_out_run_plots_hires'], logger=logger)
# CSV
file.export_raw_data_csv(df=df, outfile=csv_filename, logger=logger,
outdir=self.settings_dict['dir_out_run_raw_data_csv'],
compression=self.settings_dict['file_compression'])
return stats_coll_df
def assemble_datablock_sequence(self):
......
......@@ -169,7 +169,7 @@ class Ui_MainWindow(object):
# Data Blocks
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA75-A', 'LGR-A', 'QCL-C', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'LOSGATOS_LGR-A', 'AERODYNE_QCL-C', '-None-']
header_instr_data_blocks = qtw.QLabel('Data Blocks')
header_instr_data_blocks.setProperty('labelClass', 'header_2')
......
......@@ -20,7 +20,6 @@ def read_bin_file_to_mem(binary_filename, logger):
def read_file(binary_filename, size_header, dblocks, limit_read_lines, logger, statusbar):
binary_filesize = os.path.getsize(binary_filename)
# Get header for all data blocks
......@@ -225,6 +224,13 @@ def read_bin_instr(open_binary, dblock, total_bytes_read, logger):
var_val = remove_gain_offset(var_value=var_val, gain=props['gain_on_signal'], offset=props['offset_on_signal'])
var_val = apply_gain_offset(var_value=var_val, gain=props['apply_gain'], offset=props['add_offset'])
# Convert to hex if needed
if props['units'] == 'hexadecimal_value':
var_val = convert_val_to_hex(var_val=var_val)
# Convert to octal if needed
if props['units'] == 'octal':
var_val = convert_val_to_octal(var_val=var_val)
# Check if size info available
if 'DATA_SIZE' in var:
dblock_true_size = int(var_val)
......@@ -299,6 +305,35 @@ def read_bin_instr(open_binary, dblock, total_bytes_read, logger):
return dblock_data, total_bytes_read, end_of_data_reached
def convert_val_to_hex(var_val):
"""Convert value to hexadecimal
Examples:
- var_val = 40.0, is converted to integer 40,
is converted to hex '0x28', is converted to hex without prefix '28'
- var_val = 60.0, is converted to integer 60,
is converted to hex '0x3c', is converted to hex without prefix '3c'
Returns: string
"""
hex_val = hex(int(var_val)) # Note: has hexadecimal prefix '0x' at start, e.g. '0x28'
hex_val_no_prefix = hex_val[2:] # Remove hex prefix from val
# if hex_val_no_prefix == '3c':
# print("X")
return hex_val_no_prefix
def convert_val_to_octal(var_val):
"""Convert value to octal
Examples:
- var_val = 0.0, is converted to integer 0,
is converted to octal '0o0', is converted to octal without prefix '0'
Returns: int
"""
oct_val = oct(int(var_val)) # Note: has octal prefix '0o' at start, e.g. '0o0'
oct_val_no_prefix = oct_val[2:] # Remove octal prefix from val
return int(oct_val_no_prefix)
def generate_missing_values(dict):
missing_vals = []
for v in dict.items():
......
......@@ -28,8 +28,8 @@ def make_run_outdirs(settings_dict):
"""Set output paths and create output folders"""
# Run output folder
# settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / "TEST_OUT"
settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / settings_dict['run_id'] # todo act
settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / "TEST_OUT"
# settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / settings_dict['run_id'] # todo act
if not Path.is_dir(settings_dict['dir_out_run']):
print(f"Creating folder {settings_dict['dir_out_run']} ...")
os.makedirs(settings_dict['dir_out_run'])
......
import datetime as dt
import os
import seaborn
import matplotlib.dates as mdates
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
......@@ -123,7 +123,8 @@ def aggs_ts(df, outdir, logger):
text_args = dict(verticalalignment='top',
size=14, color='black', backgroundcolor='none', zorder=100)
ax1.text(0.01, 0.96, f"{var[0]} {var[1]} {var[2]}", transform=ax1.transAxes, horizontalalignment='left', **text_args)
ax1.text(0.01, 0.96, f"{var[0]} {var[1]} {var[2]}", transform=ax1.transAxes, horizontalalignment='left',
**text_args)
_default_format(ax=ax1, width=1, length=2, txt_ylabel=var[0], txt_ylabel_units=var[1])
_default_format(ax=ax2, width=1, length=2, txt_ylabel='counts')
......@@ -136,27 +137,6 @@ def aggs_ts(df, outdir, logger):
fig.savefig(f"{outfile}.png", format='png', bbox_inches='tight', facecolor='w',
transparent=True, dpi=150)
# text_args = dict(verticalalignment='top',
# size=6, color='black', backgroundcolor='none', zorder=100)
# col_idx = -1
# for col in df.columns:
# col_idx += 1
# ax.plot(df[col].index, df[col], 'r,', alpha=0.5, c='#5f87ae')
# ax.text(0.01, 0.96, f"{col[0]} {col[1]}", transform=ax.transAxes, horizontalalignment='left', **text_args)
# txt_info = f"mean: {df[col].mean():.3f}\nvals: {df[col].count():.0f}"
# ax.text(0.99, 0.96, txt_info, transform=ax.transAxes, horizontalalignment='right',**text_args)
#
# # Show bottom labels for last axis in column
# if (col_idx == figcollimit) | (col_idx == len(df.columns)-1):
# ax.tick_params(labelbottom=True)
# else:
# ax.tick_params(labelbottom=False)
#
# for tick in ax.xaxis.get_major_ticks():
# tick.label.set_fontsize(6)
# for tick in ax.yaxis.get_major_ticks():
# tick.label.set_fontsize(6)
def high_res_histogram(df, outfile, outdir, logger):
logger.info(" Plotting high-res data: histograms ...")
......@@ -206,16 +186,15 @@ def high_res_histogram(df, outfile, outdir, logger):
# If data for col is empty
ax.set_facecolor('xkcd:salmon')
ax.text(0.5, 0.5,
f"(!)WARNING: variable {col} is empty and was therefore not plotted",
horizontalalignment='center', verticalalignment='center', transform=ax.transAxes,
size=14, color='white', backgroundcolor='red')
f"(!)WARNING: variable {col} is empty and was therefore not plotted",
horizontalalignment='center', verticalalignment='center', transform=ax.transAxes,
size=14, color='white', backgroundcolor='red')
dblock_outfile = outdir / f"{outfile}_hires_histogram_{dblock}"
fig.savefig(f"{dblock_outfile}.png", format='png', bbox_inches='tight', facecolor='w',
transparent=True, dpi=100)
def high_res_ts(df, outfile, outdir, logger):
logger.info(" Plotting high-res data: time series ...")
......@@ -230,6 +209,7 @@ def high_res_ts(df, outfile, outdir, logger):
num_plots = len(dblock_df.columns)
cols = dblock_df.columns
# Gridspec and axes
gs = gridspec.GridSpec(num_plots, 1) # rows, cols
gs.update(wspace=0.1, hspace=0, left=0.03, right=0.99, top=0.99, bottom=0.01)
fig = plt.Figure(facecolor='white', figsize=(16, 9))
......@@ -242,17 +222,26 @@ def high_res_ts(df, outfile, outdir, logger):
text_args = dict(verticalalignment='top',
size=6, color='black', backgroundcolor='none', zorder=100)
# Plot columns
col_idx = -1
for col in dblock_df.columns:
# if dblock == '[HS50-A]':
# print(col)
col_idx += 1
ax = axes[col_idx]
ax.plot(dblock_df[col].index, dblock_df[col], 'r,', alpha=0.5, c='#5f87ae')
dataok = check_plot_data(ax=ax, df=dblock_df, col=col)
if dataok:
# Numeric data, values available
ax.plot(dblock_df[col].index, dblock_df[col], 'r,', alpha=0.5, c='#5f87ae')
txt_info = f"mean: {dblock_df[col].mean():.3f} vals: {dblock_df[col].count():.0f}\n" \
f"min: {dblock_df[col].min():.3f} max:{dblock_df[col].max():.3f}"
ax.text(0.99, 0.96, txt_info, transform=ax.transAxes, horizontalalignment='right', **text_args)
_default_format(ax=ax, width=1, length=2)
ax.text(0.01, 0.96, f"{col[0]} {col[1]} {col[2]}", transform=ax.transAxes, horizontalalignment='left',
**text_args)
txt_info = f"mean: {dblock_df[col].mean():.3f} vals: {dblock_df[col].count():.0f}\n" \
f"min: {dblock_df[col].min():.3f} max:{dblock_df[col].max():.3f}"
ax.text(0.99, 0.96, txt_info, transform=ax.transAxes, horizontalalignment='right', **text_args)
# Show bottom labels for last axis in column
if col_idx == len(dblock_df.columns) - 1:
......@@ -269,6 +258,40 @@ def high_res_ts(df, outfile, outdir, logger):
fig.savefig(f"{dblock_outfile}.png", format='png', bbox_inches='tight', facecolor='w',
transparent=True, dpi=150)
def check_plot_data(ax, df, col):
dataok = False
isnumeric = False if df[col].dtypes == object else True
ishex = False if 'hexadecimal_value' not in col[1] else True
isemtpy = False if not df[col].dropna().empty else True
txt_warning = "-NOT-FOUND-"
facecolor = None
if not isemtpy and not isnumeric and ishex:
# Expected non-numeric hexadecimal data, values available
facecolor = 'xkcd:green'
txt_warning = f"NOTE: Variable {col} is a HEXADECIMAL (non-numeric) value " \
f"and was therefore not plotted."
if not isemtpy and not isnumeric and not ishex:
# General non-numeric data, values available
facecolor = 'xkcd:salmon'
txt_warning = f"(!)WARNING: variable {col} is NON-NUMERIC and was therefore not plotted."
if isemtpy:
# No values available
facecolor = 'xkcd:aqua'
txt_warning = f"(!)WARNING: data for variable {col} is EMPTY and was therefore not plotted."
if not isemtpy and not ishex and isnumeric:
# Data are OK
dataok = True
if not dataok:
ax.text(0.5, 0.5, txt_warning,
horizontalalignment='center', verticalalignment='center', transform=ax.transAxes,
size=14, color='white', backgroundcolor=facecolor)
return dataok
def _format_spines(ax, color, lw):
spines = ['top', 'bottom', 'left', 'right']
......
run_id=BICO-20201211-000044
run_id=BICO-20201211-145226
# INSTRUMENTS
# ===========
......@@ -16,38 +16,38 @@ instrument_3=QCL-C
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/12/20_HS_LI72
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/06/20_HS_LI72
# Time Range
start_date=2020-01-07 13:00
start_date=2020-06-01 13:00
end_date=2020-12-31 00:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=1
row_limit=10000
file_limit=0
row_limit=100
# Special
select_random_files=0
select_random_files=10
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044
dir_out_run_log=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv
file_compression=gzip
plot_file_availability=1
plot_ts_hires=1
plot_ts_hires=0
plot_histogram_hires=0
plot_ts_agg=1
plot_ts_agg=0
# DIRECTORIES
# ===================
......
run_id=BICO-20201211-000044
run_id=BICO-20201211-144647
# INSTRUMENTS
# ===========
......@@ -8,7 +8,7 @@ site=CH-DAV
# Data Blocks
header=WECOM3
instrument_1=HS50-B
instrument_1=HS50-A
instrument_2=IRGA72-A
instrument_3=QCL-C
......@@ -16,38 +16,38 @@ instrument_3=QCL-C
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/12/20_HS_LI72
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/06/20_HS_LI72
# Time Range
start_date=2020-01-07 13:00
start_date=2020-06-01 13:00
end_date=2020-12-31 00:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=1
row_limit=10000
file_limit=0
row_limit=100
# Special
select_random_files=0
select_random_files=10
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044
dir_out_run_log=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\BICO-20201211-000044\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv
file_compression=gzip
plot_file_availability=1
plot_ts_hires=1
plot_ts_hires=0
plot_histogram_hires=0
plot_ts_agg=1
plot_ts_agg=0
# DIRECTORIES
# ===================
......
# =============================================
# IRGA72-A - INFRARED GAS ANALYZER (LICOR 7200)
# =============================================
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'octal', 'datablock': 'IRGA72-A'}
GA_DIAG_CODE == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'IRGA72-A'}
H2O_DRY == {'order': 4, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1_mol-1', 'datablock': 'IRGA72-A'}
CO2_DRY == {'order': 5, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'umol+1_mol-1', 'datablock': 'IRGA72-A'}
H2O_CONC == {'order': 6, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1_m-3', 'datablock': 'IRGA72-A'}
CO2_CONC == {'order': 7, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1_m-3', 'datablock': 'IRGA72-A'}
T_CELL == {'order': 8, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 100, 'offset_on_signal': 100, 'apply_gain': 1, 'add_offset': 0, 'units': 'degC', 'datablock': 'IRGA72-A'}
PRESS_CELL == {'order': 9, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 10, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'hPa', 'datablock': 'IRGA72-A'}
PRESS_BOX == {'order': 10, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 10, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'hPa', 'datablock': 'IRGA72-A'}
COOLER_V == {'order': 11, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'V', 'datablock': 'IRGA72-A'}
FLOW_VOLRATE == {'order': 12, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'L+1 min-1', 'datablock': 'IRGA72-A'}
# Values extracted from GA_DIAG_CODE:
# -------------------------------
UNUSED == {'order': 3.01, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 1, 'add_offset': 0, 'units': 'unused', 'output': 0, 'datablock': 'IRGA72-A'}
HEAD_DETECT == {'order': 3.02, 'bit_pos_start': 3, 'bit_pos_end': 4, 'apply_gain': 1, 'add_offset': 0, 'units': '1=LI-7200', 'output': 0, 'datablock': 'IRGA72-A'}
T_OUTLET == {'order': 3.03, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
T_INLET == {'order': 3.04, 'bit_pos_start': 5, 'bit_pos_end': 6, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
AUX_INPUT == {'order': 3.05, 'bit_pos_start': 6, 'bit_pos_end': 7, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
DIFF_PRESS == {'order': 3.06, 'bit_pos_start': 7, 'bit_pos_end': 8, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
CHOPPER == {'order': 3.07, 'bit_pos_start': 8, 'bit_pos_end': 9, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
DETECTOR == {'order': 3.08, 'bit_pos_start': 9, 'bit_pos_end': 10, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
PLL == {'order': 3.09, 'bit_pos_start': 10, 'bit_pos_end': 11, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
SYNC == {'order': 3.10, 'bit_pos_start': 11, 'bit_pos_end': 12, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
SIGNAL_STRENGTH == {'order': 3.11, 'bit_pos_start': 12, 'bit_pos_end': 16, 'apply_gain': 6.6666666666666666, 'add_offset': 0, 'units': '%', 'output': 1, 'datablock': 'IRGA72-A'}
# NOTES
# =====
#
# BICO ID: IRGA72-A
#
# Variables:
# DATA_SIZE ... Data size of current data block, number of bytes in Licor 7200 record (2 = missing, 26 = available)
# STATUS_CODE ... Status of IRGA data aquisition, see Table 6 in WE's sonicread.pdf
# octal value converted to integer yields:
# 0 .. Status OK, no problems (octal 0000)
# 20 .. IRGA did not respond (0020)
# 40 .. Status OK, old data used (0040)
# 200 .. not OK, IRGA data are missing (0200)
# GA_DIAG_CODE ... IRGA diagnostic value
# MSB, most significant bit; high-order bit
# The cell diagnostic value is a 2 byte unsigned integer (value between 0 and 8191)
# with the following bit map (in order of how the code reads it, orig bit position in brackets):
# - (15,14,13) UNUSED
# - (12) HEAD_DETECT: sensor head attached to LI-7550; 1 = LI-7200
# - (11) T_OUTLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (10) T_INLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (9) AUX_INPUT: 1 = internal reference voltages OK;
# 0 = internal reference voltages not OK, analyzer interface unit needs service
# - (8) DIFF_PRESS: 1 = good, 0.1 to 4.9V; 0 = out of range; d=delta
# - (7) CHOPPER: 1 = chopper wheel temp is near setpoint; 0 = not near setpoint
# - (6) DETECTOR: 1 = detector temp is near setpoint; 0 = not near setpoint
# - (5) PLL: 1 = OK; lock bit, indicates that optical wheel is rotating at the correct rate
# - (4) SYNC: always set to 1 (OK)
# - (3,2,1,0) SIGNAL_STRENGTH
# * The gain 6.6666666666666666 has that many digits after the comma so that the calculated
# max signal strength is 100%. For example, if there is one digit after the comma less,
# then the max signal strength yields 99.99999999999999% instead of 100%.
# 'output': 1 means that the var is written to the output stream of this data block, i.e. included in the
# output file.
# H2O_DRY ... H2O dry mole fraction (in dry air), mixing ratio, ppt (parts per THOUSAND)
# CO2_DRY ... CO2 dry mole fraction (in dry air), mixing ratio, ppm
# H2O_CONC ... H2O concentration density, molar density
# CO2_CONC ... CO2 concentration density, molar density
# T_CELL ... Temperature of the measurement cell
# PRESS_CELL ... Pressure in the measurement cell
# PRESS_BOX ... Pressure of the box containing the logger and the electronics of the GA
# COOLER_V ... Cooler voltage
# FLOW_VOLRATE ... Volume flow rate in the sampling line
#
# BICO Settings:
# 'order' ... Order in the data block variable sequence
# 'bytes' ... Number of bytes in the binary data
# 'format' ... Format to convert from binary to integers, floats, etc.
# 'gain_on_signal' ... Gain that was applied to the raw data signal, to get to units *divide* by this gain
# 'offset_on_signal' ... Offset that was added to the raw data signal, to get to units *subtract* this offset
# 'apply_gain' ... Gain that is applied during conversion, e.g. to convert to different units if needed
# 'add_offset' ... Offset that is added during conversion, e.g. to convert to different units if needed
# 'units' ... Units
# 'datablock' ... Data block ID
#
# Before BICO, the binary conversion was done in FCT FluxCalcTool
# Old ID in FCT: li-7200_extended
# Old data block in FCT: data_block_irga_li7200_extended
# FCT Source code: --> https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
#
# Details:
# Extended data logging to comply w/ ICOS requirements.
# Installed in CH-DAV in 2017-07.
# Returns 13 columns.
# from WE's email 2017-07-10:
# The length of the LI7200 records have increased from 25 to 26 bytes;
# the extra byte for the 2-byte status is inserted before the
# former status byte was found in the beginning of the structure;
# thus, the 1-byte status is now a short int 2-byte variable
# (MSB format like all other data except those in the 29-byte header)
#
# Binary info:
# B...unsigned char, integer, 1 Byte
# h...short integer, 2 Bytes
# >...big-endian, MSB most-significant Byte at lowest address
# Big-endian systems store the most significant byte of a word in the smallest address
# --> https://docs.python.org/3/library/struct.html
# --> https://docs.python.org/3.1/library/struct.html#format-characters
#
\ No newline at end of file
# ================================
# IRGA72-A - INFRARED GAS ANALYZER
# IRGA72-B - INFRARED GAS ANALYZER
# ================================
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'IRGA72-A'}
DIAG_VAL == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'IRGA72-A'}
H2O_DMF == {'order': 4, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 mol-1', 'datablock': 'IRGA72-A'}
CO2_DMF == {'order': 5, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'umol+1 mol-1', 'datablock': 'IRGA72-A'}
H2O_MD == {'order': 6, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA72-A'}
CO2_MD == {'order': 7, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA72-A'}
T_CELL == {'order': 8, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 100, 'offset_on_signal': 100, 'apply_gain': 1, 'add_offset': 0, 'units': 'degC', 'datablock': 'IRGA72-A'}
P_CELL == {'order': 9, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 10, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Pa', 'datablock': 'IRGA72-A'}
P_ATM == {'order': 10, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 10, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Pa', 'datablock': 'IRGA72-A'}
COOLER_V == {'order': 11, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'V', 'datablock': 'IRGA72-A'}
FLOWRATE == {'order': 12, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'L+1 min-1', 'datablock': 'IRGA72-A'}
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-B'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'IRGA72-B'}
DIAG_VAL == {'order': 3, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'IRGA72-B'}
H2O_DMF == {'order': 4, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 mol-1', 'datablock': 'IRGA72-B'}