To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 1ad02a62 authored by holukas's avatar holukas
Browse files

Refined data blocks

parent fa3c297d
......@@ -2,26 +2,31 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/file.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/file.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/vis.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/vis.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/IRGA72-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7500/IRGA75-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LOSGATOS_LGR/LGR-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/AERODYNE_QCL/QCL-C.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-100/HS100-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-A.dblock" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_R3-50/R350-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.dblock" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7500/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/AERODYNE_QCL/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LOSGATOS_LGR/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-100/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_R3-50/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -32,6 +37,7 @@
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
<option value="Gherkin feature file" />
</list>
</option>
</component>
......@@ -75,11 +81,11 @@
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\ops" />
</key>
<key name="MoveFile.RECENT_KEYS">
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\header" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\gui" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\ops" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\data_blocks\sonic_anemometers" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\infrared_gas_analyzers\LICOR_LI-7500" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\infrared_gas_analyzers\LICOR_LI-7200" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\laser_gas_analyzers\QCL" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\laser_gas_analyzers\LGR" />
</key>
</component>
<component name="RunManager" selected="Python.start_bico">
......@@ -299,7 +305,9 @@
<workItem from="1607594405028" duration="3277000" />
<workItem from="1607607175897" duration="3170000" />
<workItem from="1607615147949" duration="16371000" />
<workItem from="1607676418125" duration="16535000" />
<workItem from="1607676418125" duration="18476000" />
<workItem from="1607812206779" duration="8920000" />
<workItem from="1607939225501" duration="15331000" />
</task>
<servers />
</component>
......@@ -308,13 +316,6 @@
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>320</line>
<option name="timeStamp" value="6" />
</line-breakpoint>
</breakpoints>
<default-breakpoints>
<breakpoint type="python-exception">
<properties notifyOnTerminate="true" exception="BaseException">
......@@ -332,7 +333,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607694745154" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607987599630" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component>
</project>
\ No newline at end of file
......@@ -169,7 +169,7 @@ class Ui_MainWindow(object):
# Data Blocks
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'LOSGATOS_LGR-A', 'AERODYNE_QCL-C', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'LGR-A', 'QCL-C', '-None-']
header_instr_data_blocks = qtw.QLabel('Data Blocks')
header_instr_data_blocks.setProperty('labelClass', 'header_2')
......
......@@ -143,6 +143,7 @@ def bit_map_extract_vals(bit_map_dict, var_binary_string):
val = val * bit_map_props['apply_gain']
val = val + bit_map_props['add_offset']
bit_map_vals.append(val)
# print(bit_map_vals)
return bit_map_vals
......@@ -230,6 +231,9 @@ def read_bin_instr(open_binary, dblock, total_bytes_read, logger):
# Convert to octal if needed
if props['units'] == 'octal':
var_val = convert_val_to_octal(var_val=var_val)
# Convert to octal for LGR laser analyzer
if props['units'] == 'octal_lgr':
var_val = convert_val_to_octal_lgr(var_val=var_val)
# Check if size info available
if 'DATA_SIZE' in var:
......@@ -321,6 +325,7 @@ def convert_val_to_hex(var_val):
# print("X")
return hex_val_no_prefix
def convert_val_to_octal(var_val):
"""Convert value to octal
......@@ -329,9 +334,34 @@ def convert_val_to_octal(var_val):
is converted to octal '0o0', is converted to octal without prefix '0'
Returns: int
"""
oct_val = oct(int(var_val)) # Note: has octal prefix '0o' at start, e.g. '0o0'
# if var_val != 0:
# print("x")
var_val_int = int(var_val) # Convert to integer
# var_val_bin = bin(var_val_int)[2:] # Convert to binary string
# var_val_bin = var_val_bin.zfill(8) # F
# var_val_oct = int(var_val_bin, 8) # Elegant way to convert binary string to octal
# bin(int(var_val))[2:].zfill(16)
oct_val = oct(var_val_int) # Note: has octal prefix '0o' at start, e.g. '0o0'
oct_val_no_prefix = oct_val[2:] # Remove octal prefix from val
return int(oct_val_no_prefix)
return oct_val_no_prefix
def convert_val_to_octal_lgr(var_val):
"""Convert value to octal for LGR laser analyzer
Format for the LGR status code that is recorded in the raw binary files.
Example:
- var_val = 113.0
- var_val_int = 113
- var_val_bin = '0001'
- var_val_oct = 1
Returns: int
"""
var_val_int = int(var_val) # Convert to integer
var_val_bin = bin(var_val_int)[-4:] # Convert to binary string, relevant info is in last 4 bits
var_val_oct = int(var_val_bin, 8) # Elegant way to convert binary string to octal
return var_val_oct
def generate_missing_values(dict):
......
......@@ -87,6 +87,7 @@ class SearchAll():
@staticmethod
def search_all(dir, file_id, logger):
"""Search all files in dir that match file id"""
logger.info("Searching for files ...")
valid_files_dict = {}
for root, dirs, found_files in os.walk(dir):
for idx, file in enumerate(found_files):
......@@ -145,7 +146,7 @@ class SearchAll():
self.logger.info(f"{suffix} +++ {len(valid_files_dict)} files were larger than the selected minimum"
f" filesize of {min_filesize_lim}, keeping files: {list(valid_files_dict.keys())}")
self.logger.info(f"{suffix} --- {len(_invalid_files_dict)} files were smaller than the selected minimum"
f" filesize of {min_filesize_lim} and will not be used: {list(_invalid_files_dict.keys())}")
self.logger.info(f"{suffix} ============================")
return valid_files_dict
......@@ -187,7 +188,6 @@ def export_raw_data_csv(df, outdir, logger, outfile='temp', compression='gzip'):
def export_stats_collection_csv(df, outdir, run_id, logger):
# outpath = outdir / f"stats_"
outpath = outdir / f"stats_{run_id}" # TODO act run_id
outpath = outdir / f"stats_agg_{run_id}"
logger.info(f"Saving stats collection to {outpath}")
df.to_csv(f"{outpath}.csv", index=True)
......@@ -28,8 +28,8 @@ def make_run_outdirs(settings_dict):
"""Set output paths and create output folders"""
# Run output folder
settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / "TEST_OUT"
# settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / settings_dict['run_id'] # todo act
# settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / "TEST_OUT"
settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / settings_dict['run_id'] # todo act
if not Path.is_dir(settings_dict['dir_out_run']):
print(f"Creating folder {settings_dict['dir_out_run']} ...")
os.makedirs(settings_dict['dir_out_run'])
......
......@@ -133,7 +133,7 @@ def aggs_ts(df, outdir, logger):
font = {'family': 'sans-serif', 'size': 10}
ax1.legend(frameon=True, loc='upper right', prop=font).set_zorder(100)
outfile = outdir / f"__STATS__{var[0]}_{var[1]}_{var[2]}"
outfile = outdir / f"stats_agg_{var[0]}_{var[1]}_{var[2]}"
fig.savefig(f"{outfile}.png", format='png', bbox_inches='tight', facecolor='w',
transparent=True, dpi=150)
......
run_id=BICO-20201211-145226
run_id=BICO-20201215-001320
# INSTRUMENTS
# ===========
# Site
site=CH-DAV
site=CH-AES
# Data Blocks
header=WECOM3
instrument_1=HS50-A
instrument_2=IRGA72-A
instrument_3=QCL-C
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=LGR-A
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/06/20_HS_LI72
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/08
# Time Range
start_date=2020-06-01 13:00
......@@ -26,28 +26,28 @@ end_date=2020-12-31 00:00
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=0
row_limit=100
file_limit=1
row_limit=10000
# Special
select_random_files=10
select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv
dir_out=P:/Flux/RDS_calculations/INO/AES/BICO
dir_out_run=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320
dir_out_run_log=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\log
dir_out_run_plots=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\plots
dir_out_run_plots_hires=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\plots\hires
dir_out_run_plots_agg=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\plots\agg
dir_out_run_raw_data_csv=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\raw_data_csv
file_compression=gzip
file_compression=None
plot_file_availability=1
plot_ts_hires=0
plot_histogram_hires=0
plot_ts_agg=0
plot_ts_hires=1
plot_histogram_hires=1
plot_ts_agg=1
# DIRECTORIES
# ===================
......
run_id=BICO-20201211-144647
run_id=BICO-20201215-000308
# INSTRUMENTS
# ===========
# Site
site=CH-DAV
site=CH-AES
# Data Blocks
header=WECOM3
instrument_1=HS50-A
instrument_2=IRGA72-A
instrument_3=QCL-C
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=LGR-A
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/06/20_HS_LI72
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/08
# Time Range
start_date=2020-06-01 13:00
......@@ -26,28 +26,28 @@ end_date=2020-12-31 00:00
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=0
row_limit=100
file_limit=1
row_limit=10000
# Special
select_random_files=10
select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv
dir_out=P:/Flux/RDS_calculations/INO/AES/BICO
dir_out_run=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308
dir_out_run_log=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\log
dir_out_run_plots=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\plots
dir_out_run_plots_hires=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\plots\hires
dir_out_run_plots_agg=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\plots\agg
dir_out_run_raw_data_csv=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\raw_data_csv
file_compression=gzip
file_compression=None
plot_file_availability=1
plot_ts_hires=0
plot_histogram_hires=0
plot_ts_agg=0
plot_ts_hires=1
plot_histogram_hires=1
plot_ts_agg=1
# DIRECTORIES
# ===================
......
__version__ = "0.0.4"
__date__ = "10 Dec 2020"
__version__ = "0.0.2"
__date__ = "12 Dec 2020"
__link_source_code__ = "https://gitlab.ethz.ch/holukas/bico"
__link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases"
__link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home"
......
......@@ -5,8 +5,23 @@
- V ... Second horizontal wind component (y)
- W ... Vertical wind component (z)
- T_SONIC ... Sonic temperature
- SA_DIAG_TYPE ... Status type indicator (error, inclinometer, configuration) (StaA in manual)
- SA_DIAG_VAL ... Status value and information (StaD in manual)
- SA_DIAG_TYPE ... Status type indicator (error, inclinometer, configuration)
- Called ```StaA``` (Status Address) in manual
- defines which particular aspect of the anemometer the following data field (SA_DIAG_VAL) refers to
- 00 Error codes
- 01 Anemometer configuration
- 02 Data output configuration 1
- 03 Data output configuration 2
- 04 Error code history
- 05 Transducer gain levels
- 06 Anemometer Type
- 07 Inclinometer X axis MSB
- 08 Inclinometer X axis LSB
- 09 Inclinometer Y axis MSB
- 10 Inclinometer Y axis LSB
- SA_DIAG_VAL ... Status value and information
- Called ```StaD``` (Status Data) in manual
- Please refer to the manual for detailed information about meanings
- INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers)
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
......
# =============================================
# IRGA72-A - INFRARED GAS ANALYZER (LICOR 7200)
# =============================================
# Details: see IRGA72-A.md
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'octal', 'datablock': 'IRGA72-A'}
......@@ -17,95 +19,16 @@ FLOW_VOLRATE == {'order': 12, 'bytes': 2, 'format': 'B B', 'gain_on_signal
# Values extracted from GA_DIAG_CODE:
# -------------------------------
UNUSED == {'order': 3.01, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 1, 'add_offset': 0, 'units': 'unused', 'output': 0, 'datablock': 'IRGA72-A'}
HEAD_DETECT == {'order': 3.02, 'bit_pos_start': 3, 'bit_pos_end': 4, 'apply_gain': 1, 'add_offset': 0, 'units': '1=LI-7200', 'output': 0, 'datablock': 'IRGA72-A'}
T_OUTLET == {'order': 3.03, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
T_INLET == {'order': 3.04, 'bit_pos_start': 5, 'bit_pos_end': 6, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
AUX_INPUT == {'order': 3.05, 'bit_pos_start': 6, 'bit_pos_end': 7, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
DIFF_PRESS == {'order': 3.06, 'bit_pos_start': 7, 'bit_pos_end': 8, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
CHOPPER == {'order': 3.07, 'bit_pos_start': 8, 'bit_pos_end': 9, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
DETECTOR == {'order': 3.08, 'bit_pos_start': 9, 'bit_pos_end': 10, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
PLL == {'order': 3.09, 'bit_pos_start': 10, 'bit_pos_end': 11, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
SYNC == {'order': 3.10, 'bit_pos_start': 11, 'bit_pos_end': 12, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'}
UNUSED == {'order': 3.01, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 1, 'add_offset': 0, 'units': 'unused', 'output': 1, 'datablock': 'IRGA72-A'}
HEAD_DETECT == {'order': 3.02, 'bit_pos_start': 3, 'bit_pos_end': 4, 'apply_gain': 1, 'add_offset': 0, 'units': '1=LI-7200', 'output': 1, 'datablock': 'IRGA72-A'}
T_OUTLET == {'order': 3.03, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
T_INLET == {'order': 3.04, 'bit_pos_start': 5, 'bit_pos_end': 6, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
AUX_INPUT == {'order': 3.05, 'bit_pos_start': 6, 'bit_pos_end': 7, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
DIFF_PRESS == {'order': 3.06, 'bit_pos_start': 7, 'bit_pos_end': 8, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
CHOPPER == {'order': 3.07, 'bit_pos_start': 8, 'bit_pos_end': 9, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
DETECTOR == {'order': 3.08, 'bit_pos_start': 9, 'bit_pos_end': 10, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
PLL == {'order': 3.09, 'bit_pos_start': 10, 'bit_pos_end': 11, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
SYNC == {'order': 3.10, 'bit_pos_start': 11, 'bit_pos_end': 12, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
SIGNAL_STRENGTH == {'order': 3.11, 'bit_pos_start': 12, 'bit_pos_end': 16, 'apply_gain': 6.6666666666666666, 'add_offset': 0, 'units': '%', 'output': 1, 'datablock': 'IRGA72-A'}
# NOTES
# =====
#
# BICO ID: IRGA72-A
#
# Variables:
# DATA_SIZE ... Data size of current data block, number of bytes in Licor 7200 record (2 = missing, 26 = available)
# STATUS_CODE ... Status of IRGA data aquisition, see Table 6 in WE's sonicread.pdf
# octal value converted to integer yields:
# 0 .. Status OK, no problems (octal 0000)
# 20 .. IRGA did not respond (0020)
# 40 .. Status OK, old data used (0040)
# 200 .. not OK, IRGA data are missing (0200)
# GA_DIAG_CODE ... IRGA diagnostic value
# MSB, most significant bit; high-order bit
# The cell diagnostic value is a 2 byte unsigned integer (value between 0 and 8191)
# with the following bit map (in order of how the code reads it, orig bit position in brackets):
# - (15,14,13) UNUSED
# - (12) HEAD_DETECT: sensor head attached to LI-7550; 1 = LI-7200
# - (11) T_OUTLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (10) T_INLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (9) AUX_INPUT: 1 = internal reference voltages OK;
# 0 = internal reference voltages not OK, analyzer interface unit needs service
# - (8) DIFF_PRESS: 1 = good, 0.1 to 4.9V; 0 = out of range; d=delta
# - (7) CHOPPER: 1 = chopper wheel temp is near setpoint; 0 = not near setpoint
# - (6) DETECTOR: 1 = detector temp is near setpoint; 0 = not near setpoint
# - (5) PLL: 1 = OK; lock bit, indicates that optical wheel is rotating at the correct rate
# - (4) SYNC: always set to 1 (OK)
# - (3,2,1,0) SIGNAL_STRENGTH
# * The gain 6.6666666666666666 has that many digits after the comma so that the calculated
# max signal strength is 100%. For example, if there is one digit after the comma less,
# then the max signal strength yields 99.99999999999999% instead of 100%.
# 'output': 1 means that the var is written to the output stream of this data block, i.e. included in the
# output file.
# H2O_DRY ... H2O dry mole fraction (in dry air), mixing ratio, ppt (parts per THOUSAND)
# CO2_DRY ... CO2 dry mole fraction (in dry air), mixing ratio, ppm
# H2O_CONC ... H2O concentration density, molar density
# CO2_CONC ... CO2 concentration density, molar density
# T_CELL ... Temperature of the measurement cell
# PRESS_CELL ... Pressure in the measurement cell
# PRESS_BOX ... Pressure of the box containing the logger and the electronics of the GA
# COOLER_V ... Cooler voltage
# FLOW_VOLRATE ... Volume flow rate in the sampling line
#
# BICO Settings:
# 'order' ... Order in the data block variable sequence
# 'bytes' ... Number of bytes in the binary data
# 'format' ... Format to convert from binary to integers, floats, etc.
# 'gain_on_signal' ... Gain that was applied to the raw data signal, to get to units *divide* by this gain
# 'offset_on_signal' ... Offset that was added to the raw data signal, to get to units *subtract* this offset
# 'apply_gain' ... Gain that is applied during conversion, e.g. to convert to different units if needed
# 'add_offset' ... Offset that is added during conversion, e.g. to convert to different units if needed
# 'units' ... Units
# 'datablock' ... Data block ID
#
# Before BICO, the binary conversion was done in FCT FluxCalcTool
# Old ID in FCT: li-7200_extended
# Old data block in FCT: data_block_irga_li7200_extended
# FCT Source code: --> https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
#
# Details:
# Extended data logging to comply w/ ICOS requirements.
# Installed in CH-DAV in 2017-07.
# Returns 13 columns.
# from WE's email 2017-07-10:
# The length of the LI7200 records have increased from 25 to 26 bytes;
# the extra byte for the 2-byte status is inserted before the
# former status byte was found in the beginning of the structure;
# thus, the 1-byte status is now a short int 2-byte variable
# (MSB format like all other data except those in the 29-byte header)
#
# Binary info:
# B...unsigned char, integer, 1 Byte
# h...short integer, 2 Bytes
# >...big-endian, MSB most-significant Byte at lowest address
# Big-endian systems store the most significant byte of a word in the smallest address
# --> https://docs.python.org/3/library/struct.html
# --> https://docs.python.org/3.1/library/struct.html#format-characters
#
\ No newline at end of file
# IRGA72-A
## Variables
- DATA_SIZE ... Data size of current data block, number of bytes in Licor 7200 record
(2 = missing, 26 = available)
- STATUS_CODE ... Status of IRGA data aquisition, see Table 7 in WE's sonicread.pdf
- octal value converted to integer yields:
- 0 .. Status OK, no problems (octal 0000)
- 20 .. IRGA did not respond (0020)
- 40 .. Status OK, old data used (0040)
- 200 .. not OK, IRGA data are missing (0200)
- GA_DIAG_CODE ... IRGA diagnostic value
- MSB, most significant bit; high-order bit
- The cell diagnostic value is a 2 byte unsigned integer (value between 0 and 8191)
with the following bit map (in order of how the code reads it, orig bit position in brackets):
- (15,14,13) UNUSED
- (12) HEAD_DETECT: sensor head attached to LI-7550; 1 = LI-7200
- (11) T_OUTLET: 1 = thermocouple OK; 0 = thermocouple open circuit
- (10) T_INLET: 1 = thermocouple OK; 0 = thermocouple open circuit
- (9) AUX_INPUT: 1 = internal reference voltages OK;
0 = internal reference voltages not OK, analyzer interface unit needs service
- (8) DIFF_PRESS: 1 = good, 0.1 to 4.9V; 0 = out of range; d=delta
- (7) CHOPPER: 1 = chopper wheel temp is near setpoint; 0 = not near setpoint
- (6) DETECTOR: 1 = detector temp is near setpoint; 0 = not near setpoint
- (5) PLL: 1 = OK; lock bit, indicates that optical wheel is rotating at the correct rate
- (4) SYNC: always set to 1 (OK)
- (3,2,1,0) SIGNAL_STRENGTH
The gain 6.6666666666666666 has that many digits after the comma so that the calculated
max signal strength is 100%. For example, if there is one digit after the comma less,
then the max signal strength yields 99.99999999999999% instead of 100%.
- 'output': 1 means that the var is written to the output stream of this data block,
i.e. included in the output file.
- H2O_DRY ... H2O dry mole fraction (in dry air), mixing ratio, ppt (parts per THOUSAND)
- CO2_DRY ... CO2 dry mole fraction (in dry air), mixing ratio, ppm (parts per million)
- H2O_CONC ... H2O concentration density, molar density
- CO2_CONC ... CO2 concentration density, molar density
- T_CELL ... Temperature of the measurement cell
- PRESS_CELL ... Pressure in the measurement cell
- PRESS_BOX ... Pressure of the box containing the logger and the electronics of the GA
- COOLER_V ... Cooler voltage
- FLOW_VOLRATE ... Volume flow rate in the sampling line
## BICO Settings
- 'order' ... Order in the data block variable sequence
- 'bytes' ... Number of bytes in the binary data
- 'format' ... Format to convert from binary to integers, floats, etc.
- 'gain_on_signal' ... Gain that was applied to the raw data signal, to get to units *divide* by this gain
- 'offset_on_signal' ... Offset that was added to the raw data signal, to get to units *subtract* this offset
- 'apply_gain' ... Gain that is applied during conversion, e.g. to convert to different units if needed
- 'add_offset' ... Offset that is added during conversion, e.g. to convert to different units if needed
- 'units' ... Units
- 'datablock' ... Data block ID
*Before BICO, the binary conversion was done in FCT FluxCalcTool:*
- Old ID in FCT: li-7200_extended
- Old data block in FCT: data_block_irga_li7200_extended
- FCT Source code: --> https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
## Details
- Extended data logging to comply w/ ICOS requirements.
- Installed in CH-DAV in 2017-07.
- Returns 13 columns.
- from WE's email 2017-07-10:
- The length of the LI7200 records have increased from 25 to 26 bytes;
- the extra byte for the 2-byte status is inserted before the former status byte
was found in the beginning of the structure;
- thus, the 1-byte status is now a short int 2-byte variable
- (MSB format like all other data except those in the 29-byte header)
## Binary info
- B...unsigned char, integer, 1 Byte
- h...short integer, 2 Bytes
- ">"...big-endian, MSB most-significant Byte at lowest address
Big-endian systems store the most significant byte of a word in the smallest address
- --> https://docs.python.org/3/library/struct.html
- --> https://docs.python.org/3.1/library/struct.html#format-characters
......@@ -3,12 +3,12 @@
# ================================
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA75-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'IRGA75-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'octal', 'datablock': 'IRGA75-A'}