To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 29f62dd9 authored by holukas's avatar holukas
Browse files

Reworked and added datablocks

parent b26aa6e3
......@@ -2,22 +2,29 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A3.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A3.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/vis.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/vis.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.dblock" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -322,6 +329,9 @@
<workItem from="1609852670010" duration="24000" />
<workItem from="1609859126683" duration="11243000" />
<workItem from="1609921205302" duration="15275000" />
<workItem from="1609970074199" duration="1322000" />
<workItem from="1609972599782" duration="2676000" />
<workItem from="1610015969067" duration="13596000" />
</task>
<servers />
</component>
......@@ -333,9 +343,14 @@
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>231</line>
<line>241</line>
<option name="timeStamp" value="27" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>142</line>
<option name="timeStamp" value="28" />
</line-breakpoint>
</breakpoints>
<default-breakpoints>
<breakpoint type="python-exception">
......@@ -354,7 +369,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1609945717534" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1610055914638" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component>
</project>
\ No newline at end of file
......@@ -15,8 +15,6 @@ from gui.gui import Ui_MainWindow
from ops import bin, format_data, vis, file, stats
from settings import _version
# TODO hs-50 status code format
# TODO LATER parallelize, multiprocessing?
# import multiprocessing as mp
# print("Number of processors: ", mp.cpu_count())
......
......@@ -175,7 +175,8 @@ class Ui_MainWindow(object):
# Data Blocks
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'IRGA75-A-GN1', 'LGR-A', 'QCL-C', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'IRGA75-A-GN1', 'LGR-A',
'QCL-A', 'QCL-A2', 'QCL-A3', 'QCL-C', '-None-']
header_instr_data_blocks = qtw.QLabel('Data Blocks')
header_instr_data_blocks.setProperty('labelClass', 'header_2')
......
......@@ -63,7 +63,6 @@ class ReadInstrDatablock:
# If datablock has the expected size, proceed normally
if self.dblock_true_size == self.dblock_nominal_size:
pass
# If datablock does not have the expected size, generate missing data
elif self.dblock_bytes_read == 2:
# In this case there are analyzer data missing, i.e. the whole data block is either only 2 Bytes
......@@ -85,13 +84,12 @@ class ReadInstrDatablock:
if self.dblock_true_size != 2:
self.read_rest_of_bytes()
break
# return self.dblock_data, self.total_bytes_read, self.end_of_data_reached
# Convert to hex or octal if needed
var_val = self.convert_val(props=props, var_val=var_val)
# Convert to hex or octal if needed
var_val = self.convert_val(props=props, var_val=var_val)
# Add value to data
self.dblock_data.append(var_val)
......@@ -108,30 +106,42 @@ class ReadInstrDatablock:
def convert_val(self, props, var_val):
"""Convert var value to hex or octal"""
# Convert to hex
if props['units'] == 'hexadecimal_value':
var_val = self.convert_val_to_hex(var_val=var_val)
# Convert to octal
if props['units'] == 'diag_val_hs':
var_val = self.convert_val_to_diag_val_hs(var_val=var_val)
if props['units'] == 'status_code_irga':
var_val = self.convert_val_to_status_code_irga(var_val=var_val)
# Convert to octal for LGR laser analyzer
if props['units'] == 'status_code_lgr':
var_val = self.convert_val_to_status_code_lgr(var_val=var_val)
return var_val
def convert_val_to_hex(self, var_val):
"""Convert value to hexadecimal
def convert_val_to_diag_val_hs(self, var_val):
"""Convert value to diagnostic value for Gill HS-50 and HS-100 sonic anemometers
Saved as an integer value. The integer can be converted to binary to get more
information. Since the information in this variable is quite complex, please
refer to the sonic anemometer manual for details.
Examples:
- var_val = 40.0, is converted to integer 40,
is converted to hex '0x28', is converted to hex without prefix '28'
- var_val = 60.0, is converted to integer 60,
is converted to hex '0x3c', is converted to hex without prefix '3c'
- var_val = 47.0, is converted to integer 47, this is the value that is
then stored in the converted ASCII file.
Notes:
- The variable SA_DIAG_TYPE from the sonic HS anmometers gives info
which type of information is given in SA_DIAG_VAL.
- To get more information from the integer, it can be converted to
binary, e.g. integer 47 becomes binary "0b101111"
In Python, this can be done with bin(47) --> "0b101111"
- The first two characters "0b" mean that this is binary format
- The information is contained in "101111", it tells us which bits are
set to 1 or 0. This can be interpreted using the sonic manual, there
is a list what each bit set to 1 or 0 means.
Returns: string
"""
hex_val = hex(int(var_val)) # Note: has hexadecimal prefix '0x' at start, e.g. '0x28'
hex_val_no_prefix = hex_val[2:] # Remove hex prefix from val
return hex_val_no_prefix
# hex_val = hex(int(var_val)) # Note: has hexadecimal prefix '0x' at start, e.g. '0x28'
# hex_val_no_prefix = hex_val[2:] # Remove hex prefix from val
diag_val_hs = int(var_val)
return diag_val_hs
def convert_val_to_status_code_irga(self, var_val):
"""Convert value to octal
......
run_id=BICO-20210106-160838
run_id=BICO-20210107-224515
# INSTRUMENTS
# ===========
# Site
site=CH-AES
site=CH-CHA
# Data Blocks
header=WECOM3
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=LGR-A
instrument_3=QCL-A3
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/05
dir_source=Y:/CH-CHA_Chamau/20_sonic_ghg/2018/11
# Time Range
start_date=2020-05-13 13:00
end_date=2020-05-13 13:00
start_date=2018-11-27 09:00
end_date=2018-11-30 23:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
filename_datetime_format=yyyymmddHH.CMM
file_ext=*.C*
file_size_min=900
file_limit=0
row_limit=0
row_limit=1000
# Special
select_random_files=0
select_random_files=5
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838
dir_out_run_log=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515
dir_out_run_log=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\raw_data_csv
output_folder_name_prefix=AES
output_folder_name_prefix=CHA
file_compression=None
plot_file_availability=0
......
run_id=BICO-20210106-160838
run_id=BICO-20210107-224515
# INSTRUMENTS
# ===========
# Site
site=CH-AES
site=CH-CHA
# Data Blocks
header=WECOM3
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=LGR-A
instrument_3=QCL-A3
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/05
dir_source=Y:/CH-CHA_Chamau/20_sonic_ghg/2018/11
# Time Range
start_date=2020-05-13 13:00
end_date=2020-05-13 13:00
start_date=2018-11-27 09:00
end_date=2018-11-27 23:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
filename_datetime_format=yyyymmddHH.CMM
file_ext=*.C*
file_size_min=900
file_limit=0
row_limit=0
file_limit=1
row_limit=1000
# Special
select_random_files=0
......@@ -35,14 +35,14 @@ select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838
dir_out_run_log=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515
dir_out_run_log=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\raw_data_csv
output_folder_name_prefix=AES
output_folder_name_prefix=CHA
file_compression=None
plot_file_availability=0
......
__version__ = "0.0.5"
__date__ = "6 Jan 2021"
__version__ = "0.0.6"
__date__ = "7 Jan 2021"
__link_source_code__ = "https://gitlab.ethz.ch/holukas/bico"
__link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases"
__link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home"
......
# =========================
# HS-100 - SONIC ANEMOMETER
# =========================
# Details: see HS100-A.md
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS100-A'}
......@@ -10,42 +11,3 @@ T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'o
STAA == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'}
STAD == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS100-A'}
# NOTES
# =====
# (!) HS100-A HAS THE SAME BINARY FORMAT AS HS50-A
# (!) THE HS-100 HAS THE SAME BINARY FORMAT AS THE HS-50
#
# 'gain_on_signal' ... gain that was applied to the raw data signal, to get to units *divide* by this gain
# 'offset_on_signal' ... offset that was added to the raw data signal, to get to units *subtract* this offset
# 'apply_gain' ... gain that is applied during conversion, e.g. to convert to different units if needed
# 'add_offset' ... offset that is added during conversion, e.g. to convert to different units if needed
#
# BICO ID: HS100-A
# Old ID in FCT: hs_100
# Old data block in FCT: data_block_sonic_hs_100
#
# FCT FluxCalcTool source code:
# https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
#
# Extended data logging to comply w/ ICOS requirements.
# Installed in CH-DAV in 2017-07.
#
# from Werner Eugster email 2017-07-10:
# the former 4 data bytes containing the two inclinometer angles are now used differently
# * first byte is the StaA byte from the sonic
# * second byte is the StaD byte from the sonic
# * third and fourth byte are still a short integer as before, but now
# it contains the inclinometer X angle if the record number is an odd number,
# or the inclinometer Y angle if the record number is an even number
#
# https://docs.python.org/3.1/library/struct.html#format-characters
# B...unsigned char, integer, 1 Byte
# h...short integer, 2 Bytes
# >...big-endian, MSB most-significant Byte at lowest address
# Big-endian systems store the most significant byte of a word in the smallest address
# 2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements
#
# # https://docs.python.org/3/library/struct.html
# """
\ No newline at end of file
# HS100-A
(!) HS100-A HAS THE SAME BINARY FORMAT AS HS50-A
(!) THE HS-100 HAS THE SAME BINARY FORMAT AS THE HS-50
## Variables
- U ... First horizontal wind component (x)
- V ... Second horizontal wind component (y)
- W ... Vertical wind component (z)
- T_SONIC ... Sonic temperature
- SA_DIAG_TYPE ... Status type indicator (error, inclinometer, configuration)
- Called ```StaA``` (Status Address) in manual
- defines which particular aspect of the anemometer the following data field (SA_DIAG_VAL) refers to
- 00 Error codes
- 01 Anemometer configuration
- 02 Data output configuration 1
- 03 Data output configuration 2
- 04 Error code history
- 05 Transducer gain levels
- 06 Anemometer Type
- 07 Inclinometer X axis MSB
- 08 Inclinometer X axis LSB
- 09 Inclinometer Y axis MSB
- 10 Inclinometer Y axis LSB
- SA_DIAG_VAL ... Status value and information
- Called ```StaD``` (Status Data) in manual
- SA_DIAG_VAL is rather complex and can tak on a lot of meanings.
- Has to be interpreted in combination with SA_DIAG_TYPE, which gives the type
of SA_DIAG_VAL.
- Saved as an integer value, e.g. ```47```.
- The integer can be converted to binary, e.g. ```47``` becomes ```0b101111``` , whereby the
first two characters ```0b``` mean that this is binary format.
- From the binary, detailed information can be extracted, depending on which bits
in the binary are set to 1 or 0.
- Please refer to the manual for detailed information about what the different bits mean
when they are 1 or zero.
- INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers)
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
## BICO Settings
- 'order' ... Order in the data block variable sequence
- 'bytes' ... Number of bytes in the binary data
- 'format' ... Format to convert from binary to integers, floats, etc.
- 'gain_on_signal' ... Gain that was applied to the raw data signal, to get to units *divide* by this gain
- 'offset_on_signal' ... Offset that was added to the raw data signal, to get to units *subtract* this offset
- 'apply_gain' ... Gain that is applied during conversion, e.g. to convert to different units if needed
- 'add_offset' ... Offset that is added during conversion, e.g. to convert to different units if needed
- 'units' ... Units
- 'datablock' ... Data block ID
*Before BICO, the binary conversion was done in FCT FluxCalcTool:*
- Old ID in FCT: hs_100
- Old data block in FCT: data_block_sonic_hs_100
- FCT Source code: --> https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
## Details
- Extended data logging to comply w/ ICOS requirements.
- from WE's email 2017-07-10:
- the former 4 data bytes containing the two inclinometer angles are now used differently
- first byte is the StaA byte from the sonic
- second byte is the StaD byte from the sonic
- third and fourth byte are still a short integer as before, but now
it contains the inclinometer X angle if the record number is an odd number,
or the inclinometer Y angle if the record number is an even number
## Binary info
- B...unsigned char, integer, 1 Byte
- h...short integer, 2 Bytes
- ">"...big-endian, MSB most-significant Byte at lowest address
Big-endian systems store the most significant byte of a word in the smallest address
- --> https://docs.python.org/3/library/struct.html
- --> https://docs.python.org/3.1/library/struct.html#format-characters
......@@ -3,10 +3,11 @@
# ======================================
# Details: see HS50-A.md
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'datablock': 'HS50-A'}
SA_DIAG_TYPE == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS50-A'}
SA_DIAG_VAL == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'hexadecimal_value', 'datablock': 'HS50-A'}
SA_DIAG_TYPE == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs_type', 'datablock': 'HS50-A'}
SA_DIAG_VAL == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs', 'datablock': 'HS50-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS50-A'}
......@@ -21,7 +21,16 @@
- 10 Inclinometer Y axis LSB
- SA_DIAG_VAL ... Status value and information
- Called ```StaD``` (Status Data) in manual
- Please refer to the manual for detailed information about meanings
- SA_DIAG_VAL is rather complex and can tak on a lot of meanings.
- Has to be interpreted in combination with SA_DIAG_TYPE, which gives the type
of SA_DIAG_VAL.
- Saved as an integer value, e.g. ```47```.
- The integer can be converted to binary, e.g. ```47``` becomes ```0b101111``` , whereby the
first two characters ```0b``` mean that this is binary format.
- From the binary, detailed information can be extracted, depending on which bits
in the binary are set to 1 or 0.
- Please refer to the manual for detailed information about what the different bits mean
when they are 1 or zero.
- INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers)
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
......
# ================================
# IRGA72-B - INFRARED GAS ANALYZER
# ================================
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-B'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'IRGA72-B'}
DIAG_VAL == {'order': 3, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'IRGA72-B'}
H2O_DMF == {'order': 4, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 mol-1', 'datablock': 'IRGA72-B'}
CO2_DMF == {'order': 5, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'umol+1 mol-1', 'datablock': 'IRGA72-B'}
H2O_MD == {'order': 6, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA72-B'}
CO2_MD == {'order': 7, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA72-B'}
T_CELL == {'order': 8, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 100, 'offset_on_signal': 100, 'apply_gain': 1, 'add_offset': 0, 'units': 'degC', 'datablock': 'IRGA72-B'}
P_CELL == {'order': 9, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 0.1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Pa', 'datablock': 'IRGA72-B'}
P_ATM == {'order': 10, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 0.1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Pa', 'datablock': 'IRGA72-B'}
COOLER_V == {'order': 11, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'V', 'datablock': 'IRGA72-B'}
FLOWRATE == {'order': 12, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'L+1 min-1', 'datablock': 'IRGA72-B'}
# Values extracted from DIAG_VAL:
# --------------------------------------
UNUSED == {'order': 3.01, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 1, 'add_offset': 0, 'units': 'unused', 'output': 0, 'datablock': 'IRGA72-B'}
HEAD_DETECT == {'order': 3.02, 'bit_pos_start': 3, 'bit_pos_end': 4, 'apply_gain': 1, 'add_offset': 0, 'units': '1=LI-7200', 'output': 0, 'datablock': 'IRGA72-B'}
T_OUTLET == {'order': 3.03, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
T_INLET == {'order': 3.04, 'bit_pos_start': 5, 'bit_pos_end': 6, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
AUX_INPUT == {'order': 3.05, 'bit_pos_start': 6, 'bit_pos_end': 7, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
DIFF_PRESS == {'order': 3.06, 'bit_pos_start': 7, 'bit_pos_end': 8, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
CHOPPER == {'order': 3.07, 'bit_pos_start': 8, 'bit_pos_end': 9, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
DETECTOR == {'order': 3.08, 'bit_pos_start': 9, 'bit_pos_end': 10, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
PLL == {'order': 3.09, 'bit_pos_start': 10, 'bit_pos_end': 11, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
SYNC == {'order': 3.10, 'bit_pos_start': 11, 'bit_pos_end': 12, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
SIGNAL_STRENGTH == {'order': 3.11, 'bit_pos_start': 12, 'bit_pos_end': 16, 'apply_gain': 6.6666666666666666, 'add_offset': 0, 'units': '%', 'output': 0, 'datablock': 'IRGA72-B'}
# NOTES
# """
# DIAG_VAL:
# MSB, most significant bit; high-order bit
# The cell diagnostic value is a 2 byte unsigned integer (value between 0 and 8191)
# with the following bit map (in order of how the code reads it, orig bit position in brackets):
# - (15,14,13) UNUSED
# - (12) HEAD_DETECT: sensor head attached to LI-7550; 1 = LI-7200
# - (11) T_OUTLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (10) T_INLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (9) AUX_INPUT: 1 = internal reference voltages OK;
# 0 = internal reference voltages not OK, analyzer interface unit needs service
# - (8) DIFF_PRESS: 1 = good, 0.1 to 4.9V; 0 = out of range; d=delta
# - (7) CHOPPER: 1 = chopper wheel temp is near setpoint; 0 = not near setpoint
# - (6) DETECTOR: 1 = detector temp is near setpoint; 0 = not near setpoint
# - (5) PLL: 1 = OK; lock bit, indicates that optical wheel is rotating at the correct rate
# - (4) SYNC: always set to 1 (OK)
# - (3,2,1,0) SIGNAL_STRENGTH
# * The gain 6.6666666666666666 has that many digits after the comma so that the calculated
# max signal strength is 100%. For example, if there is one digit after the comma less,
# then the max signal strength yields 99.99999999999999% instead of 100%.
# 'output': 1 means that the var is written to the output stream of this data block, i.e. included in the
# output file.
#
# Please refer to the LI-7200 manual for more details.
#
# BICO ID: IRGA72-B
# Old ID in FCT: li-7200
# Old data block in FCT: data_block_irga_li7200
#
# FCT FluxCalcTool source code:
# https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
#
# byte 1: number of bytes in Licor 7200 record (2 = missing, 26 = available)
# B...unsigned char, integer, 1 Byte
# h...short integer, 2 Bytes
# >...big-endian, MSB at lowest address
#
# """
\ No newline at end of file
# ====================================
# IRGA75-A-GN1 - INFRARED GAS ANALYZER
# ====================================
# Details: see IRGA75-A-GN1.md
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA75-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'status_code_irga', 'datablock': 'IRGA75-A'}
DIAG_VAL == {'order': 3, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'IRGA75-A'}
GA_DIAG_CODE == {'order': 3, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'IRGA75-A'}
H2O_CONC == {'order': 4, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA75-A'}
CO2_CONC == {'order': 5, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 0.974, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA75-A'}
T_BOX == {'order': 6, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 100, 'offset_on_signal': 100, 'apply_gain': 1, 'add_offset': 0, 'units': 'degC', 'datablock': 'IRGA75-A'}
PRESS_BOX == {'order': 7, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 10, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'hPa', 'datablock': 'IRGA75-A'}
COOLER_V == {'order': 8, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'V', 'datablock': 'IRGA75-A'}
# Values extracted from DIAG_VAL:
# Values extracted from GA_DIAG_CODE:
# --------------------------------------
AGC == {'order': 3.04, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 6.25, 'add_offset': 6.25, 'units': '%', 'output': 1, 'datablock': 'IRGA75-A'}
SYNC == {'order': 3.05, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA75-A'}
......
......@@ -24,7 +24,7 @@ Based on IRGA75-A but with a different gain on CO2_CONC.
**A GAIN of 0.974 is applied to this signal to correct for the usage of a wrong calibration gas.
For more info, see here: https://www.swissfluxnet.ethz.ch/index.php/knowledge-base/wrong-calibration-gas-2017/**
- T_BOX ... Ambient temperature measured in the control box
- P_BOX ... Atmospheric pressure measured in the control box
- PRESS_BOX ... Atmospheric pressure measured in the control box
- COOLER_V ... Cooler voltage
## BICO Settings
......
# ================================
# IRGA75-A - INFRARED GAS ANALYZER
# ================================
# Details: see IRGA75-A.md
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA75-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'status_code_irga', 'datablock': 'IRGA75-A'}
......@@ -8,10 +10,10 @@ GA_DIAG_CODE == {'order': 3, 'bytes': 1, 'format': 'B', 'gain_on_signal':
H2O_CONC == {'order': 4, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA75-A'}
CO2_CONC == {'order': 5, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA75-A'}
T_BOX == {'order': 6, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 100, 'offset_on_signal': 100, 'apply_gain': 1, 'add_offset': 0, 'units': 'degC', 'datablock': 'IRGA75-A'}
P_BOX == {'order': 7, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 10, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'hPa', 'datablock': 'IRGA75-A'}
PRESS_BOX == {'order': 7, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 10, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'hPa', 'datablock': 'IRGA75-A'}
COOLER_V == {'order': 8, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'V', 'datablock': 'IRGA75-A'}
# Values extracted from DIAG_VAL:
# Values extracted from GA_DIAG_CODE:
# --------------------------------------
AGC == {'order': 3.04, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 6.25, 'add_offset': 6.25, 'units': '%', 'output': 1, 'datablock': 'IRGA75-A'}
SYNC == {'order': 3.05, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA75-A'}
......
......@@ -21,7 +21,7 @@
- H2O_CONC ... H2O concentration density, molar density
- CO2_CONC ... CO2 concentration density, molar density
- T_BOX ... Ambient temperature measured in the control box
- P_BOX ... Atmospheric pressure measured in the control box
- PRESS_BOX ... Atmospheric pressure measured in the control box
- COOLER_V ... Cooler voltage
## BICO Settings
......
# ============================================
# QCL-A - QUANTUM CASCADE LASER (AERODYNE QCL)
# ============================================
# Details: see QCL-A.md
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'QCL-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'QCL-A'}
CH4_DRY == {'order': 3, 'bytes': 4, 'format': 'B B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'nmol+1 mol-1', 'datablock': 'QCL-A'}
N2O_DRY == {'order': 4, 'bytes': 4, 'format': 'B B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'nmol+1 mol-1', 'datablock': 'QCL-A'}
H2O_DRY == {'order': 5, 'bytes': 4, 'format': 'B B B B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'nmol+1 mol-1', 'datablock': 'QCL-A'}