To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 29f62dd9 authored by holukas's avatar holukas
Browse files

Reworked and added datablocks

parent b26aa6e3
...@@ -2,22 +2,29 @@ ...@@ -2,22 +2,29 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment=""> <list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A3.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A3.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/vis.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/vis.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.dblock" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
...@@ -322,6 +329,9 @@ ...@@ -322,6 +329,9 @@
<workItem from="1609852670010" duration="24000" /> <workItem from="1609852670010" duration="24000" />
<workItem from="1609859126683" duration="11243000" /> <workItem from="1609859126683" duration="11243000" />
<workItem from="1609921205302" duration="15275000" /> <workItem from="1609921205302" duration="15275000" />
<workItem from="1609970074199" duration="1322000" />
<workItem from="1609972599782" duration="2676000" />
<workItem from="1610015969067" duration="13596000" />
</task> </task>
<servers /> <servers />
</component> </component>
...@@ -333,9 +343,14 @@ ...@@ -333,9 +343,14 @@
<breakpoints> <breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line"> <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url> <url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>231</line> <line>241</line>
<option name="timeStamp" value="27" /> <option name="timeStamp" value="27" />
</line-breakpoint> </line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>142</line>
<option name="timeStamp" value="28" />
</line-breakpoint>
</breakpoints> </breakpoints>
<default-breakpoints> <default-breakpoints>
<breakpoint type="python-exception"> <breakpoint type="python-exception">
...@@ -354,7 +369,7 @@ ...@@ -354,7 +369,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1609945717534" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1610055914638" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component> </component>
</project> </project>
\ No newline at end of file
...@@ -15,8 +15,6 @@ from gui.gui import Ui_MainWindow ...@@ -15,8 +15,6 @@ from gui.gui import Ui_MainWindow
from ops import bin, format_data, vis, file, stats from ops import bin, format_data, vis, file, stats
from settings import _version from settings import _version
# TODO hs-50 status code format
# TODO LATER parallelize, multiprocessing? # TODO LATER parallelize, multiprocessing?
# import multiprocessing as mp # import multiprocessing as mp
# print("Number of processors: ", mp.cpu_count()) # print("Number of processors: ", mp.cpu_count())
......
...@@ -175,7 +175,8 @@ class Ui_MainWindow(object): ...@@ -175,7 +175,8 @@ class Ui_MainWindow(object):
# Data Blocks # Data Blocks
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-'] sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'IRGA75-A-GN1', 'LGR-A', 'QCL-C', '-None-'] gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'IRGA75-A-GN1', 'LGR-A',
'QCL-A', 'QCL-A2', 'QCL-A3', 'QCL-C', '-None-']
header_instr_data_blocks = qtw.QLabel('Data Blocks') header_instr_data_blocks = qtw.QLabel('Data Blocks')
header_instr_data_blocks.setProperty('labelClass', 'header_2') header_instr_data_blocks.setProperty('labelClass', 'header_2')
......
...@@ -63,7 +63,6 @@ class ReadInstrDatablock: ...@@ -63,7 +63,6 @@ class ReadInstrDatablock:
# If datablock has the expected size, proceed normally # If datablock has the expected size, proceed normally
if self.dblock_true_size == self.dblock_nominal_size: if self.dblock_true_size == self.dblock_nominal_size:
pass pass
# If datablock does not have the expected size, generate missing data # If datablock does not have the expected size, generate missing data
elif self.dblock_bytes_read == 2: elif self.dblock_bytes_read == 2:
# In this case there are analyzer data missing, i.e. the whole data block is either only 2 Bytes # In this case there are analyzer data missing, i.e. the whole data block is either only 2 Bytes
...@@ -85,13 +84,12 @@ class ReadInstrDatablock: ...@@ -85,13 +84,12 @@ class ReadInstrDatablock:
if self.dblock_true_size != 2: if self.dblock_true_size != 2:
self.read_rest_of_bytes() self.read_rest_of_bytes()
break break
# return self.dblock_data, self.total_bytes_read, self.end_of_data_reached # return self.dblock_data, self.total_bytes_read, self.end_of_data_reached
# Convert to hex or octal if needed # Convert to hex or octal if needed
var_val = self.convert_val(props=props, var_val=var_val) var_val = self.convert_val(props=props, var_val=var_val)
# Add value to data # Add value to data
self.dblock_data.append(var_val) self.dblock_data.append(var_val)
...@@ -108,30 +106,42 @@ class ReadInstrDatablock: ...@@ -108,30 +106,42 @@ class ReadInstrDatablock:
def convert_val(self, props, var_val): def convert_val(self, props, var_val):
"""Convert var value to hex or octal""" """Convert var value to hex or octal"""
# Convert to hex if props['units'] == 'diag_val_hs':
if props['units'] == 'hexadecimal_value': var_val = self.convert_val_to_diag_val_hs(var_val=var_val)
var_val = self.convert_val_to_hex(var_val=var_val)
# Convert to octal
if props['units'] == 'status_code_irga': if props['units'] == 'status_code_irga':
var_val = self.convert_val_to_status_code_irga(var_val=var_val) var_val = self.convert_val_to_status_code_irga(var_val=var_val)
# Convert to octal for LGR laser analyzer
if props['units'] == 'status_code_lgr': if props['units'] == 'status_code_lgr':
var_val = self.convert_val_to_status_code_lgr(var_val=var_val) var_val = self.convert_val_to_status_code_lgr(var_val=var_val)
return var_val return var_val
def convert_val_to_hex(self, var_val): def convert_val_to_diag_val_hs(self, var_val):
"""Convert value to hexadecimal """Convert value to diagnostic value for Gill HS-50 and HS-100 sonic anemometers
Saved as an integer value. The integer can be converted to binary to get more
information. Since the information in this variable is quite complex, please
refer to the sonic anemometer manual for details.
Examples: Examples:
- var_val = 40.0, is converted to integer 40, - var_val = 47.0, is converted to integer 47, this is the value that is
is converted to hex '0x28', is converted to hex without prefix '28' then stored in the converted ASCII file.
- var_val = 60.0, is converted to integer 60,
is converted to hex '0x3c', is converted to hex without prefix '3c' Notes:
- The variable SA_DIAG_TYPE from the sonic HS anmometers gives info
which type of information is given in SA_DIAG_VAL.
- To get more information from the integer, it can be converted to
binary, e.g. integer 47 becomes binary "0b101111"
In Python, this can be done with bin(47) --> "0b101111"
- The first two characters "0b" mean that this is binary format
- The information is contained in "101111", it tells us which bits are
set to 1 or 0. This can be interpreted using the sonic manual, there
is a list what each bit set to 1 or 0 means.
Returns: string Returns: string
""" """
hex_val = hex(int(var_val)) # Note: has hexadecimal prefix '0x' at start, e.g. '0x28' # hex_val = hex(int(var_val)) # Note: has hexadecimal prefix '0x' at start, e.g. '0x28'
hex_val_no_prefix = hex_val[2:] # Remove hex prefix from val # hex_val_no_prefix = hex_val[2:] # Remove hex prefix from val
return hex_val_no_prefix diag_val_hs = int(var_val)
return diag_val_hs
def convert_val_to_status_code_irga(self, var_val): def convert_val_to_status_code_irga(self, var_val):
"""Convert value to octal """Convert value to octal
......
run_id=BICO-20210106-160838 run_id=BICO-20210107-224515
# INSTRUMENTS # INSTRUMENTS
# =========== # ===========
# Site # Site
site=CH-AES site=CH-CHA
# Data Blocks # Data Blocks
header=WECOM3 header=WECOM3
instrument_1=R350-A instrument_1=R350-A
instrument_2=IRGA75-A instrument_2=IRGA75-A
instrument_3=LGR-A instrument_3=QCL-A3
# RAW DATA # RAW DATA
# ======== # ========
# Source Folder # Source Folder
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/05 dir_source=Y:/CH-CHA_Chamau/20_sonic_ghg/2018/11
# Time Range # Time Range
start_date=2020-05-13 13:00 start_date=2018-11-27 09:00
end_date=2020-05-13 13:00 end_date=2018-11-30 23:00
# File Settings # File Settings
filename_datetime_format=yyyymmddHH.XMM filename_datetime_format=yyyymmddHH.CMM
file_ext=*.X* file_ext=*.C*
file_size_min=900 file_size_min=900
file_limit=0 file_limit=0
row_limit=0 row_limit=1000
# Special # Special
select_random_files=0 select_random_files=5
# OUTPUT # OUTPUT
# ====== # ======
dir_out=A:/FLUXES/x-TEST-OUT dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838 dir_out_run=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515
dir_out_run_log=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\log dir_out_run_log=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots dir_out_run_plots=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\hires dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\agg dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\raw_data_csv dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\raw_data_csv
output_folder_name_prefix=AES output_folder_name_prefix=CHA
file_compression=None file_compression=None
plot_file_availability=0 plot_file_availability=0
......
run_id=BICO-20210106-160838 run_id=BICO-20210107-224515
# INSTRUMENTS # INSTRUMENTS
# =========== # ===========
# Site # Site
site=CH-AES site=CH-CHA
# Data Blocks # Data Blocks
header=WECOM3 header=WECOM3
instrument_1=R350-A instrument_1=R350-A
instrument_2=IRGA75-A instrument_2=IRGA75-A
instrument_3=LGR-A instrument_3=QCL-A3
# RAW DATA # RAW DATA
# ======== # ========
# Source Folder # Source Folder
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/05 dir_source=Y:/CH-CHA_Chamau/20_sonic_ghg/2018/11
# Time Range # Time Range
start_date=2020-05-13 13:00 start_date=2018-11-27 09:00
end_date=2020-05-13 13:00 end_date=2018-11-27 23:00
# File Settings # File Settings
filename_datetime_format=yyyymmddHH.XMM filename_datetime_format=yyyymmddHH.CMM
file_ext=*.X* file_ext=*.C*
file_size_min=900 file_size_min=900
file_limit=0 file_limit=1
row_limit=0 row_limit=1000
# Special # Special
select_random_files=0 select_random_files=0
...@@ -35,14 +35,14 @@ select_random_files=0 ...@@ -35,14 +35,14 @@ select_random_files=0
# OUTPUT # OUTPUT
# ====== # ======
dir_out=A:/FLUXES/x-TEST-OUT dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838 dir_out_run=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515
dir_out_run_log=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\log dir_out_run_log=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots dir_out_run_plots=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\hires dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\plots\agg dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\AES_BICO-20210106-160838\raw_data_csv dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210107-224515\raw_data_csv
output_folder_name_prefix=AES output_folder_name_prefix=CHA
file_compression=None file_compression=None
plot_file_availability=0 plot_file_availability=0
......
__version__ = "0.0.5" __version__ = "0.0.6"
__date__ = "6 Jan 2021" __date__ = "7 Jan 2021"
__link_source_code__ = "https://gitlab.ethz.ch/holukas/bico" __link_source_code__ = "https://gitlab.ethz.ch/holukas/bico"
__link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases" __link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases"
__link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home" __link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home"
......
# ========================= # =========================
# HS-100 - SONIC ANEMOMETER # HS-100 - SONIC ANEMOMETER
# ========================= # =========================
# Details: see HS100-A.md
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS100-A'} U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS100-A'}
...@@ -10,42 +11,3 @@ T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'o ...@@ -10,42 +11,3 @@ T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'o
STAA == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'} STAA == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'}
STAD == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'} STAD == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS100-A'} INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS100-A'}
# NOTES
# =====
# (!) HS100-A HAS THE SAME BINARY FORMAT AS HS50-A
# (!) THE HS-100 HAS THE SAME BINARY FORMAT AS THE HS-50
#
# 'gain_on_signal' ... gain that was applied to the raw data signal, to get to units *divide* by this gain
# 'offset_on_signal' ... offset that was added to the raw data signal, to get to units *subtract* this offset
# 'apply_gain' ... gain that is applied during conversion, e.g. to convert to different units if needed
# 'add_offset' ... offset that is added during conversion, e.g. to convert to different units if needed
#
# BICO ID: HS100-A
# Old ID in FCT: hs_100
# Old data block in FCT: data_block_sonic_hs_100
#
# FCT FluxCalcTool source code:
# https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
#
# Extended data logging to comply w/ ICOS requirements.
# Installed in CH-DAV in 2017-07.
#
# from Werner Eugster email 2017-07-10:
# the former 4 data bytes containing the two inclinometer angles are now used differently
# * first byte is the StaA byte from the sonic
# * second byte is the StaD byte from the sonic
# * third and fourth byte are still a short integer as before, but now
# it contains the inclinometer X angle if the record number is an odd number,
# or the inclinometer Y angle if the record number is an even number
#
# https://docs.python.org/3.1/library/struct.html#format-characters
# B...unsigned char, integer, 1 Byte
# h...short integer, 2 Bytes
# >...big-endian, MSB most-significant Byte at lowest address
# Big-endian systems store the most significant byte of a word in the smallest address
# 2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements
#
# # https://docs.python.org/3/library/struct.html
# """
\ No newline at end of file
# HS100-A
(!) HS100-A HAS THE SAME BINARY FORMAT AS HS50-A
(!) THE HS-100 HAS THE SAME BINARY FORMAT AS THE HS-50
## Variables
- U ... First horizontal wind component (x)
- V ... Second horizontal wind component (y)
- W ... Vertical wind component (z)
- T_SONIC ... Sonic temperature
- SA_DIAG_TYPE ... Status type indicator (error, inclinometer, configuration)
- Called ```StaA``` (Status Address) in manual
- defines which particular aspect of the anemometer the following data field (SA_DIAG_VAL) refers to
- 00 Error codes
- 01 Anemometer configuration
- 02 Data output configuration 1
- 03 Data output configuration 2
- 04 Error code history
- 05 Transducer gain levels
- 06 Anemometer Type
- 07 Inclinometer X axis MSB
- 08 Inclinometer X axis LSB
- 09 Inclinometer Y axis MSB
- 10 Inclinometer Y axis LSB
- SA_DIAG_VAL ... Status value and information
- Called ```StaD``` (Status Data) in manual
- SA_DIAG_VAL is rather complex and can tak on a lot of meanings.
- Has to be interpreted in combination with SA_DIAG_TYPE, which gives the type
of SA_DIAG_VAL.
- Saved as an integer value, e.g. ```47```.
- The integer can be converted to binary, e.g. ```47``` becomes ```0b101111``` , whereby the
first two characters ```0b``` mean that this is binary format.
- From the binary, detailed information can be extracted, depending on which bits
in the binary are set to 1 or 0.
- Please refer to the manual for detailed information about what the different bits mean
when they are 1 or zero.
- INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers)
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
## BICO Settings
- 'order' ... Order in the data block variable sequence
- 'bytes' ... Number of bytes in the binary data
- 'format' ... Format to convert from binary to integers, floats, etc.
- 'gain_on_signal' ... Gain that was applied to the raw data signal, to get to units *divide* by this gain
- 'offset_on_signal' ... Offset that was added to the raw data signal, to get to units *subtract* this offset
- 'apply_gain' ... Gain that is applied during conversion, e.g. to convert to different units if needed
- 'add_offset' ... Offset that is added during conversion, e.g. to convert to different units if needed
- 'units' ... Units
- 'datablock' ... Data block ID
*Before BICO, the binary conversion was done in FCT FluxCalcTool:*
- Old ID in FCT: hs_100
- Old data block in FCT: data_block_sonic_hs_100
- FCT Source code: --> https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool
## Details
- Extended data logging to comply w/ ICOS requirements.
- from WE's email 2017-07-10:
- the former 4 data bytes containing the two inclinometer angles are now used differently
- first byte is the StaA byte from the sonic
- second byte is the StaD byte from the sonic
- third and fourth byte are still a short integer as before, but now
it contains the inclinometer X angle if the record number is an odd number,
or the inclinometer Y angle if the record number is an even number
## Binary info
- B...unsigned char, integer, 1 Byte
- h...short integer, 2 Bytes
- ">"...big-endian, MSB most-significant Byte at lowest address
Big-endian systems store the most significant byte of a word in the smallest address
- --> https://docs.python.org/3/library/struct.html
- --> https://docs.python.org/3.1/library/struct.html#format-characters
...@@ -3,10 +3,11 @@ ...@@ -3,10 +3,11 @@
# ====================================== # ======================================
# Details: see HS50-A.md # Details: see HS50-A.md
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'} U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'} V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'} W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'datablock': 'HS50-A'} T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'datablock': 'HS50-A'}
SA_DIAG_TYPE == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS50-A'} SA_DIAG_TYPE == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs_type', 'datablock': 'HS50-A'}
SA_DIAG_VAL == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'hexadecimal_value', 'datablock': 'HS50-A'} SA_DIAG_VAL == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs', 'datablock': 'HS50-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS50-A'} INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS50-A'}
...@@ -21,7 +21,16 @@ ...@@ -21,7 +21,16 @@
- 10 Inclinometer Y axis LSB - 10 Inclinometer Y axis LSB
- SA_DIAG_VAL ... Status value and information - SA_DIAG_VAL ... Status value and information
- Called ```StaD``` (Status Data) in manual - Called ```StaD``` (Status Data) in manual
- Please refer to the manual for detailed information about meanings - SA_DIAG_VAL is rather complex and can tak on a lot of meanings.
- Has to be interpreted in combination with SA_DIAG_TYPE, which gives the type
of SA_DIAG_VAL.
- Saved as an integer value, e.g. ```47```.
- The integer can be converted to binary, e.g. ```47``` becomes ```0b101111``` , whereby the
first two characters ```0b``` mean that this is binary format.
- From the binary, detailed information can be extracted, depending on which bits
in the binary are set to 1 or 0.
- Please refer to the manual for detailed information about what the different bits mean
when they are 1 or zero.
- INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers) - INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers)
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements* *2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
......
# ================================
# IRGA72-B - INFRARED GAS ANALYZER
# ================================
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-B'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'IRGA72-B'}
DIAG_VAL == {'order': 3, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'bit_map', 'datablock': 'IRGA72-B'}
H2O_DMF == {'order': 4, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 mol-1', 'datablock': 'IRGA72-B'}
CO2_DMF == {'order': 5, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'umol+1 mol-1', 'datablock': 'IRGA72-B'}
H2O_MD == {'order': 6, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA72-B'}
CO2_MD == {'order': 7, 'bytes': 3, 'format': 'B B B', 'gain_on_signal': 10000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'mmol+1 m-3', 'datablock': 'IRGA72-B'}
T_CELL == {'order': 8, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 100, 'offset_on_signal': 100, 'apply_gain': 1, 'add_offset': 0, 'units': 'degC', 'datablock': 'IRGA72-B'}
P_CELL == {'order': 9, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 0.1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Pa', 'datablock': 'IRGA72-B'}
P_ATM == {'order': 10, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 0.1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Pa', 'datablock': 'IRGA72-B'}
COOLER_V == {'order': 11, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'V', 'datablock': 'IRGA72-B'}
FLOWRATE == {'order': 12, 'bytes': 2, 'format': 'B B', 'gain_on_signal': 1000, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'L+1 min-1', 'datablock': 'IRGA72-B'}
# Values extracted from DIAG_VAL:
# --------------------------------------
UNUSED == {'order': 3.01, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 1, 'add_offset': 0, 'units': 'unused', 'output': 0, 'datablock': 'IRGA72-B'}
HEAD_DETECT == {'order': 3.02, 'bit_pos_start': 3, 'bit_pos_end': 4, 'apply_gain': 1, 'add_offset': 0, 'units': '1=LI-7200', 'output': 0, 'datablock': 'IRGA72-B'}
T_OUTLET == {'order': 3.03, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
T_INLET == {'order': 3.04, 'bit_pos_start': 5, 'bit_pos_end': 6, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}
AUX_INPUT == {'order': 3.05, 'bit_pos_start': 6, 'bit_pos_end': 7, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-B'}