To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 6dd802c1 authored by holukas's avatar holukas
Browse files

Added data block for IRGA75 gain correction

parent a8eb4c49
......@@ -6,7 +6,6 @@
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/changelog.txt" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/changelog.txt" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/version_info.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/version_info.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/data_blocks.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/data_blocks.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/func.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/func.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/settings.txt" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/settings.txt" afterDir="false" />
</list>
......@@ -36,6 +35,7 @@
<property name="options.searchVisible" value="true" />
<property name="options.splitter.details.proportions" value="0.2" />
<property name="options.splitter.main.proportions" value="0.3" />
<property name="restartRequiresConfirmation" value="false" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="PyConsoleOptionsProvider">
......@@ -258,6 +258,11 @@
<workItem from="1588362735628" duration="3381000" />
<workItem from="1588457525082" duration="176000" />
<workItem from="1590535471695" duration="1172000" />
<workItem from="1590539922986" duration="1349000" />
<workItem from="1590542169215" duration="719000" />
<workItem from="1590573142877" duration="4000" />
<workItem from="1591099717915" duration="2895000" />
<workItem from="1591112560073" duration="387000" />
</task>
<servers />
</component>
......@@ -273,26 +278,42 @@
</option>
</component>
<component name="WindowStateProjectService">
<state width="2517" height="509" key="GridCell.Tab.0.bottom" timestamp="1590536647837">
<state width="2517" height="509" key="GridCell.Tab.0.bottom" timestamp="1591112946980">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.bottom/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state width="2517" height="509" key="GridCell.Tab.0.center" timestamp="1590536647837">
<state width="2517" height="509" key="GridCell.Tab.0.bottom/0.0.2560.1400@0.0.2560.1400" timestamp="1591112946980" />
<state width="2517" height="509" key="GridCell.Tab.0.center" timestamp="1591112946980">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.center/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state width="2517" height="509" key="GridCell.Tab.0.left" timestamp="1590536647837">
<state width="2517" height="509" key="GridCell.Tab.0.center/0.0.2560.1400@0.0.2560.1400" timestamp="1591112946980" />
<state width="2517" height="509" key="GridCell.Tab.0.left" timestamp="1591112946980">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.left/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state width="2517" height="509" key="GridCell.Tab.0.right" timestamp="1590536647837">
<state width="2517" height="509" key="GridCell.Tab.0.left/0.0.2560.1400@0.0.2560.1400" timestamp="1591112946980" />
<state width="2517" height="509" key="GridCell.Tab.0.right" timestamp="1591112946980">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.right/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state x="448" y="189" width="1573" height="1030" key="find.popup" timestamp="1590536145757">
<state width="2517" height="509" key="GridCell.Tab.0.right/0.0.2560.1400@0.0.2560.1400" timestamp="1591112946980" />
<state width="2369" height="431" key="GridCell.Tab.1.bottom" timestamp="1590541275819">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state x="448" y="189" width="1573" height="1030" key="find.popup/0.0.2560.1400@0.0.2560.1400" timestamp="1590536145757" />
<state width="2369" height="431" key="GridCell.Tab.1.bottom/0.0.2560.1400@0.0.2560.1400" timestamp="1590541275819" />
<state width="2369" height="431" key="GridCell.Tab.1.center" timestamp="1590541275819">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2369" height="431" key="GridCell.Tab.1.center/0.0.2560.1400@0.0.2560.1400" timestamp="1590541275819" />
<state width="2369" height="431" key="GridCell.Tab.1.left" timestamp="1590541275819">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2369" height="431" key="GridCell.Tab.1.left/0.0.2560.1400@0.0.2560.1400" timestamp="1590541275819" />
<state width="2369" height="431" key="GridCell.Tab.1.right" timestamp="1590541275819">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2369" height="431" key="GridCell.Tab.1.right/0.0.2560.1400@0.0.2560.1400" timestamp="1590541275819" />
<state x="448" y="189" width="1573" height="1030" key="find.popup" timestamp="1591101833170">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state x="448" y="189" width="1573" height="1030" key="find.popup/0.0.2560.1400@0.0.2560.1400" timestamp="1591101833170" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
......@@ -302,10 +323,20 @@
<line>25</line>
<option name="timeStamp" value="1" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/setup.py</url>
<line>222</line>
<option name="timeStamp" value="2" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/setup.py</url>
<line>185</line>
<option name="timeStamp" value="5" />
</line-breakpoint>
</breakpoints>
</breakpoint-manager>
</component>
<component name="com.intellij.coverage.CoverageDataManagerImpl">
<SUITE FILE_PATH="coverage/FCT_Flux_Calc_Tool$start_FCT__1_.coverage" NAME="start_FCT (1) Coverage Results" MODIFIED="1590536550332" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/fct" />
<SUITE FILE_PATH="coverage/FCT_Flux_Calc_Tool$start_FCT__1_.coverage" NAME="start_FCT (1) Coverage Results" MODIFIED="1591112581984" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/fct" />
</component>
</project>
\ No newline at end of file
......@@ -2,15 +2,25 @@
* CHANGELOG *
*************
v 0.9.6, 2020-06-02
* added support for data block to correct for use of wrong calibration gas (LI-7200 in the NOT extended setting):
data_block_irga_li7200_co2_gain0974
* added "if data_block_order_id[x] == 'li-7500'" and "elif data_block_order_id[x] == 'li-7500_co2_gain0974'"
in setup.py to be more specific with LI-7500 data blocks. This should have been done already in v0.9.5, but
since the column order is the same in both cases there should be no issues.
v 0.9.5, 2020-05-27
* added support for data block to correct for use of wrong calibration gas (LI-7500):
data_block_irga_li7500_extended_co2_gain0974
* changed version naming convention
v 0.94, 2020-05-01
* added support for data block to correct for use of wrong calibration gas (LI-7200):
data_block_irga_li7200_extended_co2_gain0974
v 0.93, 2020-01-24
* updated EddyPro to version 7.0.6 (18 Dec 2019)
......
def version_info():
version_info_dict = {
'FluxCalcTool': 'v0.9.5',
'FluxCalcTool': 'v0.9.6',
'EddyPro': 'v7.0.6 (18 Dec 2019)',
'last edit': '27 May 2020',
'last edit': '02 Jun 2020',
'source code': 'https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool'
}
......
......@@ -909,9 +909,14 @@ def data_block_irga_li7200_extended_co2_gain0974(open_file_object): # size: 12
"""Same as data_block_irga_li7200_extended, but with an additional gain applied to
the CO2 data columns. Gain is necessary in some years when a wrong calibration gas
was used to calibrate the LI-7200.
Instrument ID in EddyPro:
li-7200_extended_co2_gain0974
Data block added in v0.94
More info here:
https://www.swissfluxnet.ethz.ch/index.php/knowledge-base/wrong-calibration-gas-2017/
Data block added in v0.94
"""
calib_gas_corr_gain = 0.974
# byte 1: number of bytes in Licor 7200 record (2 = missing, 25 = available)
......@@ -1051,6 +1056,109 @@ def data_block_irga_li7200_extended_co2_gain0974(open_file_object): # size: 12
'-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
def data_block_irga_li7200_co2_gain0974(open_file_object): # size: 12 columns / 25 bytes
"""Apply gain 0.974 to CO2 concentration measurements to correct for the
usage of a wrong calibration gas.
Note that this data block corrects the *NOT* EXTENDED LI-7200 data stream.
Instrument ID in EddyPro:
li7200_co2_gain0974
Data block added in v0.9.6
For more info see here:
https://www.swissfluxnet.ethz.ch/index.php/knowledge-base/wrong-calibration-gas-2017/
"""
calib_gas_corr_gain = 0.974
# byte 1: number of bytes in Licor 7200 record (2 = missing, 25 = available)
byte = open_file_object.read(2)
if byte: # if there are data to read
s = struct.Struct('B B')
if len(byte) == 2:
unpacked_data = s.unpack(byte)
data_size = unpacked_data[0]
status_code = unpacked_data[1]
if data_size == 25: # if no missing data
byte = open_file_object.read(
23) # read next 23 bytes for a total of 25 bytes / byte 3 is OCTAL, todo correct output for octal
if len(byte) == 23: # check if we really have 23 more bytes
s = struct.Struct('B B B B B B B B B B B B B B B B B B B B B B B')
unpacked_data = s.unpack(byte)
status_byte = unpacked_data[
0] # this is the AGC, after firmware update 100 = good, before update 100 = bad
binary = bin(status_byte)[2:].zfill(8)
lower_nibble = binary[4:]
decimal = int(str(lower_nibble), 2)
status_byte = (decimal * 6.25) # + 6.25 # AGC, window dirtiness, offset not used in ETH Flux
h2o_ppt = 0.001 * ((unpacked_data[1] * 256 * 256) + (unpacked_data[2] * 256) + unpacked_data[
3]) # dry mole fraction, mmol mol-1
# CO2 mixing ratio (dry mole fraction), umol mol-1
co2_ppm = \
0.0001 * ((unpacked_data[4] * 256 * 256) + (unpacked_data[5] * 256) + unpacked_data[6])
co2_ppm = co2_ppm * calib_gas_corr_gain
h2o_mmol_m3 = 0.001 * ((unpacked_data[7] * 256 * 256) + (unpacked_data[8] * 256) + unpacked_data[9])
# CO2 molar density, mmol m-3
co2_mmol_m3 = \
0.0001 * ((unpacked_data[10] * 256 * 256) + (unpacked_data[11] * 256) + unpacked_data[12])
co2_mmol_m3 = co2_mmol_m3 * calib_gas_corr_gain
temp_deg_c = 0.01 * ((unpacked_data[13] * 256) + unpacked_data[14]) # K
temp_deg_c -= 100 # K, offset
press_hpa = 10 * ((unpacked_data[15] * 256) + unpacked_data[16]) # Pa
press_hpa *= 0.01 # to hPa
atm_press_hpa = 10 * ((unpacked_data[17] * 256) + unpacked_data[18]) # Pa
atm_press_hpa *= 0.01 # to hPa
cooler_v = 0.001 * ((unpacked_data[19] * 256) + (unpacked_data[20]))
flow_l_min = 0.001 * ((unpacked_data[21] * 256) + (unpacked_data[22]))
return data_size, status_code, status_byte, h2o_ppt, co2_ppm, h2o_mmol_m3, co2_mmol_m3, temp_deg_c, press_hpa, atm_press_hpa, cooler_v, flow_l_min
else:
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
else:
# -----------------------------------------------
# this part was included in v 0.86,
# originally for li7200_extended but I also include this here for the "normal" li7200
# necessary b/c in CH-DAV starting in Oct 2018 sometimes a 16 Byte data_size is detected
# the exact reason for this is unknown, but it seems like the sonicread script tries
# to store an IRGA75 data block (but IRGA75 is not installed at the site)
# therefore, in this part here, we look at the data size that is given in the file (e.g. 16 Bytes),
# then read that given number of Bytes minus 2 bytes (b/c the first two Bytes were already read)
# from the file, and then output the normal missing data header.
# a similar case was reported above for the IRGA75 (see description in data block).
# -----------------------------------------------
if data_size == 2:
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', \
'-9999', '-9999', '-9999', '-9999', '-9999'
elif data_size > 0:
# in case the data size is different than the typical 2 or 25 Bytes, the given number
# of Bytes is read from the file and the regular header is output
for xx in range(0, data_size - 2):
byte = open_file_object.read(1)
if len(byte) == 1:
s = struct.Struct('B')
unpacked_data = s.unpack(byte)
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', \
'-9999', '-9999', '-9999', '-9999', '-9999'
elif data_size == 0:
print("!ATTENTION! IRGA LI-7200 data size is not 2 or 25 bytes. Found data_size = 0.")
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', \
'-9999', '-9999', '-9999', '-9999', '-9999'
# There was one problem that could not be solved at the time the data acquisition was written
# and which affects all data collected until this may (or may not) be solved in a future version: every
# 20th record or so a flow rate of 0.0 L min −1 is reported which leads to an erroneous CO 2 and H 2 O
# concentration in the primary variable(the dry mole fraction) whereas the mole density is OK.
def data_block_irga_li7200(open_file_object): # size: 12 columns / 25 bytes
# print('------- Li7200 IRGA information')
# byte 1: number of bytes in Licor 7200 record (2 = missing, 25 = available)
......
......@@ -205,12 +205,17 @@ def detect_instruments(root_dir): # uses the eddypro metadata file to detect in
if found_instruments_id[x] == 'li-7200':
data_block_order.append('data_block_irga_li7200')
data_block_order_id.append(found_instruments_id[x])
elif found_instruments_id[x] == 'li-7200_co2_gain0974':
data_block_order.append('data_block_irga_li7200_co2_gain0974')
data_block_order_id.append(found_instruments_id[x])
elif found_instruments_id[x] == 'li-7200_extended':
data_block_order.append('data_block_irga_li7200_extended')
data_block_order_id.append(found_instruments_id[x])
elif found_instruments_id[x] == 'li-7200_extended_co2_gain0974':
data_block_order.append('data_block_irga_li7200_extended_co2_gain0974')
data_block_order_id.append(found_instruments_id[x])
elif found_instruments_id[x] == 'li-7200_extended_synctest': # new in 0.87
data_block_order.append('data_block_irga_li7200_extended_synctest')
data_block_order_id.append(found_instruments_id[x])
......@@ -569,9 +574,18 @@ def construct_header(data_block_order, data_block_order_id):
if 'csat3' in data_block_order[x]:
header_csv = ('u_ms-1', 'v_ms-1', 'w_ms-1', 'ts_C', 'counter') # , 'delimiters')
# LI-7500
if 'li7500' in data_block_order[x]:
header_csv += ('data_size_75', 'status_code_75', 'status_byte_75', 'H2O_mmol_m-3_75',
'CO2_mmol_m-3_75', 'temp_degC_75', 'press_hPa_75', 'cooler_V_75')
if data_block_order_id[x] == 'li-7500':
header_csv += ('data_size_75', 'status_code_75', 'status_byte_75', 'H2O_mmol_m-3_75',
'CO2_mmol_m-3_75', 'temp_degC_75', 'press_hPa_75', 'cooler_V_75')
elif data_block_order_id[x] == 'li-7500_co2_gain0974':
# v0.9.6, but would also have been valid for v0.9.5, but columns are the same anyway: same as
# 'li-7500', but gain of 0.974 is applied to CO2 columns due to usage of a wrong calibration
# gas between 2017-2019
# https://www.swissfluxnet.ethz.ch/index.php/knowledge-base/wrong-calibration-gas-2017/
header_csv += ('data_size_75', 'status_code_75', 'status_byte_75', 'H2O_mmol_m-3_75',
'CO2_mmol_m-3_75', 'temp_degC_75', 'press_hPa_75', 'cooler_V_75')
# Gill HS-50
if 'hs_50' in data_block_order[x]:
......@@ -588,6 +602,13 @@ def construct_header(data_block_order, data_block_order_id):
header_csv += ('data_size_72', 'status_code_72', 'status_byte_72', 'H2O_ppt_72', 'CO2_ppm_72',
'H2O_mmol_m-3_72', 'CO2_mmol_m-3_72', 'temp_degC_72', 'press_hPa_72', 'atm_press_hPa_72',
'cooler_V_72', 'flow_l_min-1_72')
elif data_block_order_id[x] == 'li-7200_co2_gain0974':
# v0.9.6: same as 'li-7200', but gain of 0.974 is applied to CO2 columns due
# to usage of a wrong calibration gas between 2017-2019
# https://www.swissfluxnet.ethz.ch/index.php/knowledge-base/wrong-calibration-gas-2017/
header_csv += ('data_size_72', 'status_code_72', 'status_byte_72', 'H2O_ppt_72', 'CO2_ppm_72',
'H2O_mmol_m-3_72', 'CO2_mmol_m-3_72', 'temp_degC_72', 'press_hPa_72', 'atm_press_hPa_72',
'cooler_V_72', 'flow_l_min-1_72')
elif data_block_order_id[x] == 'li-7200_extended':
header_csv += (
......
......@@ -10,9 +10,9 @@ plot_diurnal_cycles_output=0
use_rds=0
fluxes_from=2018-12-31 00:00
fluxes_until=2019-12-31 23:59
raw_binary_file_extension=*.X*
raw_binary_file_extension=*.L*
raw_binary_min_filesize_bytes=900
site_to_process=CH-FRU
site_to_process=CH-LAE
server_raw_binary_source_dir=\\grasslandserver.ethz.ch\archive\FluxData\CH-INO_InnoFarm\20_sonic_ghg
AWS_raw_binary_source_dir=\\grasslandserver.ethz.ch\archive\FluxData\CH-AWS_CrapAlv\20_sonic_ghg
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment