To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit a8eb4c49 authored by holukas's avatar holukas
Browse files

Added data block for LI-7500 gain correction

parent c35cdad1
# Custom
_archive/
_raw_photoshop/
_eddypro_settings/
# Byte-compiled / optimized / DLL files
__pycache__/
......
......@@ -2,13 +2,11 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="163fe782-73ec-4016-81ee-3e0f2612cbf8" name="Default" comment="">
<change afterPath="$PROJECT_DIR$/.idea/vcs.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/FCT_FluxCalcTool.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/FCT_FluxCalcTool.iml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/misc.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/changelog.txt" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/changelog.txt" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/version_info.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/info/version_info.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/data_blocks.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/data_blocks.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/func.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/func.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/python_scripts/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/settings.txt" beforeDir="false" afterPath="$PROJECT_DIR$/fct/FCT_FluxCalcTool/settings.txt" afterDir="false" />
</list>
......@@ -23,12 +21,6 @@
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="ProjectFrameBounds">
<option name="x" value="-6" />
<option name="y" value="-6" />
<option name="width" value="1932" />
<option name="height" value="1172" />
</component>
<component name="ProjectId" id="1bJutfkA4B96OsaOvpIVwvZ7cee" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
......@@ -264,6 +256,8 @@
<option name="presentableId" value="Default" />
<updated>1461751930499</updated>
<workItem from="1588362735628" duration="3381000" />
<workItem from="1588457525082" duration="176000" />
<workItem from="1590535471695" duration="1172000" />
</task>
<servers />
</component>
......@@ -279,26 +273,26 @@
</option>
</component>
<component name="WindowStateProjectService">
<state width="2517" height="509" key="GridCell.Tab.0.bottom" timestamp="1588366876669">
<state width="2517" height="509" key="GridCell.Tab.0.bottom" timestamp="1590536647837">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.bottom/0.0.2560.1400@0.0.2560.1400" timestamp="1588366876669" />
<state width="2517" height="509" key="GridCell.Tab.0.center" timestamp="1588366876669">
<state width="2517" height="509" key="GridCell.Tab.0.bottom/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state width="2517" height="509" key="GridCell.Tab.0.center" timestamp="1590536647837">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.center/0.0.2560.1400@0.0.2560.1400" timestamp="1588366876669" />
<state width="2517" height="509" key="GridCell.Tab.0.left" timestamp="1588366876669">
<state width="2517" height="509" key="GridCell.Tab.0.center/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state width="2517" height="509" key="GridCell.Tab.0.left" timestamp="1590536647837">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.left/0.0.2560.1400@0.0.2560.1400" timestamp="1588366876669" />
<state width="2517" height="509" key="GridCell.Tab.0.right" timestamp="1588366876669">
<state width="2517" height="509" key="GridCell.Tab.0.left/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state width="2517" height="509" key="GridCell.Tab.0.right" timestamp="1590536647837">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state width="2517" height="509" key="GridCell.Tab.0.right/0.0.2560.1400@0.0.2560.1400" timestamp="1588366876669" />
<state x="448" y="189" width="1573" height="1030" key="find.popup" timestamp="1588366635335">
<state width="2517" height="509" key="GridCell.Tab.0.right/0.0.2560.1400@0.0.2560.1400" timestamp="1590536647837" />
<state x="448" y="189" width="1573" height="1030" key="find.popup" timestamp="1590536145757">
<screen x="0" y="0" width="2560" height="1400" />
</state>
<state x="448" y="189" width="1573" height="1030" key="find.popup/0.0.2560.1400@0.0.2560.1400" timestamp="1588366635335" />
<state x="448" y="189" width="1573" height="1030" key="find.popup/0.0.2560.1400@0.0.2560.1400" timestamp="1590536145757" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
......@@ -312,6 +306,6 @@
</breakpoint-manager>
</component>
<component name="com.intellij.coverage.CoverageDataManagerImpl">
<SUITE FILE_PATH="coverage/FCT_Flux_Calc_Tool$start_FCT__1_.coverage" NAME="start_FCT (1) Coverage Results" MODIFIED="1588365861671" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/fct" />
<SUITE FILE_PATH="coverage/FCT_Flux_Calc_Tool$start_FCT__1_.coverage" NAME="start_FCT (1) Coverage Results" MODIFIED="1590536550332" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/fct" />
</component>
</project>
\ No newline at end of file
......@@ -2,6 +2,11 @@
* CHANGELOG *
*************
v 0.9.5, 2020-05-27
* added support for data block to correct for use of wrong calibration gas (LI-7500):
data_block_irga_li7500_extended_co2_gain0974
* changed version naming convention
v 0.94, 2020-05-01
* added support for data block to correct for use of wrong calibration gas (LI-7200):
data_block_irga_li7200_extended_co2_gain0974
......
def version_info():
version_info_dict = {
'FluxCalcTool': 'v0.94',
'FluxCalcTool': 'v0.9.5',
'EddyPro': 'v7.0.6 (18 Dec 2019)',
'last edit': '1 May 2020'
'last edit': '27 May 2020',
'source code': 'https://gitlab.ethz.ch/holukas/fct-flux-calculation-tool'
}
return version_info_dict
......@@ -439,6 +439,82 @@ def data_block_irga_li7500(open_file_object): # size: 16 bytes
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
def data_block_irga_li7500_co2_gain0974(open_file_object): # size: 16 bytes
"""Apply gain 0.974 to CO2 concentration measurements to correct for the
usage of a wrong calibration gas.
Data block added in v0.9.5
For more info see here:
https://www.swissfluxnet.ethz.ch/index.php/knowledge-base/wrong-calibration-gas-2017/
"""
# byte 1: number of bytes in Licor 7500 record (2 = missing, 16 = available)
calib_gas_corr_gain = 0.974
byte = open_file_object.read(2)
if byte: # if there are data to read
s = struct.Struct('B B')
if len(byte) == 2:
unpacked_data = s.unpack(byte)
data_size = unpacked_data[0]
status_code = unpacked_data[1]
if data_size == 16: # if no missing data
byte = open_file_object.read(14)
if len(byte) == 14: # check if we really have 14 more bytes for li75
# byte = open_file_object.read(14) # for checking only
s = struct.Struct('B B B B B B B B B B B B B B')
unpacked_data = s.unpack(byte)
# print(unpacked_data) # for checking only
status_byte = unpacked_data[0] # this is the AGC (window dirtiness)
binary = bin(status_byte)[2:].zfill(8)
lower_nibble = binary[4:]
decimal = int(str(lower_nibble), 2)
status_byte = (decimal * 6.25) # + 6.25 # AGC, window dirtiness, offset not used in ETH Flux
h2o_mmol_m3 = 0.001 * ((unpacked_data[1] * 256 * 256) + (unpacked_data[2] * 256) + unpacked_data[3])
co2_mmol_m3 = 0.0001 * (
(unpacked_data[4] * 256 * 256) + (unpacked_data[5] * 256) + unpacked_data[6])
co2_mmol_m3 = co2_mmol_m3 * calib_gas_corr_gain
temp_deg_c = 0.01 * ((unpacked_data[7] * 256) + unpacked_data[8]) # K
temp_deg_c -= 100 # K, offset
press_hpa = 10 * ((unpacked_data[9] * 256) + unpacked_data[10]) # Pa
press_hpa *= 0.01 # to hPa
cooler_v = 0.0001 * (
(unpacked_data[11] * 256 * 256) + (unpacked_data[12] * 256) + unpacked_data[13])
return data_size, status_code, status_byte, h2o_mmol_m3, co2_mmol_m3, temp_deg_c, press_hpa, cooler_v
else: # this can be the case if data storage stopped abruptly
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
else:
if data_size == 2: # 2015-04-22 # this is the normal case when rest of records are missing
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
elif data_size > 0: # 2015-04-22 # 2015-04-24
# print("!ATTENTION! IRGA LI-7500 data size is not 2 or 16 bytes. Found data_size = " + str(data_size) + ".") todo
# this is implemented due to a problem in data logging at the field site CH-DAV
# during the time periods 2012b and 2013a
# where it was / is possible that two Li-7200 data blocks arrive in succession
# in the binary file, i.e. the sequence for one line was:
# r3-li72-li72 (the li75 was missing at the time)
# to account for this problem we just read the data size of the li72 (25 Bytes total)
# but output the missing values block for the li75
# this means, our "reading window" does not shift and is in the right order;
# note that this generates data_size = 25 in the output file
# and not 16 OR 2 like usual for the li75
for xx in range(0, data_size - 2):
# print(xx)
# construct binary reader for data_size number of bytes
byte = open_file_object.read(1)
if len(byte) == 1: # 2015-04-24
s = struct.Struct('B')
unpacked_data = s.unpack(byte)
# print(unpacked_data)
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
elif data_size == 0: # 2015-04-24
print("!ATTENTION! IRGA LI-7500 data size is not 2 or 16 bytes. Found data_size = 0.")
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
def data_block_sonic_hs_50_extended(open_file_object): # size: 12 bytes
"""
Extended data logging to comply w/ ICOS requirements.
......@@ -828,6 +904,7 @@ def data_block_irga_li7200_extended(open_file_object): # size: 12 columns / 25
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', \
'-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
def data_block_irga_li7200_extended_co2_gain0974(open_file_object): # size: 12 columns / 25 bytes
"""Same as data_block_irga_li7200_extended, but with an additional gain applied to
the CO2 data columns. Gain is necessary in some years when a wrong calibration gas
......@@ -973,6 +1050,7 @@ def data_block_irga_li7200_extended_co2_gain0974(open_file_object): # size: 12
return data_size, status_code, '-9999', '-9999', '-9999', '-9999', '-9999', \
'-9999', '-9999', '-9999', '-9999', '-9999', '-9999'
def data_block_irga_li7200(open_file_object): # size: 12 columns / 25 bytes
# print('------- Li7200 IRGA information')
# byte 1: number of bytes in Licor 7200 record (2 = missing, 25 = available)
......
......@@ -8,11 +8,14 @@ def intro_screen():
version_info_dict = version_info.version_info()
print("--------------------------")
print("ETH FluxCalc using EddyPro")
print("FluxCalcTool: {}".format(version_info_dict['FluxCalcTool']))
print("EddyPro: {}".format(version_info_dict['EddyPro']))
print("last edit: {}".format(version_info_dict['last edit']))
print("==============================")
print("ETH FluxCalcTool using EddyPro")
print("==============================")
print("\n")
print(f"FluxCalcTool: {version_info_dict['FluxCalcTool']}")
print(f"EddyPro: {version_info_dict['EddyPro']}")
print(f"last edit: {version_info_dict['last edit']}")
print(f"source code: {version_info_dict['source code']}")
print("--------------------------")
print("\n")
print(" ,,,,,,")
......
......@@ -215,15 +215,22 @@ def detect_instruments(root_dir): # uses the eddypro metadata file to detect in
data_block_order.append('data_block_irga_li7200_extended_synctest')
data_block_order_id.append(found_instruments_id[x])
if 'li7500' in found_instruments[x]:
if found_instruments_id[x] == 'li-7500':
data_block_order.append('data_block_irga_li7500')
data_block_order_id.append(found_instruments_id[x])
elif found_instruments_id[x] == 'li-7500_co2_gain0974':
data_block_order.append('data_block_irga_li7500_co2_gain0974')
data_block_order_id.append(found_instruments_id[x])
if 'hs_100' in found_instruments[x]:
data_block_order.append('data_block_sonic_hs_100')
data_block_order_id.append(found_instruments_id[x])
if 'csat3' in found_instruments[x]:
data_block_order.append('data_block_sonic_csat3')
data_block_order_id.append(found_instruments_id[x])
if 'li7500' in found_instruments[x]:
data_block_order.append('data_block_irga_li7500')
data_block_order_id.append(found_instruments_id[x])
if 'r2_1' in found_instruments[x]: # EddyPro uses 'r2_1' to describe the sonic r2_1
# earlier versions of sonicreadHS recorded an extra Byte in the data record
......
......@@ -8,11 +8,11 @@ calc_fluxes=0
plot_full_output=0
plot_diurnal_cycles_output=0
use_rds=0
fluxes_from=2017-12-31 00:00
fluxes_until=2018-12-31 23:59
raw_binary_file_extension=*.o*
fluxes_from=2018-12-31 00:00
fluxes_until=2019-12-31 23:59
raw_binary_file_extension=*.X*
raw_binary_min_filesize_bytes=900
site_to_process=CH-OE2
site_to_process=CH-FRU
server_raw_binary_source_dir=\\grasslandserver.ethz.ch\archive\FluxData\CH-INO_InnoFarm\20_sonic_ghg
AWS_raw_binary_source_dir=\\grasslandserver.ethz.ch\archive\FluxData\CH-AWS_CrapAlv\20_sonic_ghg
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment