To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 1ad02a62 authored by holukas's avatar holukas
Browse files

Refined data blocks

parent fa3c297d
...@@ -2,26 +2,31 @@ ...@@ -2,26 +2,31 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment=""> <list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.dblock" afterDir="false" /> <change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.md" afterDir="false" /> <change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.dblock" afterDir="false" /> <change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.md" afterDir="false" /> <change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/file.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/file.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/setup.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/ops/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/vis.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/vis.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/ops/vis.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/vis.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/IRGA72-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-B.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7500/IRGA75-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.dblock" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LOSGATOS_LGR/LGR-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/AERODYNE_QCL/QCL-C.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7200/IRGA72-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-100/HS100-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/infrared_gas_analyzers/LICOR_LI-7500/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-A.dblock" beforeDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/AERODYNE_QCL/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/HS50-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-B.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/laser_gas_analyzers/LOSGATOS_LGR/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_R3-50/R350-A.dblock" afterDir="false" /> <change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-100/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_HS-50/HS50-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/sonic_anemometers/GILL_R3-50/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
...@@ -32,6 +37,7 @@ ...@@ -32,6 +37,7 @@
<option name="RECENT_TEMPLATES"> <option name="RECENT_TEMPLATES">
<list> <list>
<option value="Python Script" /> <option value="Python Script" />
<option value="Gherkin feature file" />
</list> </list>
</option> </option>
</component> </component>
...@@ -75,11 +81,11 @@ ...@@ -75,11 +81,11 @@
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\ops" /> <recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\ops" />
</key> </key>
<key name="MoveFile.RECENT_KEYS"> <key name="MoveFile.RECENT_KEYS">
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\header" /> <recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\gui" /> <recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\infrared_gas_analyzers\LICOR_LI-7500" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings" /> <recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\infrared_gas_analyzers\LICOR_LI-7200" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\ops" /> <recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\laser_gas_analyzers\QCL" />
<recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\data_blocks\sonic_anemometers" /> <recent name="L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings\data_blocks\laser_gas_analyzers\LGR" />
</key> </key>
</component> </component>
<component name="RunManager" selected="Python.start_bico"> <component name="RunManager" selected="Python.start_bico">
...@@ -299,7 +305,9 @@ ...@@ -299,7 +305,9 @@
<workItem from="1607594405028" duration="3277000" /> <workItem from="1607594405028" duration="3277000" />
<workItem from="1607607175897" duration="3170000" /> <workItem from="1607607175897" duration="3170000" />
<workItem from="1607615147949" duration="16371000" /> <workItem from="1607615147949" duration="16371000" />
<workItem from="1607676418125" duration="16535000" /> <workItem from="1607676418125" duration="18476000" />
<workItem from="1607812206779" duration="8920000" />
<workItem from="1607939225501" duration="15331000" />
</task> </task>
<servers /> <servers />
</component> </component>
...@@ -308,13 +316,6 @@ ...@@ -308,13 +316,6 @@
</component> </component>
<component name="XDebuggerManager"> <component name="XDebuggerManager">
<breakpoint-manager> <breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>320</line>
<option name="timeStamp" value="6" />
</line-breakpoint>
</breakpoints>
<default-breakpoints> <default-breakpoints>
<breakpoint type="python-exception"> <breakpoint type="python-exception">
<properties notifyOnTerminate="true" exception="BaseException"> <properties notifyOnTerminate="true" exception="BaseException">
...@@ -332,7 +333,7 @@ ...@@ -332,7 +333,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607694745154" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1607987599630" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" /> <SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component> </component>
</project> </project>
\ No newline at end of file
...@@ -169,7 +169,7 @@ class Ui_MainWindow(object): ...@@ -169,7 +169,7 @@ class Ui_MainWindow(object):
# Data Blocks # Data Blocks
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-'] sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'LOSGATOS_LGR-A', 'AERODYNE_QCL-C', '-None-'] gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'LGR-A', 'QCL-C', '-None-']
header_instr_data_blocks = qtw.QLabel('Data Blocks') header_instr_data_blocks = qtw.QLabel('Data Blocks')
header_instr_data_blocks.setProperty('labelClass', 'header_2') header_instr_data_blocks.setProperty('labelClass', 'header_2')
......
...@@ -143,6 +143,7 @@ def bit_map_extract_vals(bit_map_dict, var_binary_string): ...@@ -143,6 +143,7 @@ def bit_map_extract_vals(bit_map_dict, var_binary_string):
val = val * bit_map_props['apply_gain'] val = val * bit_map_props['apply_gain']
val = val + bit_map_props['add_offset'] val = val + bit_map_props['add_offset']
bit_map_vals.append(val) bit_map_vals.append(val)
# print(bit_map_vals)
return bit_map_vals return bit_map_vals
...@@ -230,6 +231,9 @@ def read_bin_instr(open_binary, dblock, total_bytes_read, logger): ...@@ -230,6 +231,9 @@ def read_bin_instr(open_binary, dblock, total_bytes_read, logger):
# Convert to octal if needed # Convert to octal if needed
if props['units'] == 'octal': if props['units'] == 'octal':
var_val = convert_val_to_octal(var_val=var_val) var_val = convert_val_to_octal(var_val=var_val)
# Convert to octal for LGR laser analyzer
if props['units'] == 'octal_lgr':
var_val = convert_val_to_octal_lgr(var_val=var_val)
# Check if size info available # Check if size info available
if 'DATA_SIZE' in var: if 'DATA_SIZE' in var:
...@@ -321,6 +325,7 @@ def convert_val_to_hex(var_val): ...@@ -321,6 +325,7 @@ def convert_val_to_hex(var_val):
# print("X") # print("X")
return hex_val_no_prefix return hex_val_no_prefix
def convert_val_to_octal(var_val): def convert_val_to_octal(var_val):
"""Convert value to octal """Convert value to octal
...@@ -329,9 +334,34 @@ def convert_val_to_octal(var_val): ...@@ -329,9 +334,34 @@ def convert_val_to_octal(var_val):
is converted to octal '0o0', is converted to octal without prefix '0' is converted to octal '0o0', is converted to octal without prefix '0'
Returns: int Returns: int
""" """
oct_val = oct(int(var_val)) # Note: has octal prefix '0o' at start, e.g. '0o0' # if var_val != 0:
# print("x")
var_val_int = int(var_val) # Convert to integer
# var_val_bin = bin(var_val_int)[2:] # Convert to binary string
# var_val_bin = var_val_bin.zfill(8) # F
# var_val_oct = int(var_val_bin, 8) # Elegant way to convert binary string to octal
# bin(int(var_val))[2:].zfill(16)
oct_val = oct(var_val_int) # Note: has octal prefix '0o' at start, e.g. '0o0'
oct_val_no_prefix = oct_val[2:] # Remove octal prefix from val oct_val_no_prefix = oct_val[2:] # Remove octal prefix from val
return int(oct_val_no_prefix) return oct_val_no_prefix
def convert_val_to_octal_lgr(var_val):
"""Convert value to octal for LGR laser analyzer
Format for the LGR status code that is recorded in the raw binary files.
Example:
- var_val = 113.0
- var_val_int = 113
- var_val_bin = '0001'
- var_val_oct = 1
Returns: int
"""
var_val_int = int(var_val) # Convert to integer
var_val_bin = bin(var_val_int)[-4:] # Convert to binary string, relevant info is in last 4 bits
var_val_oct = int(var_val_bin, 8) # Elegant way to convert binary string to octal
return var_val_oct
def generate_missing_values(dict): def generate_missing_values(dict):
......
...@@ -87,6 +87,7 @@ class SearchAll(): ...@@ -87,6 +87,7 @@ class SearchAll():
@staticmethod @staticmethod
def search_all(dir, file_id, logger): def search_all(dir, file_id, logger):
"""Search all files in dir that match file id""" """Search all files in dir that match file id"""
logger.info("Searching for files ...")
valid_files_dict = {} valid_files_dict = {}
for root, dirs, found_files in os.walk(dir): for root, dirs, found_files in os.walk(dir):
for idx, file in enumerate(found_files): for idx, file in enumerate(found_files):
...@@ -145,7 +146,7 @@ class SearchAll(): ...@@ -145,7 +146,7 @@ class SearchAll():
self.logger.info(f"{suffix} +++ {len(valid_files_dict)} files were larger than the selected minimum" self.logger.info(f"{suffix} +++ {len(valid_files_dict)} files were larger than the selected minimum"
f" filesize of {min_filesize_lim}, keeping files: {list(valid_files_dict.keys())}") f" filesize of {min_filesize_lim}, keeping files: {list(valid_files_dict.keys())}")
self.logger.info(f"{suffix} --- {len(_invalid_files_dict)} files were smaller than the selected minimum" self.logger.info(f"{suffix} --- {len(_invalid_files_dict)} files were smaller than the selected minimum"
f" filesize of {min_filesize_lim} and will not be used: {list(_invalid_files_dict.keys())}") f" filesize of {min_filesize_lim} and will not be used: {list(_invalid_files_dict.keys())}")
self.logger.info(f"{suffix} ============================") self.logger.info(f"{suffix} ============================")
return valid_files_dict return valid_files_dict
...@@ -187,7 +188,6 @@ def export_raw_data_csv(df, outdir, logger, outfile='temp', compression='gzip'): ...@@ -187,7 +188,6 @@ def export_raw_data_csv(df, outdir, logger, outfile='temp', compression='gzip'):
def export_stats_collection_csv(df, outdir, run_id, logger): def export_stats_collection_csv(df, outdir, run_id, logger):
# outpath = outdir / f"stats_" outpath = outdir / f"stats_agg_{run_id}"
outpath = outdir / f"stats_{run_id}" # TODO act run_id
logger.info(f"Saving stats collection to {outpath}") logger.info(f"Saving stats collection to {outpath}")
df.to_csv(f"{outpath}.csv", index=True) df.to_csv(f"{outpath}.csv", index=True)
...@@ -28,8 +28,8 @@ def make_run_outdirs(settings_dict): ...@@ -28,8 +28,8 @@ def make_run_outdirs(settings_dict):
"""Set output paths and create output folders""" """Set output paths and create output folders"""
# Run output folder # Run output folder
settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / "TEST_OUT" # settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / "TEST_OUT"
# settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / settings_dict['run_id'] # todo act settings_dict['dir_out_run'] = Path(settings_dict['dir_out']) / settings_dict['run_id'] # todo act
if not Path.is_dir(settings_dict['dir_out_run']): if not Path.is_dir(settings_dict['dir_out_run']):
print(f"Creating folder {settings_dict['dir_out_run']} ...") print(f"Creating folder {settings_dict['dir_out_run']} ...")
os.makedirs(settings_dict['dir_out_run']) os.makedirs(settings_dict['dir_out_run'])
......
...@@ -133,7 +133,7 @@ def aggs_ts(df, outdir, logger): ...@@ -133,7 +133,7 @@ def aggs_ts(df, outdir, logger):
font = {'family': 'sans-serif', 'size': 10} font = {'family': 'sans-serif', 'size': 10}
ax1.legend(frameon=True, loc='upper right', prop=font).set_zorder(100) ax1.legend(frameon=True, loc='upper right', prop=font).set_zorder(100)
outfile = outdir / f"__STATS__{var[0]}_{var[1]}_{var[2]}" outfile = outdir / f"stats_agg_{var[0]}_{var[1]}_{var[2]}"
fig.savefig(f"{outfile}.png", format='png', bbox_inches='tight', facecolor='w', fig.savefig(f"{outfile}.png", format='png', bbox_inches='tight', facecolor='w',
transparent=True, dpi=150) transparent=True, dpi=150)
......
run_id=BICO-20201211-145226 run_id=BICO-20201215-001320
# INSTRUMENTS # INSTRUMENTS
# =========== # ===========
# Site # Site
site=CH-DAV site=CH-AES
# Data Blocks # Data Blocks
header=WECOM3 header=WECOM3
instrument_1=HS50-A instrument_1=R350-A
instrument_2=IRGA72-A instrument_2=IRGA75-A
instrument_3=QCL-C instrument_3=LGR-A
# RAW DATA # RAW DATA
# ======== # ========
# Source Folder # Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/06/20_HS_LI72 dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/08
# Time Range # Time Range
start_date=2020-06-01 13:00 start_date=2020-06-01 13:00
...@@ -26,28 +26,28 @@ end_date=2020-12-31 00:00 ...@@ -26,28 +26,28 @@ end_date=2020-12-31 00:00
filename_datetime_format=yyyymmddHH.XMM filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X* file_ext=*.X*
file_size_min=900 file_size_min=900
file_limit=0 file_limit=1
row_limit=100 row_limit=10000
# Special # Special
select_random_files=10 select_random_files=0
# OUTPUT # OUTPUT
# ====== # ======
dir_out=A:/FLUXES/x-TEST-OUT dir_out=P:/Flux/RDS_calculations/INO/AES/BICO
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT dir_out_run=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log dir_out_run_log=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots dir_out_run_plots=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires dir_out_run_plots_hires=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg dir_out_run_plots_agg=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv dir_out_run_raw_data_csv=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-001320\raw_data_csv
file_compression=gzip file_compression=None
plot_file_availability=1 plot_file_availability=1
plot_ts_hires=0 plot_ts_hires=1
plot_histogram_hires=0 plot_histogram_hires=1
plot_ts_agg=0 plot_ts_agg=1
# DIRECTORIES # DIRECTORIES
# =================== # ===================
......
run_id=BICO-20201211-144647 run_id=BICO-20201215-000308
# INSTRUMENTS # INSTRUMENTS
# =========== # ===========
# Site # Site
site=CH-DAV site=CH-AES
# Data Blocks # Data Blocks
header=WECOM3 header=WECOM3
instrument_1=HS50-A instrument_1=R350-A
instrument_2=IRGA72-A instrument_2=IRGA75-A
instrument_3=QCL-C instrument_3=LGR-A
# RAW DATA # RAW DATA
# ======== # ========
# Source Folder # Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/06/20_HS_LI72 dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/08
# Time Range # Time Range
start_date=2020-06-01 13:00 start_date=2020-06-01 13:00
...@@ -26,28 +26,28 @@ end_date=2020-12-31 00:00 ...@@ -26,28 +26,28 @@ end_date=2020-12-31 00:00
filename_datetime_format=yyyymmddHH.XMM filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X* file_ext=*.X*
file_size_min=900 file_size_min=900
file_limit=0 file_limit=1
row_limit=100 row_limit=10000
# Special # Special
select_random_files=10 select_random_files=0
# OUTPUT # OUTPUT
# ====== # ======
dir_out=A:/FLUXES/x-TEST-OUT dir_out=P:/Flux/RDS_calculations/INO/AES/BICO
dir_out_run=A:\FLUXES\x-TEST-OUT\TEST_OUT dir_out_run=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308
dir_out_run_log=A:\FLUXES\x-TEST-OUT\TEST_OUT\log dir_out_run_log=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots dir_out_run_plots=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\hires dir_out_run_plots_hires=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\TEST_OUT\plots\agg dir_out_run_plots_agg=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\TEST_OUT\raw_data_csv dir_out_run_raw_data_csv=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-000308\raw_data_csv
file_compression=gzip file_compression=None
plot_file_availability=1 plot_file_availability=1
plot_ts_hires=0 plot_ts_hires=1
plot_histogram_hires=0 plot_histogram_hires=1
plot_ts_agg=0 plot_ts_agg=1
# DIRECTORIES # DIRECTORIES
# =================== # ===================
......
__version__ = "0.0.4" __version__ = "0.0.2"
__date__ = "10 Dec 2020" __date__ = "12 Dec 2020"
__link_source_code__ = "https://gitlab.ethz.ch/holukas/bico" __link_source_code__ = "https://gitlab.ethz.ch/holukas/bico"
__link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases" __link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases"
__link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home" __link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home"
......
...@@ -5,8 +5,23 @@ ...@@ -5,8 +5,23 @@
- V ... Second horizontal wind component (y) - V ... Second horizontal wind component (y)
- W ... Vertical wind component (z) - W ... Vertical wind component (z)
- T_SONIC ... Sonic temperature - T_SONIC ... Sonic temperature
- SA_DIAG_TYPE ... Status type indicator (error, inclinometer, configuration) (StaA in manual) - SA_DIAG_TYPE ... Status type indicator (error, inclinometer, configuration)
- SA_DIAG_VAL ... Status value and information (StaD in manual) - Called ```StaA``` (Status Address) in manual
- defines which particular aspect of the anemometer the following data field (SA_DIAG_VAL) refers to
- 00 Error codes
- 01 Anemometer configuration
- 02 Data output configuration 1
- 03 Data output configuration 2
- 04 Error code history
- 05 Transducer gain levels
- 06 Anemometer Type
- 07 Inclinometer X axis MSB
- 08 Inclinometer X axis LSB
- 09 Inclinometer Y axis MSB
- 10 Inclinometer Y axis LSB
- SA_DIAG_VAL ... Status value and information
- Called ```StaD``` (Status Data) in manual
- Please refer to the manual for detailed information about meanings
- INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers) - INC_XY ... Inclinometer, alternatively x (odd record numbers) and y (even record numbers)
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements* *2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
......
# ============================================= # =============================================
# IRGA72-A - INFRARED GAS ANALYZER (LICOR 7200) # IRGA72-A - INFRARED GAS ANALYZER (LICOR 7200)
# ============================================= # =============================================
# Details: see IRGA72-A.md
DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-A'} DATA_SIZE == {'order': 1, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'Bytes', 'datablock': 'IRGA72-A'}
STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'octal', 'datablock': 'IRGA72-A'} STATUS_CODE == {'order': 2, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'octal', 'datablock': 'IRGA72-A'}
...@@ -17,95 +19,16 @@ FLOW_VOLRATE == {'order': 12, 'bytes': 2, 'format': 'B B', 'gain_on_signal ...@@ -17,95 +19,16 @@ FLOW_VOLRATE == {'order': 12, 'bytes': 2, 'format': 'B B', 'gain_on_signal
# Values extracted from GA_DIAG_CODE: # Values extracted from GA_DIAG_CODE:
# ------------------------------- # -------------------------------
UNUSED == {'order': 3.01, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 1, 'add_offset': 0, 'units': 'unused', 'output': 0, 'datablock': 'IRGA72-A'} UNUSED == {'order': 3.01, 'bit_pos_start': 0, 'bit_pos_end': 3, 'apply_gain': 1, 'add_offset': 0, 'units': 'unused', 'output': 1, 'datablock': 'IRGA72-A'}
HEAD_DETECT == {'order': 3.02, 'bit_pos_start': 3, 'bit_pos_end': 4, 'apply_gain': 1, 'add_offset': 0, 'units': '1=LI-7200', 'output': 0, 'datablock': 'IRGA72-A'} HEAD_DETECT == {'order': 3.02, 'bit_pos_start': 3, 'bit_pos_end': 4, 'apply_gain': 1, 'add_offset': 0, 'units': '1=LI-7200', 'output': 1, 'datablock': 'IRGA72-A'}
T_OUTLET == {'order': 3.03, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} T_OUTLET == {'order': 3.03, 'bit_pos_start': 4, 'bit_pos_end': 5, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
T_INLET == {'order': 3.04, 'bit_pos_start': 5, 'bit_pos_end': 6, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} T_INLET == {'order': 3.04, 'bit_pos_start': 5, 'bit_pos_end': 6, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
AUX_INPUT == {'order': 3.05, 'bit_pos_start': 6, 'bit_pos_end': 7, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} AUX_INPUT == {'order': 3.05, 'bit_pos_start': 6, 'bit_pos_end': 7, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
DIFF_PRESS == {'order': 3.06, 'bit_pos_start': 7, 'bit_pos_end': 8, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} DIFF_PRESS == {'order': 3.06, 'bit_pos_start': 7, 'bit_pos_end': 8, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
CHOPPER == {'order': 3.07, 'bit_pos_start': 8, 'bit_pos_end': 9, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} CHOPPER == {'order': 3.07, 'bit_pos_start': 8, 'bit_pos_end': 9, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
DETECTOR == {'order': 3.08, 'bit_pos_start': 9, 'bit_pos_end': 10, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} DETECTOR == {'order': 3.08, 'bit_pos_start': 9, 'bit_pos_end': 10, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
PLL == {'order': 3.09, 'bit_pos_start': 10, 'bit_pos_end': 11, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} PLL == {'order': 3.09, 'bit_pos_start': 10, 'bit_pos_end': 11, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
SYNC == {'order': 3.10, 'bit_pos_start': 11, 'bit_pos_end': 12, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 0, 'datablock': 'IRGA72-A'} SYNC == {'order': 3.10, 'bit_pos_start': 11, 'bit_pos_end': 12, 'apply_gain': 1, 'add_offset': 0, 'units': '1=OK', 'output': 1, 'datablock': 'IRGA72-A'}
SIGNAL_STRENGTH == {'order': 3.11, 'bit_pos_start': 12, 'bit_pos_end': 16, 'apply_gain': 6.6666666666666666, 'add_offset': 0, 'units': '%', 'output': 1, 'datablock': 'IRGA72-A'} SIGNAL_STRENGTH == {'order': 3.11, 'bit_pos_start': 12, 'bit_pos_end': 16, 'apply_gain': 6.6666666666666666, 'add_offset': 0, 'units': '%', 'output': 1, 'datablock': 'IRGA72-A'}
# NOTES
# =====
#
# BICO ID: IRGA72-A
#
# Variables:
# DATA_SIZE ... Data size of current data block, number of bytes in Licor 7200 record (2 = missing, 26 = available)
# STATUS_CODE ... Status of IRGA data aquisition, see Table 6 in WE's sonicread.pdf
# octal value converted to integer yields:
# 0 .. Status OK, no problems (octal 0000)
# 20 .. IRGA did not respond (0020)
# 40 .. Status OK, old data used (0040)
# 200 .. not OK, IRGA data are missing (0200)
# GA_DIAG_CODE ... IRGA diagnostic value
# MSB, most significant bit; high-order bit
# The cell diagnostic value is a 2 byte unsigned integer (value between 0 and 8191)
# with the following bit map (in order of how the code reads it, orig bit position in brackets):
# - (15,14,13) UNUSED
# - (12) HEAD_DETECT: sensor head attached to LI-7550; 1 = LI-7200
# - (11) T_OUTLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (10) T_INLET: 1 = thermocouple OK; 0 = thermocouple open circuit
# - (9) AUX_INPUT: 1 = internal reference voltages OK;
# 0 = internal reference voltages not OK, analyzer interface unit needs service
# - (8) DIFF_PRESS: 1 = good, 0.1 to 4.9V; 0 = out of range; d=delta
# - (7) CHOPPER: 1 = chopper wheel temp is near setpoint; 0 = not near setpoint
# - (6) DETECTOR: 1 = detector temp is near setpoint; 0 = not near setpoint
# - (5) PLL: 1 = OK; lock bit, indicates that optical wheel is rotating at the correct rate
# - (4) SYNC: always set to 1 (OK)
# - (3,2,1,0) SIGNAL_STRENGTH
# * The gain 6.6666666666666666 has that many digits after the comma so that the calculated
# max signal strength is 100%. For example, if there is one digit after the comma less,
# then the max signal strength yields 99.99999999999999% instead of 100%.
# 'output': 1 means that the var is written to the output stream of this data block, i.e. included in the