To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 566dc2cd authored by holukas's avatar holukas
Browse files

Better binary conversion, new dblocks

parent 43cb832c
......@@ -2,13 +2,48 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-B.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/ops/bin_conversion_exceptions.py" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A4.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A4.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R2-A.dblock" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R2-A.md" afterDir="false" />
<change afterPath="$PROJECT_DIR$/bico/settings/data_blocks/_help_bico_settings.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/gui/gui.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/gui/gui.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/file.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/file.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico - Kopie (2).settings" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico - Kopie.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico - Kopie.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico - Kopie.settingsOld" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS100-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-B.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/HS50-B.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA72-B.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A-GN1.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/IRGA75-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/LGR-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A2.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A3.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-A3.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/QCL-C.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.dblock" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.md" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/data_blocks/R350-A.md" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -18,8 +53,8 @@
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Python Script" />
<option value="Gherkin feature file" />
<option value="Python Script" />
</list>
</option>
</component>
......@@ -322,7 +357,10 @@
<workItem from="1610135291866" duration="1775000" />
<workItem from="1610145522443" duration="1445000" />
<workItem from="1610235475279" duration="791000" />
<workItem from="1610289783723" duration="6634000" />
<workItem from="1610289783723" duration="12792000" />
<workItem from="1610363681251" duration="869000" />
<workItem from="1610377337549" duration="10396000" />
<workItem from="1610442714131" duration="4439000" />
</task>
<servers />
</component>
......@@ -334,13 +372,8 @@
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>241</line>
<option name="timeStamp" value="27" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>142</line>
<option name="timeStamp" value="28" />
<line>334</line>
<option name="timeStamp" value="3" />
</line-breakpoint>
</breakpoints>
<default-breakpoints>
......@@ -360,7 +393,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1610322270675" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1610448630228" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component>
</project>
\ No newline at end of file
......@@ -15,6 +15,7 @@ from gui.gui import Ui_MainWindow
from ops import bin, format_data, vis, file, stats
from settings import _version
# TODO LATER parallelize, multiprocessing?
# import multiprocessing as mp
# print("Number of processors: ", mp.cpu_count())
......@@ -334,16 +335,14 @@ class Bico(qtw.QMainWindow, Ui_MainWindow):
logger.info(f" Reading binary file #{counter_bin_files} of {num_bin_files}: {bin_file}...")
logger.info(f" Data block sequence: {self.dblocks_seq}")
# Read binary data
data_lines, header, tic, counter_lines = \
bin.read_file(binary_filename=bin_filepath,
size_header=self.bin_size_header,
dblocks=dblocks_props,
limit_read_lines=int(self.settings_dict['row_limit']),
logger=self.logger,
statusbar=self.statusbar)
# if not data_lines:
# continue
# Read binary data file
obj = bin.ReadFile(binary_filename=bin_filepath,
size_header=self.bin_size_header,
dblocks=dblocks_props,
limit_read_lines=int(self.settings_dict['row_limit']),
logger=self.logger)
obj.run()
data_lines, header, tic, counter_lines = obj.get_data()
bin.speedstats(tic=tic, counter_lines=counter_lines, logger=logger)
......@@ -363,9 +362,9 @@ class Bico(qtw.QMainWindow, Ui_MainWindow):
csv_filename = f"{self.settings_dict['site']}_{csv_filedate}"
# Export file CSV
file.export_raw_data_csv(df=df, outfile=csv_filename, logger=logger,
outdir=self.settings_dict['dir_out_run_raw_data_csv'],
compression=self.settings_dict['file_compression'])
file.export_raw_data_ascii(df=df, outfile=csv_filename, logger=logger,
outdir=self.settings_dict['dir_out_run_raw_data_ascii'],
compression=self.settings_dict['file_compression'])
# Plot high-resolution data
if self.settings_dict['plot_ts_hires'] == '1':
......
......@@ -174,9 +174,9 @@ class Ui_MainWindow(object):
'CH-INO', 'CH-LAE', 'CH-LAS', 'CH-OE2'])
# Data Blocks
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A','R350-A', '-None-']
sonic_anemometers = ['HS50-A', 'HS50-B', 'HS100-A', 'R2-A','R350-A', '-None-']
gas_analyzers = ['IRGA72-A', 'IRGA72-B', 'IRGA75-A', 'IRGA75-A-GN1', 'LGR-A',
'QCL-A', 'QCL-A2', 'QCL-A3', 'QCL-C', '-None-']
'QCL-A', 'QCL-A2', 'QCL-A3', 'QCL-A4', 'QCL-C', '-None-']
header_instr_data_blocks = qtw.QLabel('Data Blocks')
header_instr_data_blocks.setProperty('labelClass', 'header_2')
......
This diff is collapsed.
def dblock_r2a_t_sonic(var_val):
"""Special conversion for T_SONIC in R2-A"""
var_val *= 0.02
var_val *= var_val
var_val /= 403
var_val -= 273.15 # To °C
return var_val
......@@ -178,7 +178,7 @@ class SearchAll():
return valid_files_dict
def export_raw_data_csv(df, outdir, logger, outfile='temp', compression='gzip'):
def export_raw_data_ascii(df, outdir, logger, outfile='temp', compression='gzip'):
logger.info(" Saving raw data csv ...")
outpath = outdir / outfile
if compression == 'gzip':
......
......@@ -41,12 +41,12 @@ def make_run_outdirs(settings_dict):
settings_dict['dir_out_run_plots_agg'] = settings_dict['dir_out_run_plots'] / 'agg'
# Files
settings_dict['dir_out_run_raw_data_csv'] = settings_dict['dir_out_run'] / 'raw_data_csv'
settings_dict['dir_out_run_raw_data_ascii'] = settings_dict['dir_out_run'] / 'raw_data_ascii'
settings_dict['dir_out_run_log'] = settings_dict['dir_out_run'] / 'log'
# Make dirs
create_dirs = ['dir_out_run_plots', 'dir_out_run_plots_hires', 'dir_out_run_plots_agg',
'dir_out_run_raw_data_csv', 'dir_out_run_log']
'dir_out_run_raw_data_ascii', 'dir_out_run_log']
for d in create_dirs:
if not Path.is_dir(settings_dict[d]):
print(f"Creating folder {settings_dict[d]} ...")
......
run_id=FR-20181213-005305
now_time=2018-12-13 00:53:05
# INSTRUMENTS
# ===========
# Site
site=CH-DAV
# Data Blocks
instrument_1=HS50-A
instrument_2=IRGA72-A
instrument_3=QCL-C
# RAW DATA
# ========
# Time Range
start_date=2001-06-30 22:00
end_date=2018-12-31 00:00
# File Settings
file_ext=*.X*
file_size_min=900
file_limit=99999
# OUTPUT
# ======
plot_file_availability=1
plot_ts_hires=1
plot_ts_agg=1
# DIRECTORIES
# ===================
dir_out=
dir_bico=
dir_script=
dir_settings=
dir_root=
dir_server_CH-AWS=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-AWS_CrapAlv\20_sonic_ghg
dir_server_CH-CHA=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-CHA_Chamau\20_sonic_ghg
dir_server_CH-DAV=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-DAV_Davos\20_sonic_ghg
dir_server_CH-FRU=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-FRU_Fruebuel\20_sonic_ghg
dir_server_CH-INO=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-INO_InnoFarm\20_sonic_ghg
dir_server_CH-LAE=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-LAE_Laegeren\20_sonic_ghg
dir_server_CH-LAS=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-LAS_Lae-Subcanopy\20_sonic_ghg
dir_server_CH-OE2=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-OE2_Oensingen\20_sonic_ghg
dir_server_CH-TWE=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\US-TWE_ToolikWetland\20_sonic_ghg
run_id=FR-20181213-005305
now_time=2018-12-13 00:53:05
run_id=BICO-20210111-023112
# INSTRUMENTS
# ===========
......@@ -8,37 +7,59 @@ now_time=2018-12-13 00:53:05
site=CH-DAV
# Data Blocks
instrument_1=HS50-A
instrument_2=IRGA72-A
header=WECOM3
instrument_1=HS50-B
instrument_2=IRGA72-B
instrument_3=QCL-C
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2015/11
# Time Range
start_date=2001-06-30 22:00
end_date=2018-12-31 00:00
start_date=2015-11-15 08:00
end_date=2015-11-15 23:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=99999
file_limit=0
row_limit=0
# Special
select_random_files=0
# OUTPUT
# ======
plot_file_availability=1
plot_ts_hires=1
plot_ts_agg=1
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-023112
dir_out_run_log=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-023112\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-023112\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-023112\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-023112\plots\agg
dir_out_run_raw_data_ascii=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-023112\raw_data_ascii
output_folder_name_prefix=DAV_2015
file_compression=None
plot_file_availability=0
plot_ts_hires=0
plot_histogram_hires=0
plot_ts_agg=0
# DIRECTORIES
# ===================
dir_out=
dir_bico=
dir_script=
dir_settings=
dir_root=
dir_bico=L:\Dropbox\luhk_work\programming\BICO_Binary_Converter
dir_script=L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico
dir_settings=L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings
dir_root=L:\Dropbox\luhk_work\programming
dir_server_CH-AES=
dir_server_CH-AWS=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-AWS_CrapAlv\20_sonic_ghg
dir_server_CH-CHA=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-CHA_Chamau\20_sonic_ghg
dir_server_CH-DAE=
dir_server_CH-DAV=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-DAV_Davos\20_sonic_ghg
dir_server_CH-FRU=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-FRU_Fruebuel\20_sonic_ghg
dir_server_CH-INO=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-INO_InnoFarm\20_sonic_ghg
......
run_id=BICO-20201228-001616
# INSTRUMENTS
# ===========
# Site
site=CH-AES
# Data Blocks
header=WECOM3
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=LGR-A
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-INO_InnoFarm/20_sonic_ghg/2020/06
# Time Range
start_date=2020-06-01 00:00
end_date=2020-06-02 00:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
file_size_min=900
file_limit=0
row_limit=0
# Special
select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-004608
dir_out_run_log=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-004608\log
dir_out_run_plots=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-004608\plots
dir_out_run_plots_hires=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-004608\plots\hires
dir_out_run_plots_agg=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-004608\plots\agg
dir_out_run_raw_data_csv=P:\Flux\RDS_calculations\INO\AES\BICO\BICO-20201215-004608\raw_data_csv
file_compression=gzip
plot_file_availability=1
plot_ts_hires=1
plot_histogram_hires=1
plot_ts_agg=1
# DIRECTORIES
# ===================
dir_bico=L:\Dropbox\luhk_work\programming\BICO_Binary_Converter
dir_script=L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico
dir_settings=L:\Dropbox\luhk_work\programming\BICO_Binary_Converter\bico\settings
dir_root=L:\Dropbox\luhk_work\programming
dir_server_CH-AES=
dir_server_CH-AWS=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-AWS_CrapAlv\20_sonic_ghg
dir_server_CH-CHA=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-CHA_Chamau\20_sonic_ghg
dir_server_CH-DAE=
dir_server_CH-DAV=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-DAV_Davos\20_sonic_ghg
dir_server_CH-FRU=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-FRU_Fruebuel\20_sonic_ghg
dir_server_CH-INO=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-INO_InnoFarm\20_sonic_ghg
dir_server_CH-LAE=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-LAE_Laegeren\20_sonic_ghg
dir_server_CH-LAS=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-LAS_Lae-Subcanopy\20_sonic_ghg
dir_server_CH-OE2=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\CH-OE2_Oensingen\20_sonic_ghg
dir_server_CH-TWE=\\green.ethz.ch\share-green-gl-archive-1-$\FluxData\US-TWE_ToolikWetland\20_sonic_ghg
run_id=BICO-20210111-004431
run_id=BICO-20210112-115031
# INSTRUMENTS
# ===========
# Site
site=CH-DAV
site=CH-OE2
# Data Blocks
header=WECOM3
instrument_1=HS50-B
instrument_2=IRGA72-B
instrument_3=QCL-C
instrument_1=R2-A
instrument_2=IRGA72-A
instrument_3=-None-
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2015/11
dir_source=Y:/CH-OE2_Oensingen/20_sonic_ghg/2020/10
# Time Range
start_date=2015-11-15 08:00
end_date=2015-11-15 23:00
start_date=2020-10-16 07:00
end_date=2020-10-16 23:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
filename_datetime_format=yyyymmddHH.oMM
file_ext=*.o*
file_size_min=900
file_limit=0
row_limit=0
file_limit=1
row_limit=100
# Special
select_random_files=0
......@@ -35,14 +35,14 @@ select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-004431
dir_out_run_log=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-004431\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-004431\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-004431\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-004431\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-004431\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-115031
dir_out_run_log=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-115031\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-115031\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-115031\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-115031\plots\agg
dir_out_run_raw_data_ascii=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-115031\raw_data_ascii
output_folder_name_prefix=DAV_2015
output_folder_name_prefix=OE2
file_compression=None
plot_file_availability=0
......
run_id=BICO-20210111-003720
run_id=BICO-20210112-114802
# INSTRUMENTS
# ===========
# Site
site=CH-DAV
site=CH-OE2
# Data Blocks
header=WECOM3
instrument_1=HS50-B
instrument_2=IRGA72-B
instrument_3=QCL-C
instrument_1=R2-A
instrument_2=IRGA72-A
instrument_3=-None-
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2015/11
dir_source=Y:/CH-OE2_Oensingen/20_sonic_ghg/2020/10
# Time Range
start_date=2015-11-15 08:00
end_date=2015-11-15 23:00
start_date=2020-10-16 07:00
end_date=2020-10-16 23:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
filename_datetime_format=yyyymmddHH.oMM
file_ext=*.o*
file_size_min=900
file_limit=0
row_limit=10
file_limit=1
row_limit=100
# Special
select_random_files=10
select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-003720
dir_out_run_log=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-003720\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-003720\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-003720\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-003720\plots\agg
dir_out_run_raw_data_csv=A:\FLUXES\x-TEST-OUT\DAV_2015_BICO-20210111-003720\raw_data_csv
dir_out_run=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-114802
dir_out_run_log=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-114802\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-114802\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-114802\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-114802\plots\agg
dir_out_run_raw_data_ascii=A:\FLUXES\x-TEST-OUT\OE2_BICO-20210112-114802\raw_data_ascii
output_folder_name_prefix=DAV_2015
output_folder_name_prefix=OE2
file_compression=None
plot_file_availability=0
......
__version__ = "0.0.8"
__date__ = "10 Jan 2021"
__version__ = "0.1.0"
__date__ = "12 Jan 2021"
__link_source_code__ = "https://gitlab.ethz.ch/holukas/bico"
__link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases"
__link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home"
......
......@@ -4,10 +4,10 @@
# Details: see HS100-A.md
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS100-A'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS100-A'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS100-A'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'datablock': 'HS100-A'}
STAA == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'}
STAD == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'datablock': 'HS100-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS100-A'}
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'conversion_type': 'regular', 'datablock': 'HS100-A'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'conversion_type': 'regular', 'datablock': 'HS100-A'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'conversion_type': 'regular', 'datablock': 'HS100-A'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'conversion_type': 'regular', 'datablock': 'HS100-A'}
STAA == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'conversion_type': 'regular', 'datablock': 'HS100-A'}
STAD == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'dimensionless', 'conversion_type': 'regular', 'datablock': 'HS100-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'conversion_type': 'regular', 'datablock': 'HS100-A'}
......@@ -38,15 +38,7 @@
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
## BICO Settings
- 'order' ... Order in the data block variable sequence
- 'bytes' ... Number of bytes in the binary data
- 'format' ... Format to convert from binary to integers, floats, etc.
- 'gain_on_signal' ... Gain that was applied to the raw data signal, to get to units *divide* by this gain
- 'offset_on_signal' ... Offset that was added to the raw data signal, to get to units *subtract* this offset
- 'apply_gain' ... Gain that is applied during conversion, e.g. to convert to different units if needed
- 'add_offset' ... Offset that is added during conversion, e.g. to convert to different units if needed
- 'units' ... Units
- 'datablock' ... Data block ID
- For an explanation of the different variable property settings, please see ```_help_bico_settings.md```.
*Before BICO, the binary conversion was done in FCT FluxCalcTool:*
- Old ID in FCT: hs_100
......
......@@ -4,10 +4,10 @@
# Details: see HS50-A.md
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'datablock': 'HS50-A'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'datablock': 'HS50-A'}
SA_DIAG_TYPE == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs_type', 'datablock': 'HS50-A'}
SA_DIAG_VAL == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs', 'datablock': 'HS50-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'datablock': 'HS50-A'}
U == {'order': 1, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'conversion_type': 'regular', 'datablock': 'HS50-A'}
V == {'order': 2, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'conversion_type': 'regular', 'datablock': 'HS50-A'}
W == {'order': 3, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'm+1_s-1', 'conversion_type': 'regular', 'datablock': 'HS50-A'}
T_SONIC == {'order': 4, 'bytes': 2, 'format': '>h', 'gain_on_signal': 100, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'K', 'conversion_type': 'regular', 'datablock': 'HS50-A'}
SA_DIAG_TYPE == {'order': 5, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs_type', 'conversion_type': 'regular', 'datablock': 'HS50-A'}
SA_DIAG_VAL == {'order': 6, 'bytes': 1, 'format': 'B', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'diag_val_hs', 'conversion_type': 'regular', 'datablock': 'HS50-A'}
INC_XY == {'order': 7, 'bytes': 2, 'format': '>h', 'gain_on_signal': 1, 'offset_on_signal': 0, 'apply_gain': 1, 'add_offset': 0, 'units': 'deg', 'conversion_type': 'regular', 'datablock': 'HS50-A'}
......@@ -36,15 +36,7 @@
*2017-07-25: EddyPro can currently not handle StaA and StaD, but it will be implemented soon for ICOS requirements*
## BICO Settings
- 'order' ... Order in the data block variable sequence
- 'bytes' ... Number of bytes in the binary data
- 'format' ... Format to convert from binary to integers, floats, etc.
- 'gain_on_signal' ... Gain that was applied to the raw data signal, to get to units *divide* by this gain
- 'offset_on_signal' ... Offset that was added to the raw data signal, to get to units *subtract* this offset
- 'apply_gain' ... Gain that is applied during conversion, e.g. to convert to different units if needed
- 'add_offset' ... Offset that is added during conversion, e.g. to convert to different units if needed
- 'units' ... Units
- 'datablock' ... Data block ID
- For an explanation of the different variable property settings, please see ```_help_bico_settings.md```.
*Before BICO, the binary conversion was done in FCT FluxCalcTool:*
- Old ID in FCT: hs-50_extended
......
......@@ -4,12 +4,12 @@
# Details: see HS50-B.md