To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 99dbbcd7 authored by holukas's avatar holukas
Browse files

Minor adjustments

parent 6700214e
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="" />
<list default="true" id="254ecb79-655b-4854-8af6-177bb7347e8a" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/bico.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/bico.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/ops/bin.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/ops/bin.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settings" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settings" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/Bico.settingsOld" afterDir="false" />
<change beforePath="$PROJECT_DIR$/bico/settings/_version.py" beforeDir="false" afterPath="$PROJECT_DIR$/bico/settings/_version.py" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
......@@ -349,9 +356,14 @@
<breakpoints>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/bico.py</url>
<line>425</line>
<line>419</line>
<option name="timeStamp" value="41" />
</line-breakpoint>
<line-breakpoint enabled="true" suspend="THREAD" type="python-line">
<url>file://$PROJECT_DIR$/bico/ops/bin.py</url>
<line>94</line>
<option name="timeStamp" value="183" />
</line-breakpoint>
</breakpoints>
<default-breakpoints>
<breakpoint type="python-exception">
......@@ -371,7 +383,7 @@
<SUITE FILE_PATH="coverage/BICO$main.coverage" NAME="bico Coverage Results" MODIFIED="1598258463691" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$gui.coverage" NAME="gui Coverage Results" MODIFIED="1606435918815" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$main.coverage" NAME="main Coverage Results" MODIFIED="1596498134562" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1610724233273" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$start_bico.coverage" NAME="start_bico Coverage Results" MODIFIED="1610742316824" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/bico" />
<SUITE FILE_PATH="coverage/BICO_Binary_Converter$example.coverage" NAME="example Coverage Results" MODIFIED="1606348759035" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$/example" />
</component>
</project>
\ No newline at end of file
......@@ -297,8 +297,6 @@ class Bico(qtw.QMainWindow, Ui_MainWindow):
copyfile(old_settings_file, run_settings_file_path)
pass
# return settings_dict
def select_dir(self, start_dir, dir_setting, update_label, dialog_txt):
""" Select directory, update dict and label"""
selected_dir = qtw.QFileDialog.getExistingDirectory(None, dialog_txt, str(start_dir)) # Open dialog
......@@ -326,10 +324,6 @@ class Bico(qtw.QMainWindow, Ui_MainWindow):
self.settings_dict['filename_datetime_parsing_string'])
ascii_filedate = bin_filedate.strftime('%Y%m%d%H%M') # w/o extension
ascii_filename = f"{self.settings_dict['site']}_{ascii_filedate}" # w/o extension
# ascii_filename = f"{self.settings_dict['site']}_{ascii_filedate}.csv"
# ascii_filepath = self.settings_dict['dir_out_run_raw_data_ascii'] / ascii_filename
# ascii_filename_gzip = f"{self.settings_dict['site']}_{ascii_filedate}.csv.gz"
# ascii_filepath_gzip = self.settings_dict['dir_out_run_raw_data_ascii'] / ascii_filename_gzip
counter_bin_files += 1
self.statusbar.showMessage(f"Working on file #{counter_bin_files}: {bin_file}")
......
......@@ -83,7 +83,6 @@ class ConvertData:
# File header
self.dblock_headers = self.make_file_header()
# self.write_multirow_header_to_ascii(asciiWriter=asciiWriter)
# Data records
while not end_of_data_reached:
......@@ -91,29 +90,17 @@ class ConvertData:
file_newrow_records = []
_end_of_data_reached = []
results = [self.read_instr_dblock(dblock=d) for d in self.dblocks]
for r in results:
file_newrow_records.extend(r[0])
_end_of_data_reached.append(r[1])
# end_of_data_reached = True if True in _end_of_data_reached else False
onerow_records = [self.read_instr_dblock(dblock=d) for d in self.dblocks]
for dblock_records in onerow_records:
file_newrow_records.extend(dblock_records[0])
_end_of_data_reached.append(dblock_records[1])
if True in _end_of_data_reached:
end_of_data_reached = True
file_newrow_records = False
# for instr in self.dblocks:
# incoming_dblock_data, end_of_data_reached = self.read_instr_dblock(dblock=instr)
# if not end_of_data_reached:
# file_newrow_records.extend(incoming_dblock_data)
# # file_newrow_records = file_newrow_records + incoming_dblock_data
# else:
# file_newrow_records = False
# break # Breaks FOR loop
if file_newrow_records:
self.file_counter_lines += 1
# asciiWriter.writerow(file_newrow_records)
self.file_data_rows.append(file_newrow_records)
# self.data_df.append(file_newrow_records)
# Limit = 0 means no limit
if self.limit_read_lines > 0:
......@@ -121,10 +108,8 @@ class ConvertData:
break
self.open_binary.close()
# open_ascii.close()
self.logger.info(f" Finished conversion to ASCII.")
# self.logger.info(f" ASCII data saved to file {self.ascii_filename}")
self.file_speedstats()
def read_instr_dblock(self, dblock):
......@@ -136,8 +121,6 @@ class ConvertData:
dblock_vars_read = 0
end_of_data_reached = False
# todo hier weiter var-by-var
for var, props in dblock.items():
if 'bit_pos_start' in props.keys(): # Skip variables from bit map, will be extracted later
......@@ -204,7 +187,7 @@ class ConvertData:
dblock_bytes_read=dblock_bytes_read)
break
# Convert to hex or octal if needed
# Convert if needed
var_val = self.convert_val(units=props['units'], var_val=var_val)
# Add value to data
......@@ -218,6 +201,7 @@ class ConvertData:
for bmv in bit_map_vals:
dblock_data.append(bmv)
# return dblock_data
return dblock_data, end_of_data_reached
......
run_id=BICO-20210115-162354
run_id=BICO-20210115-212517
# INSTRUMENTS
# ===========
......@@ -8,26 +8,26 @@ site=CH-DAV
# Data Blocks
header=WECOM3
instrument_1=HS100-A
instrument_2=IRGA72-A
instrument_3=QCL-C
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=QCL-A
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/09/20_HS_LI72
dir_source=Y:/CH-CHA_Chamau/20_sonic_ghg/2020/06
# Time Range
start_date=2020-09-23 06:00
end_date=2020-09-30 23:00
start_date=2020-06-01 07:00
end_date=2020-06-02 07:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
filename_datetime_format=yyyymmddHH.CMM
file_ext=*.C*
file_size_min=900
file_limit=1
row_limit=1000
file_limit=0
row_limit=0
# Special
select_random_files=0
......@@ -35,14 +35,14 @@ select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354
dir_out_run_log=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\plots\agg
dir_out_run_raw_data_ascii=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\raw_data_ascii
dir_out_run=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212517
dir_out_run_log=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212517\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212517\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212517\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212517\plots\agg
dir_out_run_raw_data_ascii=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212517\raw_data_ascii
output_folder_name_prefix=DAV
output_folder_name_prefix=CHA
file_compression=None
add_instr_to_varname=0
......
run_id=BICO-20210115-162354
run_id=BICO-20210115-212257
# INSTRUMENTS
# ===========
......@@ -8,26 +8,26 @@ site=CH-DAV
# Data Blocks
header=WECOM3
instrument_1=HS100-A
instrument_2=IRGA72-A
instrument_3=QCL-C
instrument_1=R350-A
instrument_2=IRGA75-A
instrument_3=QCL-A
# RAW DATA
# ========
# Source Folder
dir_source=Y:/CH-DAV_Davos/20_sonic_ghg/2020/09/20_HS_LI72
dir_source=Y:/CH-CHA_Chamau/20_sonic_ghg/2020/06
# Time Range
start_date=2020-09-23 06:00
end_date=2020-09-30 23:00
start_date=2020-06-01 13:00
end_date=2020-06-02 13:00
# File Settings
filename_datetime_format=yyyymmddHH.XMM
file_ext=*.X*
filename_datetime_format=yyyymmddHH.CMM
file_ext=*.C*
file_size_min=900
file_limit=1
row_limit=1000
file_limit=0
row_limit=0
# Special
select_random_files=0
......@@ -35,14 +35,14 @@ select_random_files=0
# OUTPUT
# ======
dir_out=A:/FLUXES/x-TEST-OUT
dir_out_run=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354
dir_out_run_log=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\plots\agg
dir_out_run_raw_data_ascii=A:\FLUXES\x-TEST-OUT\DAV_BICO-20210115-162354\raw_data_ascii
dir_out_run=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212257
dir_out_run_log=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212257\log
dir_out_run_plots=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212257\plots
dir_out_run_plots_hires=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212257\plots\hires
dir_out_run_plots_agg=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212257\plots\agg
dir_out_run_raw_data_ascii=A:\FLUXES\x-TEST-OUT\CHA_BICO-20210115-212257\raw_data_ascii
output_folder_name_prefix=DAV
output_folder_name_prefix=CHA
file_compression=None
add_instr_to_varname=0
......
__version__ = "0.3.0"
__date__ = "14 Jan 2021"
__date__ = "15 Jan 2021"
__link_source_code__ = "https://gitlab.ethz.ch/holukas/bico"
__link_releases__ = "https://gitlab.ethz.ch/holukas/bico/-/releases"
__link_wiki__ = "https://gitlab.ethz.ch/holukas/bico/-/wikis/home"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment