Commit f87f3386 authored by Roman Trüb's avatar Roman Trüb
Browse files

removed datatrace tmp files

parent b8fdf2f5
......@@ -435,27 +435,24 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue
varnames = ""
with open(input_filename, "r") as f:
varnames = f.readline().strip().split()
(fd1, tmpfile1) = tempfile.mkstemp()
(fd2, tmpfile2) = tempfile.mkstemp()
dwt.parse_dwt_output(input_filename, tmpfile1)
# parse raw datatrace log
df_parsed = dwt.parse_dwt_output(input_filename)
# apply linear regression to correct the timestamps
dwt.correct_ts_with_regression(tmpfile1, tmpfile2)
df_corrected = dwt.correct_ts_with_regression(df_parsed)
except ValueError:
logqueue.put_nowait((loggername, logging.WARNING, "Empty data trace results file."))
with open(resultfile_path, "a") as outfile:
df = pd.read_csv(tmpfile2)
# debug
# nan values cannot be converted to int -> drop corresponding lines
df = df_corrected
# remove lines with timestamp values
# since there were nan values, comparator column was stored as nan but we need int; round is necessary otherwise 0.999999 is converted to 0 which is wrong
df.comparator = df.comparator.round().astype(int) =
# add observer and node ID
df['obsid'] = obsid
df['nodeid'] = nodeid
# convert comparator ID to variable name
df['varname'] = df.comparator.apply(lambda x: (varnames[x] if x < len(varnames) else str(x)))
# append datatrace elements from obsever to datatrace log file
with open(resultfile_path, "a") as outfile:
columns=['global_ts', 'obsid', 'nodeid', 'varname', 'data', 'operation', 'PC'],
......@@ -40,46 +40,36 @@ import pandas as pd
import collections
# create the pandas data frame to store the parsed values in
df = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
# df = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
df_append = pd.DataFrame(index=["comp0", "comp1", "comp2", "comp3"],
columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
# create a series used in the case we only have a timestamp and no packets (local ts overflow)
nan = np.nan
new_row = pd.Series([nan, nan, nan, nan, nan])
index_ = ['global_ts', 'data', 'PC', 'operation', 'local_ts']
new_row.index = index_
# solution w/o global vars would be to define the df and new_row as static variables in parser and then somehow pass
# the current df upwards to the parse_fun every time there could be a program stop.
# parse_fun will then directly create the csv file.
def parse_dwt_output(input_file='swo_read_log', output_file='swo_read_log.csv'):
def parse_dwt_output(input_file):
Executes the read and parse functions which will read from the given input_file and parse the content
It will save the parsed contents in the file specified as second argument
input_file (str): name of the file to parse
output_file (str): name of the file to parse
int: True if the program was halted by Key interrupt
df: dataframe containing the parsed data
global df
# make sure the dataframe is empty
df = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
swo_queue = collections.deque()
global_ts_queue = collections.deque()
# read raw file into queues
read_fun(swo_queue, global_ts_queue, input_file)
parse_fun(swo_queue, global_ts_queue)
# convert the pandas data frame to a csv file
df.to_csv(output_file, index=False, header=True)
# parse data in queues and generate dataframe
df = parse_fun(swo_queue, global_ts_queue)
return 0 # exit the program execution
return df
def read_fun(swo_queue, global_ts_queue, input_file):
......@@ -88,7 +78,6 @@ def read_fun(swo_queue, global_ts_queue, input_file):
It also handles special cases by putting a varying number of global timestamps into the queue
# global new_row
data_is_next = True # we start with data
local_ts_count = 0
global_ts_count = 0
......@@ -156,17 +145,13 @@ def read_fun(swo_queue, global_ts_queue, input_file):
def parse_fun(swo_queue, global_ts_queue):
Calls the parse function on packets from the queue in a loop until queue is empty
Parses packets from the queue
df_out = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
while swo_queue:
swo_byte = swo_queue.pop()
parser(swo_byte, swo_queue, global_ts_queue)
def parser(swo_byte, swo_queue, global_ts_queue):
Parses packets from the queue
# sync packet, problem: in begin we have don't have 5 zero bytes as specified for a sync pack but there are 9
# Just read all zeros until get an 0x80, then we are in sync.
if swo_byte == 0:
......@@ -182,7 +167,8 @@ def parser(swo_byte, swo_queue, global_ts_queue):
# one byte local TS
pass # do not comment, returning if detected is required!
elif lower_bytes == 0x00:
timestamp_parse(swo_queue, global_ts_queue)
new_row = parse_timestamp(swo_queue, global_ts_queue)
df_out = df_out.append(new_row, ignore_index=True)
elif lower_bytes == 0x04:
# reserved
pass # do not comment, returning if detected is required!
......@@ -198,6 +184,8 @@ def parser(swo_byte, swo_queue, global_ts_queue):
raise Exception("ERROR: unrecognized SWO byte: {}".format(swo_byte))
return df_out
def parse_hard(header_swo_byte, swo_queue):
......@@ -244,12 +232,11 @@ def parse_hard(header_swo_byte, swo_queue):
raise Exception('ERROR: Unknown data trace packet type observed!')
def timestamp_parse(swo_queue, global_ts_queue):
def parse_timestamp(swo_queue, global_ts_queue):
Parses timestamp packets and writes a line into output file after every timestamp
global df_append
global df
buf = [0, 0, 0, 0]
i = 0
local_ts_delta = 0
......@@ -275,47 +262,51 @@ def timestamp_parse(swo_queue, global_ts_queue):
if not empty['comp0']:['comp0', 'local_ts'] = local_ts_delta['comp0', 'global_ts'] = global_ts
series0 = df_append.loc['comp0']
df = df.append(series0, ignore_index=True)
new_row = df_append.loc['comp0'].copy()
elif not empty['comp1']:['comp1', 'local_ts'] = local_ts_delta['comp1', 'global_ts'] = global_ts
series1 = df_append.loc['comp1']
df = df.append(series1, ignore_index=True)
new_row = df_append.loc['comp1'].copy()
elif not empty['comp2']:['comp2', 'local_ts'] = local_ts_delta['comp2', 'global_ts'] = global_ts
series2 = df_append.loc['comp2']
df = df.append(series2, ignore_index=True)
new_row = df_append.loc['comp2'].copy()
elif not empty['comp3']:['comp3', 'local_ts'] = local_ts_delta['comp3', 'global_ts'] = global_ts
series3 = df_append.loc['comp3']
df = df.append(series3, ignore_index=True)
new_row = df_append.loc['comp3'].copy()
# overflow was received, so no comparator data, only global and local ts
elif empty['comp0'] and empty['comp1'] and empty['comp2'] and empty['comp3']:
# create a series used in the case we only have a timestamp and no packets (local ts overflow)
new_row = pd.Series([np.nan, np.nan, np.nan, np.nan, np.nan])
new_row.index = ['global_ts', 'data', 'PC', 'operation', 'local_ts']["local_ts"] = local_ts_delta['global_ts'] = global_ts
df = df.append(new_row, ignore_index=True)
# reset the df_append to nan values
for col in df_append.columns:
df_append[col].values[:] = np.nan
return new_row
def correct_ts_with_regression(input_file='swo_read_log.csv', output_file='swo_read_log_corrected.csv'):
def correct_ts_with_regression(df_in):
Calculates a regression based on the values in log_table.csv
Then projects the global timestamps onto the regression and writes the corrected values in log_table_corrected.csv
df_in: dataframe containing parsed data
df_out: dataframe containing time corrected data
# read data in the file into a pandas data frame
df = pd.read_csv(input_file)
df_out = df_in.copy()
# extract the global and local timestamps and put into a numpy array
x = df['local_ts'].to_numpy(dtype=float)
y = df['global_ts'].to_numpy(dtype=float)
x = df_out['local_ts'].to_numpy(dtype=float)
y = df_out['global_ts'].to_numpy(dtype=float)
# add up the local timestamps and calculate the global timestamp relative to the first global timestamp
sum_local_ts = 0
......@@ -331,12 +322,13 @@ def correct_ts_with_regression(input_file='swo_read_log.csv', output_file='swo_r
global_ts[...] = b[0] + b[1] * local_ts
# write the array back into the pandas df to replace the global timestamps
df['global_ts'] = y
# convert the pandas data frame to a csv file
df.to_csv(output_file, index=False, header=True)
df_out['global_ts'] = y
# The file log_table_corrected.csv now contains the corrected global timestamps together with the DWT packets
return df_out
def estimate_coef(x, y):
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment