Commit f87f3386 authored by Roman Trüb's avatar Roman Trüb
Browse files

removed datatrace tmp files

parent b8fdf2f5
...@@ -435,27 +435,24 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue ...@@ -435,27 +435,24 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue
varnames = "" varnames = ""
with open(input_filename, "r") as f: with open(input_filename, "r") as f:
varnames = f.readline().strip().split() varnames = f.readline().strip().split()
(fd1, tmpfile1) = tempfile.mkstemp() # parse raw datatrace log
(fd2, tmpfile2) = tempfile.mkstemp() df_parsed = dwt.parse_dwt_output(input_filename)
dwt.parse_dwt_output(input_filename, tmpfile1)
# apply linear regression to correct the timestamps # apply linear regression to correct the timestamps
try: try:
dwt.correct_ts_with_regression(tmpfile1, tmpfile2) df_corrected = dwt.correct_ts_with_regression(df_parsed)
except ValueError: except ValueError:
logqueue.put_nowait((loggername, logging.WARNING, "Empty data trace results file.")) logqueue.put_nowait((loggername, logging.WARNING, "Empty data trace results file."))
else: else:
with open(resultfile_path, "a") as outfile: df = df_corrected
df = pd.read_csv(tmpfile2) # remove lines with timestamp values
# debug
df.to_csv('/home/flocklab/tmp/tmp_df.txt')
# nan values cannot be converted to int -> drop corresponding lines
df.dropna(inplace=True) df.dropna(inplace=True)
# since there were nan values, comparator column was stored as nan but we need int; round is necessary otherwise 0.999999 is converted to 0 which is wrong # add observer and node ID
df.comparator = df.comparator.round().astype(int)
df.data = df.data.round().astype(int)
df['obsid'] = obsid df['obsid'] = obsid
df['nodeid'] = nodeid df['nodeid'] = nodeid
# convert comparator ID to variable name
df['varname'] = df.comparator.apply(lambda x: (varnames[x] if x < len(varnames) else str(x))) df['varname'] = df.comparator.apply(lambda x: (varnames[x] if x < len(varnames) else str(x)))
# append datatrace elements from obsever to datatrace log file
with open(resultfile_path, "a") as outfile:
df.to_csv( df.to_csv(
outfile, outfile,
columns=['global_ts', 'obsid', 'nodeid', 'varname', 'data', 'operation', 'PC'], columns=['global_ts', 'obsid', 'nodeid', 'varname', 'data', 'operation', 'PC'],
......
...@@ -40,46 +40,36 @@ import pandas as pd ...@@ -40,46 +40,36 @@ import pandas as pd
import collections import collections
# create the pandas data frame to store the parsed values in # create the pandas data frame to store the parsed values in
df = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts']) # df = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
df_append = pd.DataFrame(index=["comp0", "comp1", "comp2", "comp3"], df_append = pd.DataFrame(index=["comp0", "comp1", "comp2", "comp3"],
columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts']) columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
# create a series used in the case we only have a timestamp and no packets (local ts overflow)
nan = np.nan
new_row = pd.Series([nan, nan, nan, nan, nan])
index_ = ['global_ts', 'data', 'PC', 'operation', 'local_ts']
new_row.index = index_
# solution w/o global vars would be to define the df and new_row as static variables in parser and then somehow pass # solution w/o global vars would be to define the df and new_row as static variables in parser and then somehow pass
# the current df upwards to the parse_fun every time there could be a program stop. # the current df upwards to the parse_fun every time there could be a program stop.
# parse_fun will then directly create the csv file. # parse_fun will then directly create the csv file.
def parse_dwt_output(input_file='swo_read_log', output_file='swo_read_log.csv'): def parse_dwt_output(input_file):
""" """
Executes the read and parse functions which will read from the given input_file and parse the content Executes the read and parse functions which will read from the given input_file and parse the content
It will save the parsed contents in the file specified as second argument It will save the parsed contents in the file specified as second argument
Parameters: Parameters:
input_file (str): name of the file to parse input_file (str): name of the file to parse
output_file (str): name of the file to parse
Returns: Returns:
int: True if the program was halted by Key interrupt df: dataframe containing the parsed data
""" """
global df
# make sure the dataframe is empty
df = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
swo_queue = collections.deque() swo_queue = collections.deque()
global_ts_queue = collections.deque() global_ts_queue = collections.deque()
# read raw file into queues
read_fun(swo_queue, global_ts_queue, input_file) read_fun(swo_queue, global_ts_queue, input_file)
parse_fun(swo_queue, global_ts_queue)
# convert the pandas data frame to a csv file # parse data in queues and generate dataframe
df.to_csv(output_file, index=False, header=True) df = parse_fun(swo_queue, global_ts_queue)
return 0 # exit the program execution return df
def read_fun(swo_queue, global_ts_queue, input_file): def read_fun(swo_queue, global_ts_queue, input_file):
...@@ -88,7 +78,6 @@ def read_fun(swo_queue, global_ts_queue, input_file): ...@@ -88,7 +78,6 @@ def read_fun(swo_queue, global_ts_queue, input_file):
It also handles special cases by putting a varying number of global timestamps into the queue It also handles special cases by putting a varying number of global timestamps into the queue
""" """
# global new_row
data_is_next = True # we start with data data_is_next = True # we start with data
local_ts_count = 0 local_ts_count = 0
global_ts_count = 0 global_ts_count = 0
...@@ -156,17 +145,13 @@ def read_fun(swo_queue, global_ts_queue, input_file): ...@@ -156,17 +145,13 @@ def read_fun(swo_queue, global_ts_queue, input_file):
def parse_fun(swo_queue, global_ts_queue): def parse_fun(swo_queue, global_ts_queue):
""" """
Calls the parse function on packets from the queue in a loop until queue is empty Parses packets from the queue
""" """
df_out = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
while swo_queue: while swo_queue:
swo_byte = swo_queue.pop() swo_byte = swo_queue.pop()
parser(swo_byte, swo_queue, global_ts_queue)
def parser(swo_byte, swo_queue, global_ts_queue):
"""
Parses packets from the queue
"""
# sync packet, problem: in begin we have don't have 5 zero bytes as specified for a sync pack but there are 9 # sync packet, problem: in begin we have don't have 5 zero bytes as specified for a sync pack but there are 9
# Just read all zeros until get an 0x80, then we are in sync. # Just read all zeros until get an 0x80, then we are in sync.
if swo_byte == 0: if swo_byte == 0:
...@@ -182,7 +167,8 @@ def parser(swo_byte, swo_queue, global_ts_queue): ...@@ -182,7 +167,8 @@ def parser(swo_byte, swo_queue, global_ts_queue):
# one byte local TS # one byte local TS
pass # do not comment, returning if detected is required! pass # do not comment, returning if detected is required!
elif lower_bytes == 0x00: elif lower_bytes == 0x00:
timestamp_parse(swo_queue, global_ts_queue) new_row = parse_timestamp(swo_queue, global_ts_queue)
df_out = df_out.append(new_row, ignore_index=True)
elif lower_bytes == 0x04: elif lower_bytes == 0x04:
# reserved # reserved
pass # do not comment, returning if detected is required! pass # do not comment, returning if detected is required!
...@@ -198,6 +184,8 @@ def parser(swo_byte, swo_queue, global_ts_queue): ...@@ -198,6 +184,8 @@ def parser(swo_byte, swo_queue, global_ts_queue):
else: else:
raise Exception("ERROR: unrecognized SWO byte: {}".format(swo_byte)) raise Exception("ERROR: unrecognized SWO byte: {}".format(swo_byte))
return df_out
def parse_hard(header_swo_byte, swo_queue): def parse_hard(header_swo_byte, swo_queue):
""" """
...@@ -244,12 +232,11 @@ def parse_hard(header_swo_byte, swo_queue): ...@@ -244,12 +232,11 @@ def parse_hard(header_swo_byte, swo_queue):
raise Exception('ERROR: Unknown data trace packet type observed!') raise Exception('ERROR: Unknown data trace packet type observed!')
def timestamp_parse(swo_queue, global_ts_queue): def parse_timestamp(swo_queue, global_ts_queue):
""" """
Parses timestamp packets and writes a line into output file after every timestamp Parses timestamp packets and writes a line into output file after every timestamp
""" """
global df_append global df_append
global df
buf = [0, 0, 0, 0] buf = [0, 0, 0, 0]
i = 0 i = 0
local_ts_delta = 0 local_ts_delta = 0
...@@ -275,47 +262,51 @@ def timestamp_parse(swo_queue, global_ts_queue): ...@@ -275,47 +262,51 @@ def timestamp_parse(swo_queue, global_ts_queue):
if not empty['comp0']: if not empty['comp0']:
df_append.at['comp0', 'local_ts'] = local_ts_delta df_append.at['comp0', 'local_ts'] = local_ts_delta
df_append.at['comp0', 'global_ts'] = global_ts df_append.at['comp0', 'global_ts'] = global_ts
series0 = df_append.loc['comp0'] new_row = df_append.loc['comp0'].copy()
df = df.append(series0, ignore_index=True)
elif not empty['comp1']: elif not empty['comp1']:
df_append.at['comp1', 'local_ts'] = local_ts_delta df_append.at['comp1', 'local_ts'] = local_ts_delta
df_append.at['comp1', 'global_ts'] = global_ts df_append.at['comp1', 'global_ts'] = global_ts
series1 = df_append.loc['comp1'] new_row = df_append.loc['comp1'].copy()
df = df.append(series1, ignore_index=True)
elif not empty['comp2']: elif not empty['comp2']:
df_append.at['comp2', 'local_ts'] = local_ts_delta df_append.at['comp2', 'local_ts'] = local_ts_delta
df_append.at['comp2', 'global_ts'] = global_ts df_append.at['comp2', 'global_ts'] = global_ts
series2 = df_append.loc['comp2'] new_row = df_append.loc['comp2'].copy()
df = df.append(series2, ignore_index=True)
elif not empty['comp3']: elif not empty['comp3']:
df_append.at['comp3', 'local_ts'] = local_ts_delta df_append.at['comp3', 'local_ts'] = local_ts_delta
df_append.at['comp3', 'global_ts'] = global_ts df_append.at['comp3', 'global_ts'] = global_ts
series3 = df_append.loc['comp3'] new_row = df_append.loc['comp3'].copy()
df = df.append(series3, ignore_index=True)
# overflow was received, so no comparator data, only global and local ts # overflow was received, so no comparator data, only global and local ts
elif empty['comp0'] and empty['comp1'] and empty['comp2'] and empty['comp3']: elif empty['comp0'] and empty['comp1'] and empty['comp2'] and empty['comp3']:
# create a series used in the case we only have a timestamp and no packets (local ts overflow)
new_row = pd.Series([np.nan, np.nan, np.nan, np.nan, np.nan])
new_row.index = ['global_ts', 'data', 'PC', 'operation', 'local_ts']
new_row.at["local_ts"] = local_ts_delta new_row.at["local_ts"] = local_ts_delta
new_row.at['global_ts'] = global_ts new_row.at['global_ts'] = global_ts
df = df.append(new_row, ignore_index=True)
# reset the df_append to nan values # reset the df_append to nan values
for col in df_append.columns: for col in df_append.columns:
df_append[col].values[:] = np.nan df_append[col].values[:] = np.nan
return new_row
def correct_ts_with_regression(input_file='swo_read_log.csv', output_file='swo_read_log_corrected.csv'):
def correct_ts_with_regression(df_in):
""" """
Calculates a regression based on the values in log_table.csv Calculates a regression based on the values in log_table.csv
Then projects the global timestamps onto the regression and writes the corrected values in log_table_corrected.csv Then projects the global timestamps onto the regression and writes the corrected values in log_table_corrected.csv
Params:
df_in: dataframe containing parsed data
Returns:
df_out: dataframe containing time corrected data
""" """
# read data in the file into a pandas data frame df_out = df_in.copy()
df = pd.read_csv(input_file)
# extract the global and local timestamps and put into a numpy array # extract the global and local timestamps and put into a numpy array
x = df['local_ts'].to_numpy(dtype=float) x = df_out['local_ts'].to_numpy(dtype=float)
y = df['global_ts'].to_numpy(dtype=float) y = df_out['global_ts'].to_numpy(dtype=float)
# add up the local timestamps and calculate the global timestamp relative to the first global timestamp # add up the local timestamps and calculate the global timestamp relative to the first global timestamp
sum_local_ts = 0 sum_local_ts = 0
...@@ -331,12 +322,13 @@ def correct_ts_with_regression(input_file='swo_read_log.csv', output_file='swo_r ...@@ -331,12 +322,13 @@ def correct_ts_with_regression(input_file='swo_read_log.csv', output_file='swo_r
global_ts[...] = b[0] + b[1] * local_ts global_ts[...] = b[0] + b[1] * local_ts
# write the array back into the pandas df to replace the global timestamps # write the array back into the pandas df to replace the global timestamps
df['global_ts'] = y df_out['global_ts'] = y
# convert the pandas data frame to a csv file
df.to_csv(output_file, index=False, header=True)
# The file log_table_corrected.csv now contains the corrected global timestamps together with the DWT packets # The file log_table_corrected.csv now contains the corrected global timestamps together with the DWT packets
return df_out
def estimate_coef(x, y): def estimate_coef(x, y):
""" """
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment