Commit 704bd292 authored by Roman Trüb's avatar Roman Trüb
Browse files

further cleanup of dwt_parse.py (global vars removed, fixed parts of parsing)

parent 4cb06835
...@@ -444,8 +444,13 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue ...@@ -444,8 +444,13 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue
logqueue.put_nowait((loggername, logging.WARNING, "Empty data trace results file.")) logqueue.put_nowait((loggername, logging.WARNING, "Empty data trace results file."))
else: else:
df = df_corrected df = df_corrected
# remove lines with timestamp values # remove timestamp rows (which contain nan values) -> drop corresponding lines, note: PC column can contain nan!
df.dropna(inplace=True) df.dropna(subset=['comparator', 'operation'], inplace=True)
# convert columns to int if required; if there were nan values, comparator column was stored as nan but we need int; round is necessary otherwise 0.999999 is converted to 0 which is wrong
if 'float' in str(df_corrected.comparator.dtypes):
df_corrected.comparator = df_corrected.comparator.round().astype(int)
if 'float' in str(df_corrected.data.dtypes):
df_corrected.data = df_corrected.data.round().astype(int)
# add observer and node ID # add observer and node ID
df['obsid'] = obsid df['obsid'] = obsid
df['nodeid'] = nodeid df['nodeid'] = nodeid
...@@ -469,8 +474,6 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue ...@@ -469,8 +474,6 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue
# shutil.copyfile(tmpfile1, "%s_uncorrected.csv" % resultfile_path) # shutil.copyfile(tmpfile1, "%s_uncorrected.csv" % resultfile_path)
# delete files # delete files
os.remove(input_filename) os.remove(input_filename)
os.remove(tmpfile1)
os.remove(tmpfile2)
processeditem = list(queueitem) processeditem = list(queueitem)
processeditem[0] = ITEM_PROCESSED processeditem[0] = ITEM_PROCESSED
return (_errors, tuple(processeditem)) return (_errors, tuple(processeditem))
......
...@@ -39,11 +39,6 @@ import numpy as np ...@@ -39,11 +39,6 @@ import numpy as np
import pandas as pd import pandas as pd
import collections import collections
# create the pandas data frame to store the parsed values in
# df = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
df_append = pd.DataFrame(index=["comp0", "comp1", "comp2", "comp3"],
columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
# solution w/o global vars would be to define the df and new_row as static variables in parser and then somehow pass # solution w/o global vars would be to define the df and new_row as static variables in parser and then somehow pass
# the current df upwards to the parse_fun every time there could be a program stop. # the current df upwards to the parse_fun every time there could be a program stop.
# parse_fun will then directly create the csv file. # parse_fun will then directly create the csv file.
...@@ -72,13 +67,24 @@ def parse_dwt_output(input_file): ...@@ -72,13 +67,24 @@ def parse_dwt_output(input_file):
return df return df
def map_size(ss):
if ss == 1:
return 1
elif ss == 2:
return 2
elif ss == 3:
return 4
else:
raise Exception('ERROR: Invalid ss size: ss should not be ==0 or >3')
def read_fun(swo_queue, global_ts_queue, input_file): def read_fun(swo_queue, global_ts_queue, input_file):
""" """
Reads from the input file and then puts values into the queue. Reads from the input file and then puts values into the queue.
It also handles special cases by putting a varying number of global timestamps into the queue It also handles special cases by putting a varying number of global timestamps into the queue
""" """
data_is_next = True # we start with data data_is_next = True # we expect that raw file starts with data (not with global timestamp)
local_ts_count = 0 local_ts_count = 0
global_ts_count = 0 global_ts_count = 0
currently_hw_packet = False # we start with zeros so next is header currently_hw_packet = False # we start with zeros so next is header
...@@ -86,44 +92,63 @@ def read_fun(swo_queue, global_ts_queue, input_file): ...@@ -86,44 +92,63 @@ def read_fun(swo_queue, global_ts_queue, input_file):
next_is_header = True # we start with zeros so next is header next_is_header = True # we start with zeros so next is header
current_packet_size = 0 current_packet_size = 0
with open(input_file) as open_file_object: with open(input_file) as open_file_object:
for line in open_file_object: for i, line in enumerate(open_file_object):
if data_is_next: # Test if this is a line with data if i == 0:
numbers = [] # initialise again, else has the numbers from previous line also continue # ignore first line with varnames
for word in line.split(): # extract the numbers in the line into a python list if data_is_next:
if word.isdigit(): # Line with data
numbers.append(int(word))
numbers = []
if currently_hw_packet: for word in line.split():
if current_packet_size: # still bytes left if not word.isdigit():
current_packet_size -= 1 raise Exception('ERROR: element of line is not digits as expected for a line with data')
else: # no more bytes in the hw packet => next is header
currently_hw_packet = False numbers.append(int(word))
next_is_header = True
if currently_hw_packet:
if currently_ts_packet: if current_packet_size: # still bytes left
if not (int(word) & 0x80): # means this is the last byte of ts, next byte is header current_packet_size -= 1
currently_ts_packet = False else: # no more bytes in the hw packet => next is header
next_is_header = True currently_hw_packet = False
next_is_header = True
if next_is_header:
if word == '192' or word == '208' or word == '224' or word == '240': if currently_ts_packet:
local_ts_count += 1 # need to find the local ts continuation_bit = int(word) & 0x80
currently_ts_packet = True if continuation_bit == 0: # continuation_bit==0 indicates that this is the last byte of the local timstamp packet
next_is_header = False currently_ts_packet = False
if word == '71' or word == '135' or word == '143': next_is_header = True
# TODO: handle overflow packets
if next_is_header:
# if word == '192' or word == '208' or word == '224' or word == '240':
if int(word) & 0b11001111 == 0b11000000:
# Local timestamp packet
local_ts_count += 1 # need to find the local ts
currently_ts_packet = True
next_is_header = False
# elif word == '71' or word == '135' or word == '143':
elif int(word) >> 2 & 0b1 == 0b1 and int(word) & 0b11 != 0b00:
# Hardware source packet
discriminator_id = int(word) >> 3 & 0b11111
if discriminator_id >= 8 and discriminator_id <= 23:
# Data tracing
currently_hw_packet = True currently_hw_packet = True
next_is_header = False next_is_header = False
current_packet_size = int(word) & 0x03 current_packet_size = map_size(int(word) & 0b11) - 1
else:
# Other packet (Event counter wrapping, Exception tracing, PC sampling)
next_is_header = False
current_packet_size = map_size(int(word) & 0b11) - 1
for byte in numbers: # data line for byte in numbers: # data line
swo_queue.appendleft(byte) # put all the data into the queue that the parsing function will read from swo_queue.appendleft(byte) # put all the data into the queue that the parsing function will read from
# now indicate that this is the end of a line
# swo_queue.appendleft(LINE_ENDS)
if numbers: if numbers:
data_is_next = False data_is_next = False
else: # If not it is a line with a global timestamp else:
# Line with global timestamp
data_is_next = True data_is_next = True
global_ts_count += 1 global_ts_count += 1
if global_ts_count > local_ts_count: # case where had a global ts in middle of packet if global_ts_count > local_ts_count: # case where had a global ts in middle of packet
...@@ -139,7 +164,7 @@ def read_fun(swo_queue, global_ts_queue, input_file): ...@@ -139,7 +164,7 @@ def read_fun(swo_queue, global_ts_queue, input_file):
# finished reading all lines # finished reading all lines
open_file_object.close() open_file_object.close()
# # debug # # DEBUG
# print("read function ended (timestamp queue size: %d, swo queue size: %d)" % (len(global_ts_queue), len(swo_queue))) # print("read function ended (timestamp queue size: %d, swo queue size: %d)" % (len(global_ts_queue), len(swo_queue)))
...@@ -148,6 +173,8 @@ def parse_fun(swo_queue, global_ts_queue): ...@@ -148,6 +173,8 @@ def parse_fun(swo_queue, global_ts_queue):
Parses packets from the queue Parses packets from the queue
""" """
df_out = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts']) df_out = pd.DataFrame(columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
df_append = pd.DataFrame(index=["comp0", "comp1", "comp2", "comp3"],
columns=['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts'])
while swo_queue: while swo_queue:
swo_byte = swo_queue.pop() swo_byte = swo_queue.pop()
...@@ -162,46 +189,35 @@ def parse_fun(swo_queue, global_ts_queue): ...@@ -162,46 +189,35 @@ def parse_fun(swo_queue, global_ts_queue):
# now in sync # now in sync
swo_byte = swo_queue.pop() # then get next byte swo_byte = swo_queue.pop() # then get next byte
lower_bytes = swo_byte & 0x0f if swo_byte & 0b11001111 == 0b11000000:
if lower_bytes == 0x00 and not swo_byte & 0x80: # 1-byte local timestamp has a zero in front (C = 0) # Local timestamp packet
# one byte local TS new_row_list = parse_timestamp(swo_queue, global_ts_queue, df_append)
pass # do not comment, returning if detected is required! df_out = df_out.append(new_row_list, ignore_index=True)
elif lower_bytes == 0x00: elif swo_byte >> 2 & 0b1 == 0b1 and swo_byte & 0b11 != 0b00:
new_row = parse_timestamp(swo_queue, global_ts_queue) # Hardware source packet
df_out = df_out.append(new_row, ignore_index=True) discriminator_id = swo_byte >> 3 & 0b11111
elif lower_bytes == 0x04: if discriminator_id in [0, 1, 2]:
# reserved # 0 Event counter wrapping, 1 Exception tracing, 2 PC sampling
pass # do not comment, returning if detected is required! pass
elif lower_bytes == 0x08: if discriminator_id >= 8 and discriminator_id <= 23:
# ITM ext # Data tracing
pass # do not comment, returning if detected is required! parse_hard(swo_byte, swo_queue, df_append)
elif lower_bytes == 0x0c:
# DWT ext
pass # do not comment, returning if detected is required!
else:
if swo_byte & 0x04:
parse_hard(swo_byte, swo_queue)
else: else:
raise Exception("ERROR: unrecognized SWO byte: {}".format(swo_byte)) # Other undefined packet
print("Unknown discriminator ID in hardware source packet header: {}".format(swo_byte))
# raise Exception("ERROR: Unknown discriminator ID in hardware source packet header: {}".format(swo_byte)) # packets with undefined discriminator_id appear sporadically -> we cannot throw error here
else:
print("unrecognized DWT packet header: {}".format(swo_byte))
return df_out return df_out
def parse_hard(header_swo_byte, swo_queue): def parse_hard(header_swo_byte, swo_queue, df_append):
""" """
Parses a DWT hardware packet Parses a DWT hardware packet
""" """
global df_append
# the given swo_byte is a header for a PC, an address, data read or data write packet # the given swo_byte is a header for a PC, an address, data read or data write packet
size_bytes = header_swo_byte & 0x03 size = map_size(header_swo_byte & 0b11)
if size_bytes == 3:
size = 4
elif size_bytes == 2:
size = 2
elif size_bytes == 1:
size = 1
else:
raise Exception("invalid packet size in swo header byte")
buf = [0, 0, 0, 0] buf = [0, 0, 0, 0]
for i in range(0, size): for i in range(0, size):
...@@ -214,7 +230,7 @@ def parse_hard(header_swo_byte, swo_queue): ...@@ -214,7 +230,7 @@ def parse_hard(header_swo_byte, swo_queue):
comparator_id = (header_swo_byte >> 4) & 0b11 # id is in bit 4 and 5 comparator_id = (header_swo_byte >> 4) & 0b11 # id is in bit 4 and 5
comparator_label = 'comp{}'.format(comparator_id) comparator_label = 'comp{}'.format(comparator_id)
# debug # DEBUG
# with open('/home/flocklab/tmp/log2.txt', 'a') as f_debug: # with open('/home/flocklab/tmp/log2.txt', 'a') as f_debug:
# f_debug.write('{}\n'.format(comparator_id)) # f_debug.write('{}\n'.format(comparator_id))
...@@ -232,11 +248,10 @@ def parse_hard(header_swo_byte, swo_queue): ...@@ -232,11 +248,10 @@ def parse_hard(header_swo_byte, swo_queue):
raise Exception('ERROR: Unknown data trace packet type observed!') raise Exception('ERROR: Unknown data trace packet type observed!')
def parse_timestamp(swo_queue, global_ts_queue): def parse_timestamp(swo_queue, global_ts_queue, df_append):
""" """
Parses timestamp packets and writes a line into output file after every timestamp Parses timestamp packets and writes a line into output file after every timestamp
""" """
global df_append
buf = [0, 0, 0, 0] buf = [0, 0, 0, 0]
i = 0 i = 0
local_ts_delta = 0 local_ts_delta = 0
...@@ -256,39 +271,43 @@ def parse_timestamp(swo_queue, global_ts_queue): ...@@ -256,39 +271,43 @@ def parse_timestamp(swo_queue, global_ts_queue):
# now we want to complete all the half filled rows in the df_append (contain only data, PC, operation, comparator) # now we want to complete all the half filled rows in the df_append (contain only data, PC, operation, comparator)
# there can be several data (in several rows) but we only have 1 local ts and 1 global ts to use for all of them # there can be several data (in several rows) but we only have 1 local ts and 1 global ts to use for all of them
empty = df_append[:].isnull().apply(lambda x: all(x), axis=1) empty = df_append[:].isnull().apply(lambda x: all(x), axis=1)
# DEBUG
# print('number of non-empty rows in df_append: {}'.format(len(empty) - np.sum(empty)))
# we only have one global timestamp for one local timestamp so need to pop and reuse # we only have one global timestamp for one local timestamp so need to pop and reuse
global_ts = global_ts_queue.pop() global_ts = global_ts_queue.pop()
ret = []
if not empty['comp0']: if not empty['comp0']:
df_append.at['comp0', 'local_ts'] = local_ts_delta df_append.at['comp0', 'local_ts'] = local_ts_delta
df_append.at['comp0', 'global_ts'] = global_ts df_append.at['comp0', 'global_ts'] = global_ts
new_row = df_append.loc['comp0'].copy() ret += [df_append.loc['comp0'].copy()]
elif not empty['comp1']: if not empty['comp1']:
df_append.at['comp1', 'local_ts'] = local_ts_delta df_append.at['comp1', 'local_ts'] = local_ts_delta
df_append.at['comp1', 'global_ts'] = global_ts df_append.at['comp1', 'global_ts'] = global_ts
new_row = df_append.loc['comp1'].copy() ret += [df_append.loc['comp1'].copy()]
elif not empty['comp2']: if not empty['comp2']:
df_append.at['comp2', 'local_ts'] = local_ts_delta df_append.at['comp2', 'local_ts'] = local_ts_delta
df_append.at['comp2', 'global_ts'] = global_ts df_append.at['comp2', 'global_ts'] = global_ts
new_row = df_append.loc['comp2'].copy() ret += [df_append.loc['comp2'].copy()]
elif not empty['comp3']: if not empty['comp3']:
df_append.at['comp3', 'local_ts'] = local_ts_delta df_append.at['comp3', 'local_ts'] = local_ts_delta
df_append.at['comp3', 'global_ts'] = global_ts df_append.at['comp3', 'global_ts'] = global_ts
new_row = df_append.loc['comp3'].copy() ret += [df_append.loc['comp3'].copy()]
# overflow was received, so no comparator data, only global and local ts if empty['comp0'] and empty['comp1'] and empty['comp2'] and empty['comp3']:
elif empty['comp0'] and empty['comp1'] and empty['comp2'] and empty['comp3']: # overflow was received, so no comparator data, only global and local ts
# create a series used in the case we only have a timestamp and no packets (local ts overflow) # create a series used in the case we only have a timestamp and no packets (local ts overflow)
new_row = pd.Series([np.nan, np.nan, np.nan, np.nan, np.nan]) new_row = pd.Series([np.nan, np.nan, np.nan, np.nan, np.nan])
new_row.index = ['global_ts', 'data', 'PC', 'operation', 'local_ts'] new_row.index = ['global_ts', 'data', 'PC', 'operation', 'local_ts']
new_row.at["local_ts"] = local_ts_delta new_row.at["local_ts"] = local_ts_delta
new_row.at['global_ts'] = global_ts new_row.at['global_ts'] = global_ts
ret += [new_row]
# reset the df_append to nan values # reset the df_append to nan values
for col in df_append.columns: for col in df_append.columns:
df_append[col].values[:] = np.nan df_append[col].values[:] = np.nan
return new_row return ret
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment