To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 8ac600aa authored by Roman Trüb's avatar Roman Trüb
Browse files

prescaler and loopdelay values from flocklab config.ini file are now used for...

prescaler and loopdelay values from flocklab config.ini file are now used for datatrace postprocessing
made piecewise regression work with arbitrary cpuSpeed
removed code for offset correction based on reset
parent d0495833
......@@ -477,11 +477,18 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, resultfi
## parse the file
# first line of the log file contains the variable names
varnames = ""
cpuSpeed = None
if arg:
cpuSpeed = arg
else:
raise Exception('A datatrace error occurred: cpuSpeed not provided!')
with open(input_filename, "r") as f:
varnames = f.readline().strip().split()[:-1] # ignore last element (sleep_overhead value)
try:
# process raw datatrace log (parse & apply time correction)
dfData, dfLocalTs, dfOverflow, dfError = dwt.processDatatraceOutput(input_filename)
dfData, dfLocalTs, dfOverflow, dfError = dwt.processDatatraceOutput(input_filename, cpuSpeed)
except Exception as e:
write_to_error_log('{}'.format(time.time()), obsid, 'A datatrace error occurred when processing raw output ({}). Potential cause: SWO/CPU speed mismatch (see cpuSpeed tag in xml config) or target did not start properly.'.format(e))
else:
......@@ -1070,12 +1077,25 @@ def main(argv):
else:
servicesUsed_dict[service] = False
# check which file format the user wants for the power profiling
# FIXME: this implementation assumes that the same file format is configured for all observers but this does not hold in general since there can be multiple powerProfilingConf blocks
if servicesUsed_dict['powerprofiling']:
if tree.xpath('//d:powerProfilingConf/d:fileFormat', namespaces=ns):
ppFileFormat = tree.xpath('//d:powerProfilingConf/d:fileFormat', namespaces=ns)[0].text
logger.debug("User wants file format %s for power profiling." % (ppFileFormat))
else:
logger.debug("Element <fileFormat> not detected.")
# extract cpuSpeed for datatracing (for all observer configured for datatracing)
dtCpuSpeed = {}
if servicesUsed_dict['datatrace']:
if tree.xpath('//d:debugConf', namespaces=ns):
for debugConf in tree.xpath('//d:debugConf', namespaces=ns):
# print(lxml.etree.tostring(debugConf, pretty_print=True).decode()) # DEBUG
obsList = [int(obsIdStr) for obsIdStr in debugConf.xpath('.//d:obsIds', namespaces=ns)[0].text.split(' ')]
cpuSpeed = int(debugConf.xpath('.//d:cpuSpeed', namespaces=ns)[0].text)
for obsId in obsList:
dtCpuSpeed[obsId] = cpuSpeed
if len(dtCpuSpeed) == 0:
logger.debug("Element <cpuSpeed> not detected.")
except:
msg = "XML parsing failed: %s: %s" % (str(sys.exc_info()[0]), str(sys.exc_info()[1]))
errors.append(msg)
......@@ -1262,7 +1282,7 @@ def main(argv):
worker_f = worker_serial
elif (re.search("^datatrace_[0-9]{14}\.log$", f) != None):
pool = service_pools_dict['datatrace']
worker_args = [nextitem, nodeid, testresultsfile_dict['datatrace'][0], testresultsfile_dict['datatrace'][1], logqueue, None]
worker_args = [nextitem, nodeid, testresultsfile_dict['datatrace'][0], testresultsfile_dict['datatrace'][1], logqueue, dtCpuSpeed[obsid]]
worker_f = worker_datatrace
elif (re.search("^error_[0-9]{14}\.log$", f) != None):
pool = service_pools_dict['logs']
......
......@@ -40,33 +40,33 @@ import numpy as np
import pandas as pd
from scipy import stats, interpolate
import collections
# import lib.flocklab as flocklab # imported only when needed
################################################################################
# Constants
################################################################################
# data trace configs
FULL_TIMESTAMP = 1999999 # timestamp in LocalTimestampPkt when overflow happened
# see ARM CoreSight Components Technical Reference Manual
PRESCALER = 16 # prescaler configured in Trace Control Register (ITM_TCR)
# NOTE: needs to match the settings on the observer!
# time offset between datatrace and GPIO service (ts_datatrace + offset = ts_gpio)
DT_FIXED_OFFSET = -5.1e-3 # shift half of loop delay plus 0.1ms from other delays
# DT_FIXED_OFFSET_RESET = +11.65e-3 # time correction based on reset is disabled, see comment at end of timeCorrection()
# outlier filter
FILTER_THRESHOLD = 0.15 # Threshold for percentage of filtered messages to produce an error
# data trace data
FULL_TIMESTAMP = 1999999 # timestamp in LocalTimestampPkt when overflow happened
# see ARM CoreSight Components Technical Reference Manual
# outlier filter for time sync points
FILTER_THRESHOLD = 0.15 # Threshold for percentage of filtered messages to produce an error (range [0, 1])
RESIDUAL_UNFILTERED_THRESHOLD = 0.300 # Threshold for residuals magnitude to producing error (in seconds)
# timestamp correction based on Linux platform
# time correction
DT_FIXED_OFFSET_ADDITIONAL = 0.1e-3 # mean offset in addition to half of loop delay
# time correction based on Linux platform
# old platform: 4.14.108-ti-r124
# new platform: 4.14.108-ti-r134/r137
PLATFORM_SLEEP_OVERHEAD_THRESHOLD = 0.300e-3 # threshold of the measured sleepOverhead to identify platform (Linux version)
PLATFORM_WINDOW = 2 # window for calculating rolling byte rate
PLATFORM_BYTE_RATE_THRESHOLD = 500 # threshold on generated byte rate to apply offset correction (for old linux platform only)
PLATFORM_WINDOW = 2 # time window for calculating rolling byte rate (in seconds)
PLATFORM_BYTE_RATE_THRESHOLD = 500 # threshold on generated byte rate to apply offset correction (for old linux platform only) (in Bytes/s)
PLATFORM_OFFSET = 1.25e-3 # offset for offset correction (for old linux platform only)
# regression
PIECEWISE_REGRESSION = True
PIECEWISE_REGRESSION_SEGMENT_LENGTH = 120 # in seconds
PIECEWISE_REGRESSION = True # if True, use piecewise regression; if False, use single linear regression
PIECEWISE_REGRESSION_SEGMENT_LENGTH = 120 # length of equally spaced non-overlapping segments (in seconds)
################################################################################
# SwoParser Class
......@@ -316,24 +316,36 @@ class SwoParser():
################################################################################
# METHODS
################################################################################
def processDatatraceOutput(input_file):
def processDatatraceOutput(input_file, cpuSpeed, prescaler=None, loopdelay=None):
"""
Executes the read, parse, timestamp adding, and time correction functions to
parse a raw datatrace file into a dataframe of results.
Parameters:
input_file (str): path to the raw data trace file
input_file: path to the raw data trace file (str)
cpuSpeed: CPU frequency of target (required)
prescaler: prescaler config for local timestamp counter (ITM_TCR reg) on the target (default: config from flocklab config.ini)
loopdelay: loop delay (in seconds) for fetching SWO output from debug probe on observer (default: config from flocklab config.ini)
Returns:
dfData: dataframe containing the data trace data
dfLocalTs: dataframe containing the local timestamps
dfData: dataframe containing the data trace data
dfLocalTs: dataframe containing the local timestamps
dfOverflow: dataframe containing the overflow packets
dfError: dataframe containing messages about errors which occurred during processing (note this only contains errors that are not raised!)
dfError: dataframe containing messages about errors which occurred during processing (note this only contains errors that are not raised!)
"""
# # DEBUG
# import matplotlib.pyplot as plt
# plt.close('all')
# read prescaler and loopdelay from flocklab config.ini if not provided as argument
if prescaler is None or loopdelay is None:
import lib.flocklab as flocklab
flocklab.load_config()
if prescaler is None:
prescaler = flocklab.config.getint('observer', 'datatrace_prescaler')
if loopdelay is None:
loopdelay = flocklab.config.getint('observer', 'datatrace_loopdelay')*1e-3 # loopdelay is given in ms in config.ini -> convert to seconds
# read raw file into list
dataTsList, sleepOverhead = readRaw(input_file)
......@@ -360,12 +372,12 @@ def processDatatraceOutput(input_file):
# print([type(e).__name__ for e in batch])
# combine data packets and add localTs
dfData, dfLocalTs, dfOverflow = combinePkts(batchList)
dfData, dfLocalTs, dfOverflow = combinePkts(batchList, prescaler)
if len(dfLocalTs) < 2:
raise Exception('ERROR: dfLocalTs is empty or does not contain enough pkts -> unable to apply time correction!')
dfDataCorr, dfLocalTsCorr = timeCorrection(dfData, dfLocalTs, sleepOverhead, firstSyncEpoch=(i==0))
dfDataCorr, dfLocalTsCorr = timeCorrection(dfData, dfLocalTs, sleepOverhead, cpuSpeed, prescaler, loopdelay)
dfDataCorrList.append(dfDataCorr)
dfLocalTsCorrList.append(dfLocalTsCorr)
......@@ -553,7 +565,7 @@ def splitEpochs(pktList):
return batchList
def combinePkts(batchList):
def combinePkts(batchList, prescaler):
"""
Combines data packets and adds localTs
"""
......@@ -588,7 +600,7 @@ def combinePkts(batchList):
raise Exception('ERROR: batch does not contain exactly 1 reference packet (contains {} LocalTimestampPkt and {} OverflowPkt)!'.format(len(localTsPkts), len(overflowPkts)))
if localTsPkts:
localTsCum += localTsPkts[0].ts + 1/PRESCALER # +1 cycle (scaled by prescaler) because transition from last sent value to 0 takes one clock cycle (see ARM CoreSight Components Technical Reference Manual, p. 302)
localTsCum += localTsPkts[0].ts + 1/prescaler # +1 cycle (scaled by prescaler) because transition from last sent value to 0 takes one clock cycle (see ARM CoreSight Components Technical Reference Manual, p. 302)
# process data pkts
while dataPkts:
......@@ -642,20 +654,27 @@ def combinePkts(batchList):
return dfData, dfLocalTs, dfOverflow
def timeCorrection(dfData, dfLocalTs, sleepOverhead, firstSyncEpoch):
def timeCorrection(dfData, dfLocalTs, sleepOverhead, cpuSpeed, prescaler, loopdelay):
"""
Calculates a regression based on the values in dfLocalTs and adds corrected global timestamps.
Params:
dfData: dataframe containing data trace data
dfLocalTs: dataframe containing local timestamp data
sleepOverhead: overhead of calling time.sleep on the corresponding observer
firstSyncEpoch: Indicate if this is the first sync epoch (only relevent for offset correction based on initial reset -> not used anymore)
dfData: dataframe containing data trace data
dfLocalTs: dataframe containing local timestamp data
sleepOverhead: overhead of calling time.sleep on the corresponding observer
cpuSpeed: CPU frequency of target (required)
prescaler: prescaler config for local timestamp counter (ITM_TCR reg) on the target
loopdelay: loop delay (in seconds) for fetching SWO output from debug probe on observer
Returns:
dfDataCorr: dataframe with added corrected global timestamps
dfLocalTsCorr: dataframe with added corrected global timestamps
dfDataCorr: dataframe with added corrected global timestamps
dfLocalTsCorr: dataframe with added corrected global timestamps
"""
# mean offset due to loopdelay and other delays between local and global timestamp
fixedOffset = (1/2)*loopdelay + DT_FIXED_OFFSET_ADDITIONAL # 1/2: expected mean value is half of loopdelay
# factor to convert seconds to local timestamp counter ticks
secondsToLocalTsTicks = cpuSpeed/prescaler
regIsMonotonic = False
dfDataCorr = dfData.copy()
dfLocalTsCorr = dfLocalTs.copy()
......@@ -663,8 +682,8 @@ def timeCorrection(dfData, dfLocalTs, sleepOverhead, firstSyncEpoch):
# make sure only local timestamp of which are synchronized are used for regression
df = dfLocalTsCorr[dfLocalTsCorr.tc == 0]
x = df['local_ts'].to_numpy(dtype=float)
y = df['global_ts_uncorrected'].to_numpy(dtype=float) + DT_FIXED_OFFSET
x = df['local_ts'].to_numpy(dtype=float) # in local timestamp counter ticks
y = df['global_ts_uncorrected'].to_numpy(dtype=float) - fixedOffset # in seconds
# offset correction for old linux platform
# measured sleepOverhead is used to identify platform (linux version)
......@@ -694,7 +713,7 @@ def timeCorrection(dfData, dfLocalTs, sleepOverhead, firstSyncEpoch):
if PIECEWISE_REGRESSION:
# piecewise regression -> interpolate.splrep()
totalTime = x[-1] - x[0]
numSegments = max(1, np.floor(totalTime/(PIECEWISE_REGRESSION_SEGMENT_LENGTH*3e6))) # FIXME: hard-coded factor (3e6) to convert 1s into local timestamp counter cycles depends on clock speed (here 48MHz) and prescaler (here 16)
numSegments = max(1, np.floor(totalTime/(PIECEWISE_REGRESSION_SEGMENT_LENGTH*secondsToLocalTsTicks)))
# TODO: determine number of segments based on duration of 1 full timestamp (timediff between local timestamps with ts=1999999)
internalKnots = np.linspace(x[0], x[-1], num=numSegments, endpoint=False)[1:]
tckUnfiltered = interpolate.splrep(x, y, k=3, s=0, t=internalKnots)
......@@ -749,7 +768,7 @@ def timeCorrection(dfData, dfLocalTs, sleepOverhead, firstSyncEpoch):
# generate warning if regression splines are not monotoincally increasing (i.e. ordering of events on the same node is not necessary preserved)
if PIECEWISE_REGRESSION:
xEval = np.arange(x[0], x[-1], PIECEWISE_REGRESSION_SEGMENT_LENGTH*3e6/5) # FIXME: hard-coded factor (3e6) to convert 1s into local timestamp counter cycles depends on clock speed (here 48MHz) and prescaler (here 16)
xEval = np.arange(x[0], x[-1], PIECEWISE_REGRESSION_SEGMENT_LENGTH*secondsToLocalTsTicks/5)
yEval = interpolate.splev(xEval, tckFinal, der=0)
regIsMonotonic = np.all(np.diff(yEval) > 0)
else:
......@@ -821,21 +840,6 @@ def timeCorrection(dfData, dfLocalTs, sleepOverhead, firstSyncEpoch):
dfDataCorr['global_ts'] = dfDataCorr.local_ts * slopeFinal + interceptFinal
dfLocalTsCorr['global_ts'] = dfLocalTsCorr.local_ts * slopeFinal + interceptFinal
# NOTE: Disabled offset correction based on reset again as it seemed that duration between release of reset pin and start of local timestamp counter is application dependent, i.e. not constant
# # correct offset of global timestamp using the synchronized release from reset at start of the test on FlockLab 2
# # NOTE: this only works for the first syncEpoch (epoch begins with start of test on FlockLab 2)
# if firstSyncEpoch:
# df = dfLocalTsCorr[dfLocalTsCorr.tc == 0]
# firstSyncPoint = df.iloc[0] # first sync point without delay indicated
# startCorr = firstSyncPoint.global_ts - firstSyncPoint.local_ts*slopeFinal
# startActual = int(startCorr) # FlockLab 2 makes sure that test is started (i.e. reset is released) exactly when new second starts
# offsetReset = startCorr - startActual
# # DEBUG
# print('INFO: offset (based on initial reset): {}'.format(offsetReset))
#
# dfDataCorr['global_ts'] = dfDataCorr.global_ts - offsetReset + DT_FIXED_OFFSET_RESET
# dfLocalTsCorr['global_ts'] = dfLocalTsCorr.global_ts - offsetReset + DT_FIXED_OFFSET_RESET
return dfDataCorr, dfLocalTsCorr
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment