dwt_parse.py 17.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#! /usr/bin/env python3

"""
Copyright (c) 2020, ETH Zurich, Computer Engineering Group
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of the copyright holder nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

Author: Lukas Daschinger
"""

import sys
import time
import numpy as np
import pandas as pd
import collections

Roman Trüb's avatar
Roman Trüb committed
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
class SwoParser():
    def __init__(self):
        self._streamStarted = False
        self._currentPkt = []
        self._ignoreBytes = 0

    class SwoPkt():
        def __init__(self, header):
            self._header = header
            self._plBytes = []

        def addByte(self, byteVal):
            raise Exception('ERROR: This function is a prototype and should not directly be called!')

        def isComplete(self):
            raise Exception('ERROR: This function is a prototype and should not directly be called!')

        def __str__(self):
            raise Exception('ERROR: This function is a prototype and should not directly be called!')


    class LocalTimestampPkt(SwoPkt):
        def __init__(self, header):
            # TODO: determine from header whether timestamp pkt has payload or not (currently local timestamp packet format 2 (single-byte) is not supported)
            super().__init__(header)
            self._complete = False
            self._format2 = (self._header & 0b10001111 == 0) # format 2 (single-byte packet)
            self._tc = (header >> 4) & 0b11 if not self._format2 else None

        def addByte(self, byteVal):
            self._plBytes.append(byteVal)

        def isComplete(self):
            if self._format2:
                # format 2 (single-byte packet) case
                return True
            else:
                # format 1 (at least one payload byte)
                if not self._plBytes:
                    return False
                continuation_bit = self._plBytes[-1] & 0x80
                # continuation_bit==0 indicates that this is the last byte of the local timstamp packet
                return continuation_bit==0

        @property
        def ts(self):
            if not self.isComplete():
                raise Exception('ERROR: Cannot get timestamp from incomplete LocalTimestampPkt')

            if self._format2:
                ret = (self._header & 0b01110000) >> 4
            else:
                ret = 0
                for i, byte in enumerate(self._plBytes):
                    ret |= (byte & 0x7f) << i * 7  # remove the first bit and shift by 0,7,14,21 depending on value
            return ret

        @property
        def tc(self):
            return self._tc

        def __str__(self):
            ret = "LocalTimestampPkt {} {:#010b}{}:".format(self._header, self._header, "" if self.isComplete() else " (incomplete)")
            ret += "\n  bytes: {}".format(self._plBytes)
            ret += "\n  format: {}".format(2 if self._format2 else 1)
            if self.isComplete():
                ret += "\n  ts: {}".format(self.ts)
                ret += "\n  tc: {}".format(self.tc)
            return ret

    class DatatracePkt(SwoPkt):
        def __init__(self, header):
            super().__init__(header)
            self._payloadSize = map_size(header & 0b11)
            self._pktType = (header >> 6) & 0b11 # 1: PC value or address; 2: data value; otherweise: reserved
            self._comparator = (header >> 4) & 0b11 # comparator that generated the data
            self._addressPkt = None # True if data trace address pkt, False if data trace PC value pkt
            self._writeAccess = None # True if write access, False if read access
            if self._pktType == 1:  # PC value or address
                self._addressPkt = (header >> 3 & 0b1)
            elif self._pktType == 2: # data value
                self._writeAccess = (header >> 3 & 0b1)
            else:
                raise Exception('ERROR: Reserved data trace packet type encountered!')

        def addByte(self, byteVal):
            self._plBytes.append(byteVal)
            # TODO set isComplete var to true if contin==0

        def isComplete(self):
            return len(self._plBytes) == self._payloadSize

        @property
        def pktType(self):
            return self._pktType

        @property
        def comparator(self):
            return self._comparator

        @property
        def addressPkt(self):
            return self._addressPkt

        @property
        def writeAccess(self):
            return self._writeAccess

        @property
        def value(self):
            return (self._plBytes[3] << 24) + (self._plBytes[2] << 16) + (self._plBytes[1] << 8) + (self._plBytes[0] << 0)

        def __str__(self):
            ret = "DatatracePkt {} {:#010b}{}:".format(self._header, self._header, "" if self.isComplete() else " (incomplete)")
            ret += "\n  bytes: {}".format(self._plBytes)
            ret += "\n  pktType: {}".format(self.pktType)
            ret += "\n  comparator: {}".format(self.comparator)
            ret += "\n  payloadSize: {}".format(self._payloadSize)
            if self.pktType == 1:
                # PC value or address;
                ret += "\n  addressPkt: {}".format(self.addressPkt)
            elif self.pktType == 2:
                # data value; otherweise: reserved
                ret += "\n  writeAccess: {}".format(self.writeAccess)
            else:
                raise Exception("ERROR: DataTracePkt with reserved packetType!")
168

Roman Trüb's avatar
Roman Trüb committed
169
170
            if self.isComplete():
                ret += "\n  value: {}".format(self.value)
171

Roman Trüb's avatar
Roman Trüb committed
172
            return ret
173

Roman Trüb's avatar
Roman Trüb committed
174
175
176
    class OverflowPkt(SwoPkt):
        def __init__(self, header):
            super().__init__(header)
177

Roman Trüb's avatar
Roman Trüb committed
178
179
        def isComplete(self):
            # overflow packet consists of a single header byte
180
181
            return True

Roman Trüb's avatar
Roman Trüb committed
182
183
        def __str__(self):
            return "OverflowPkt"
184
185


Roman Trüb's avatar
Roman Trüb committed
186
187
188
189
190
191
192
    def addSwoByte(self, swoByte):
        """
        Args:
            swoByte: single SWO byte (header or payload) which shall be parsed.
            NOTE: SWO bytes need to be inserted in the correct sequence (as outputted by the SWO port)
        Returns:
            Parsed packet object if provided swoByte leads to the completion of a packet, None otherwise
193

Roman Trüb's avatar
Roman Trüb committed
194
        """
195

Roman Trüb's avatar
Roman Trüb committed
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
        # sync to packet in byte stream
        if not self._streamStarted:
            # read all zeros until get an 0x80, then we are in sync (Synchronization packet)
            # NOTE: dpp2lora bytestream does not contain required single-bit in Synchronization packet
            if swoByte == 0:
                return None
            elif swoByte == 0x08:
                return None
            else:
                self._streamStarted = True
                print('>>>>>>>> Stream started!')

        # ignore paylaod bytes of nrecognized packet
        if self._ignoreBytes:
            self._ignoreBytes -= 1
            return None

        # parse packets with content
        if len(self._currentPkt) == 0:
            # we do not currently have a begun packet -> start new one
            if swoByte & 0b11001111 == 0b11000000:
                # Local timestamp packet header
                self._currentPkt.append(type(self).LocalTimestampPkt(header=swoByte))
            elif swoByte & 0b10001111 == 0b0:
                # Local timestamp packet header (single-byte)
                self._currentPkt.append(type(self).LocalTimestampPkt(header=swoByte))
            elif swoByte == 0b01110000:
                self._currentPkt.append(type(self).OverflowPkt(header=swoByte))
            elif swoByte >> 2 & 0b1  == 0b1 and swoByte & 0b11 != 0b00:
                # Hardware source packet header
                discriminator_id = swoByte >> 3 & 0b11111
                plSize = map_size(swoByte & 0b11)
                if discriminator_id in [0, 1, 2]:
                    # 0 Event counter wrapping, 1 Exception tracing, 2 PC sampling
                    self._ignoreBytes = plSize
                elif discriminator_id >= 8 and discriminator_id <= 23:
                    # Data tracing
                    self._currentPkt.append(type(self).DatatracePkt(header=swoByte))
                else:
                    # Other undefined header
                    raise Exception("ERROR: Unrecognized discriminator ID in hardware source packet header: {}".format(swoByte)) # packets with undefined discriminator_id appear sporadically -> we cannot throw error here
            else:
                print("ERROR: Unrecognized DWT packet header: {} {:#010b}".format(swoByte, swoByte))
        else:
            # we currently have a begun packet -> add data
            self._currentPkt[0].addByte(swoByte)
242

Roman Trüb's avatar
Roman Trüb committed
243
244
245
246
247
        # check whether current packet is complete
        if self._currentPkt[0].isComplete():
            return self._currentPkt.pop()
        else:
            return None
248

249
250


Roman Trüb's avatar
Roman Trüb committed
251
def parse_dwt_output(input_file):
252
    """
Roman Trüb's avatar
Roman Trüb committed
253
    Executes the read and parse functions which will read from the given input_file and parse the content.
254
255
256
257

    Parameters:
        input_file (str): name of the file to parse
    Returns:
Roman Trüb's avatar
Roman Trüb committed
258
        df: dataframe containing the parsed data
259
260

    """
261
    read_queue = collections.deque()
262

263
264
    # read raw file into queue
    read_fun(read_queue, input_file)
265

Roman Trüb's avatar
Roman Trüb committed
266
    # parse data in queues and generate dataframe
267
    df = parse_fun(read_queue)
268

Roman Trüb's avatar
Roman Trüb committed
269
    return df
270
271


272
273
274
275
276
277
278
279
280
281
282
def map_size(ss):
    if ss == 1:
        return 1
    elif ss == 2:
        return 2
    elif ss == 3:
        return 4
    else:
        raise Exception('ERROR: Invalid ss size: ss should not be ==0 or >3')


Roman Trüb's avatar
Roman Trüb committed
283
284
285
286
287
288
289
290
def is_float(str):
    try:
        float(str)
        return True
    except ValueError:
        return False


291
def read_fun(read_queue, input_file):
292
    """
Roman Trüb's avatar
Roman Trüb committed
293
    Reads from the input file and then puts values into the queue.
294
295
296
    It also handles special cases by putting a varying number of global timestamps into the queue
    """

297
    data_is_next = True  # we expect that raw file starts with data (not with global timestamp)
298
    with open(input_file) as open_file_object:
299
300
        for i, line in enumerate(open_file_object):
            if i == 0:
Roman Trüb's avatar
Roman Trüb committed
301
302
                # ignore first line with varnames
                continue
303
304
305
306
307
308
309
310
311
312

            if data_is_next:
                # Line with data

                numbers = []
                for word in line.split():
                    if not word.isdigit():
                        raise Exception('ERROR: element of line is not digits as expected for a line with data')

                    numbers.append(int(word))
313
314
                read_queue.appendleft(('data', numbers))
                data_is_next = False
315
316
            else:
                # Line with global timestamp
Roman Trüb's avatar
Roman Trüb committed
317
318
319
320
321

                # check if line actually contains a float
                if not is_float(line):
                    raise Exception('ERROR: line is not float as expected for a line with global timestamp')

322
                read_queue.appendleft(('global_ts', float(line)))
323
                data_is_next = True
324

325

326
def parse_fun(read_queue):
327
328
329
    """
    Parses packets from the queue
    """
Roman Trüb's avatar
Roman Trüb committed
330
331
    columns = ['global_ts', 'comparator', 'data', 'PC', 'operation', 'local_ts']
    out_list = []
332
333
334
335

    completedDataPkts = []
    completedLocalTimestampPkts = []
    completedOverflowPkts = []
Roman Trüb's avatar
Roman Trüb committed
336
337

    swoParser = SwoParser()
338
339
340
341
342
343

    while read_queue:
        elem = read_queue.pop()
        elemType, elemData = elem
        if elemType == 'data':
            for swoByte in elemData:
Roman Trüb's avatar
Roman Trüb committed
344
345
346
347
348
349
350
351
352
                ret = swoParser.addSwoByte(swoByte)
                if ret:
                    print(ret)
                    if type(ret) == SwoParser.LocalTimestampPkt:
                        completedLocalTimestampPkts.append(ret)
                    elif type(ret) == SwoParser.DatatracePkt:
                        completedDataPkts.append(ret)
                    elif type(ret) == SwoParser.OverflowPkt:
                        completedOverflowPkts.append(ret)
353
                    else:
Roman Trüb's avatar
Roman Trüb committed
354
355
                        raise Exception['ERROR: Packet completed but type is unknown!']

356
        elif elemType == 'global_ts':
Roman Trüb's avatar
Roman Trüb committed
357
            if not swoParser._streamStarted:
358
359
360
                continue

            if not completedLocalTimestampPkts:
Roman Trüb's avatar
Roman Trüb committed
361
                # cannot create rows from current data since there is no complete local timestamp pkt
362
                print('WARNING: cannot create rows from current data since there is no complete local timestamp pkt')
Roman Trüb's avatar
Roman Trüb committed
363
                continue
364

Roman Trüb's avatar
Roman Trüb committed
365
            # use collected packets to create rows with data
366
367
            if completedDataPkts:
                for comparatorId in [0, 1, 2, 3]:
Roman Trüb's avatar
Roman Trüb committed
368
                    new_row = collections.OrderedDict(zip(columns, [None]*len(columns)))
369
370
371
372
                    for dataPkt in completedDataPkts:
                        if dataPkt.comparator == comparatorId:
                            if dataPkt.pktType == 1: # data trace pc value pkt
                                if not dataPkt.addressPkt: # we want PC value
Roman Trüb's avatar
Roman Trüb committed
373
                                    new_row['PC'] = hex(dataPkt.value)
374
                            elif dataPkt.pktType == 2: # data trace data value pkt
Roman Trüb's avatar
Roman Trüb committed
375
376
377
378
379
380
381
382
                                new_row['operation'] = 'w' if dataPkt.writeAccess else 'r'
                                new_row['data'] = dataPkt.value
                    if not new_row['data'] is None:
                        new_row['comparator'] = comparatorId
                        new_row['local_ts'] = completedLocalTimestampPkts[-1].ts
                        new_row['global_ts'] = elemData
                        out_list += [new_row]
            elif completedLocalTimestampPkts:
383
                # no completedDataPkts -> we have overflow timestamp -> add it to use it for regression
Roman Trüb's avatar
Roman Trüb committed
384
385
386
387
                new_row = collections.OrderedDict(zip(columns, [None]*len(columns)))
                new_row['local_ts'] = completedLocalTimestampPkts[-1].ts
                new_row['global_ts'] = elemData
                out_list += [new_row]
388
389
            if completedOverflowPkts:
                for overflowPkt in completedOverflowPkts:
Roman Trüb's avatar
Roman Trüb committed
390
391
392
393
394
395
396
397
                    new_row = collections.OrderedDict(zip(columns, [None]*len(columns)))
                    new_row['global_ts'] = elemData
                    out_list += [new_row]

            # clear all collected packets
            completedDataPkts.clear()
            completedLocalTimestampPkts.clear()
            completedOverflowPkts.clear()
398
        else:
Roman Trüb's avatar
Roman Trüb committed
399
            # Unrecognized elemType
400
401
            raise Exception('ERROR: Unknown element type!')

Roman Trüb's avatar
Roman Trüb committed
402
    return pd.DataFrame(out_list)
Roman Trüb's avatar
Roman Trüb committed
403

404
405


Roman Trüb's avatar
Roman Trüb committed
406
def correct_ts_with_regression(df_in):
407
408
409
    """
    Calculates a regression based on the values in log_table.csv
    Then projects the global timestamps onto the regression and writes the corrected values in log_table_corrected.csv
Roman Trüb's avatar
Roman Trüb committed
410
411
412
413
414

    Params:
        df_in: dataframe containing parsed data
    Returns:
        df_out: dataframe containing time corrected data
415
    """
Roman Trüb's avatar
Roman Trüb committed
416
    df_out = df_in.copy()
417
418

    # extract the global and local timestamps and put into a numpy array
Roman Trüb's avatar
Roman Trüb committed
419
420
    x = df_out['local_ts'].to_numpy(dtype=float)
    y = df_out['global_ts'].to_numpy(dtype=float)
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435

    # add up the local timestamps and calculate the global timestamp relative to the first global timestamp
    sum_local_ts = 0

    for local_ts in np.nditer(x, op_flags=['readwrite']):
        sum_local_ts = local_ts[...] = sum_local_ts + local_ts

    # use the data from the arrays to calculate the regression
    b = estimate_coef(x, y)

    # correct the global timestamp in the data frame
    for local_ts, global_ts in np.nditer([x, y], op_flags=['readwrite']):
        global_ts[...] = b[0] + b[1] * local_ts

    # write the array back into the pandas df to replace the global timestamps
Roman Trüb's avatar
Roman Trüb committed
436
    df_out['global_ts'] = y
437

Roman Trüb's avatar
Roman Trüb committed
438
    # The file log_table_corrected.csv now contains the corrected global timestamps together with the DWT packets
439

Roman Trüb's avatar
Roman Trüb committed
440
441
442
    return df_out


443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478

def estimate_coef(x, y):
    """
    Calculates coefficient b_0 and b_1 of the regression.

    Parameters:
      x (numpy array): the local timestamps received from target
      y (numpy array): the global timestamps taken on the observer

    Returns:
      int: b_0 offset of linear regression
      int: b_1 slope of linear regression
    """
    # number of observations/points
    n = np.size(x)

    # mean of x and y vector
    m_x, m_y = np.mean(x), np.mean(y)

    # calculating cross-deviation and deviation about x
    SS_xy = np.sum(y * x) - n * m_y * m_x
    SS_xx = np.sum(x * x) - n * m_x * m_x

    # calculating regression coefficients
    b_1 = SS_xy / SS_xx
    b_0 = m_y - b_1 * m_x

    return b_0, b_1



if __name__ == '__main__':
    if len(sys.argv) > 1:
        filename = sys.argv[1]
        parse_dwt_output(filename, filename + ".csv")
        correct_ts_with_regression(filename + ".csv", filename + "_corrected.csv")