Commit 8f103a63 authored by Roman Trüb's avatar Roman Trüb
Browse files

cleanup of worker_datatrace (conversion of float to int)

parent a1612eef
......@@ -445,25 +445,31 @@ def worker_datatrace(queueitem=None, nodeid=None, resultfile_path=None, logqueue
logqueue.put_nowait((loggername, logging.WARNING, "Empty data trace results file."))
else:
with open(resultfile_path, "a") as outfile:
infile = open(tmpfile2, "r")
for line in infile:
# input format: global_ts, comparator, data, PC, operation, local_ts
(timestamp, var, val, pc, access, localts) = line.strip().split(',')
if access == 'operation' or access == '' or var == '':
continue
if flocklab.parse_int(var) < len(varnames):
var = varnames[flocklab.parse_int(var)]
# output format: timestamp,observer_id,node_id,variable,value,access,pc
outfile.write("%s,%d,%s,%s,%s,%s,%s\n" % (timestamp, obsid, nodeid, var, val, access, pc))
infile.close()
df = pd.read_csv(tmpfile2)
# debug
df.to_csv('/home/flocklab/tmp/tmp_df.txt')
# nan values cannot be converted to int -> drop corresponding lines
df.dropna(inplace=True)
# since there were nan values, comparator column was stored as nan but we need int; round is necessary otherwise 0.999999 is converted to 0 which is wrong
df.comparator = df.comparator.round().astype(int)
df.data = df.data.round().astype(int)
df['obsid'] = obsid
df['nodeid'] = nodeid
df['varname'] = df.comparator.apply(lambda x: (varnames[x] if x < len(varnames) else str(x)))
df.to_csv(
outfile,
columns=['global_ts', 'obsid', 'nodeid', 'varname', 'data', 'operation', 'PC'],
index=False,
header=False
)
except:
msg = "Error in datatrace worker process: %s: %s\n%s" % (str(sys.exc_info()[0]), str(sys.exc_info()[1]), traceback.format_exc())
_errors.append((msg, obsid))
logqueue.put_nowait((loggername, logging.ERROR, msg))
finally:
# debug
#shutil.copyfile(input_filename, "%s_raw" % resultfile_path)
#shutil.copyfile(tmpfile1, "%s_uncorrected.csv" % resultfile_path)
# shutil.copyfile(input_filename, "%s_raw" % resultfile_path)
# shutil.copyfile(tmpfile1, "%s_uncorrected.csv" % resultfile_path)
# delete files
os.remove(input_filename)
os.remove(tmpfile1)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment