Commit 3d7fdcbb authored by Andreas Biri's avatar Andreas Biri 👻
Browse files

added informational message if no aggregation data is available

parent 17377959
......@@ -57,7 +57,7 @@ class STECAnalyser:
self.load_config(DEFAULT_CONFIG_FILE)
self._DataMgr = DataManager(deployment=DEPLOYMENT, config_file=DEFAULT_CONFIG_FILE, project_name='stec', start_time=self.DATA_START_TIME, end_time=self.DATA_END_TIME)
self._DataProc = DataProcessor(config_file=DEFAULT_CONFIG_FILE)
self._DataProc = DataProcessor(config_file=DEFAULT_CONFIG_FILE, project_name='stec')
def load_config(self, config_file=None):
......
......@@ -76,7 +76,7 @@ class STECFlocklabManager:
self.load_config(DEFAULT_CONFIG_FILE)
self._DataMgr = DataManager(deployment=DEPLOYMENT, config_file=DEFAULT_CONFIG_FILE, project_name='stec', start_time=self.DATA_START_TIME, end_time=self.DATA_END_TIME)
self._DataProc = DataProcessor(config_file=DEFAULT_CONFIG_FILE)
self._DataProc = DataProcessor(config_file=DEFAULT_CONFIG_FILE, project_name='stec')
self._codet_sample = None
def load_config(self, config_file=None):
......
......@@ -63,7 +63,7 @@ class STECUtils:
self.load_config(DEFAULT_CONFIG_FILE)
self._DataMgr = DataManager(deployment=DEPLOYMENT, config_file=DEFAULT_CONFIG_FILE, project_name='stec', start_time=self.DATA_START_TIME, end_time=self.DATA_END_TIME)
self._DataProc = DataProcessor(config_file=DEFAULT_CONFIG_FILE)
self._DataProc = DataProcessor(config_file=DEFAULT_CONFIG_FILE, project_name='stec')
def load_config(self, config_file=None):
......
%% Cell type:markdown id: tags:
**Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)**
# Co-detection visualisation
This script visualizes the co-detections in the targeted time frame and displays key metrics in the plots below.
%% Cell type:code id: tags:
``` python
import datetime as dt
import pandas as pd
import numpy as np
import sys
import os
import plotly.express as px
from statistics import mean, median
from IPython.display import display
sys.path.append("../") # FIXME: Work around if not built as a package
from data_management.data_manager import DataManager
from data_management.data_processing import DataProcessor
# Settings
DEPLOYMENT = 'dirruhorn'
VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_DATA = '_dpp_geophone_adcData__conv'
VS_PRECIPITATION = '_vaisalawxt520prec'
DATA_START_TIME = dt.datetime.strptime("01/06/2018", "%d/%m/%Y")
DATA_END_TIME = dt.datetime.strptime("01/01/2020", "%d/%m/%Y")
DATES_EXCLUDED = [['31/05/2017'], ['08/06/2017', '09/06/2017'], ['10/07/2017'], ['25/07/2017'], ['31/08/2017'], ['08/09/2017'], ['30/09/2017'], ['17/04/2018'], ['24/04/2018'], ['10/07/2018','11/07/2018'], ['27/09/2018'], ['27/03/2019'], ['19/06/2019'], ['03/04/2020'], ['03/06/2020'], ['11/08/2020'], ['15/08/2020']]
S_TO_US = 1000 * 1000
# Paper constants - use font size 14 if using 1/2 column
column_width_pt = 241.14749
page_width_pt = 506.295
pt_to_px = 1.3281472327365
# Print settings
print('Going to fetch co-detections at deployment site {0:s} from {1:s} - {2:s}'.format(DEPLOYMENT.capitalize(), DATA_START_TIME.strftime("%d/%m/%Y"), DATA_END_TIME.strftime("%d/%m/%Y")))
```
%% Cell type:code id: tags:
``` python
# Create necessary objects
DataMgr = DataManager(deployment=DEPLOYMENT, config_file='../stec.conf', project_name='stec', start_time=DATA_START_TIME, end_time=DATA_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf')
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')
# Fetch data
# Create URL including conditions
fields = 'device_id,start_time,end_time,generation_time,trg_duration'
url = DataMgr.assemble_gsn_url(VS_ACOUSTIC_METADATA, fields)
# Fetch data
try:
df = DataMgr.fetch_csv_data(url)
# Filter days with precipitation
prec_dates = DataMgr.fetch_rain_dates(include_gruengarten=True, include_breithorn=False, include_grabengufer=False)
DataProc.mark_rain_dates(df, rain_dates=prec_dates)
# Filter rain days if desired
include_rain = False
if not include_rain and len(prec_dates):
DATES_EXCLUDED = DATES_EXCLUDED + prec_dates
print("Added %d days to list of excepted dates due to precipitation" % (len(prec_dates)))
except Exception as ex_acoustic:
print("Could not fetch CSV data from path:\n{:s}\n\n{:s}".format(url, ex_acoustic))
sys.exit(1)
```
%% Cell type:code id: tags:
``` python
# Fetch meteo data
weather_station_gruengarten_pos = 13 # Dirruhorn Gruengarten - still existing
weather_station_grabengufer_pos = 42 # Grabengufer - still existing
weather_station_breithorn_pos = 68 # Breithorn - still existing
weather_station_glacier_pos = 69 # Dirruhorn Blockgletscher - existed until June 2020
try:
# Must use different queries, as OR conditions not work correctly with GSN
meteo_glacier = DataMgr.fetch_precipitation_data(weather_station_glacier_pos)
# Also include data from Dirruhorn Gruengarten
meteo = meteo_glacier.append(DataMgr.fetch_precipitation_data(weather_station_gruengarten_pos), ignore_index=False, verify_integrity=False)
except Exception as ex_meteo:
print("Could not fetch precipitation data for station:\n{:s}\n\n{:s}".format(weather_station_glacier_pos, ex_meteo))
sys.exit(1)
# Drop rows with null due to faulty data from Vaisala sensor
meteo.dropna(inplace=True)
# Drop days where both rain and hail is non-existent
no_precipitation = meteo[(meteo['rain_intensity'] == 0) & (meteo['hail_intensity'] == 0)].index.tolist()
meteo.drop(no_precipitation, inplace=True)
```
%% Cell type:code id: tags:
``` python
# Print event statistics
events = DataProc.compute_event_stats(df, dates_excluded=DATES_EXCLUDED, auto_extend_evt=False)
DataProc.print_event_stats(events, html_output=True)
# Data export
# PDF of start of last event in previous co-detection
# DataProc.compute_event_pdf(events[events['rain'] == 0], 'delta_start', nr_bins=100, print_output=False)
```
%% Cell type:code id: tags:
``` python
# Print co-detection statistics
codets = DataProc.find_codetections(df, dates_excluded=DATES_EXCLUDED, auto_extend_evt=False)
DataProc.print_codet_stats(codets, html_output=True)
```
%% Cell type:markdown id: tags:
## Plotting
After having pre-processed the data, we now plot the data for visual inspection.
%% Cell type:code id: tags:
``` python
# Prepare data for plotting
# Convert to seconds for easier analysis
events.loc[:, 'offset_min':'evt_duration'] = events.loc[:, 'offset_min':'evt_duration'].divide(S_TO_US)
# Convert timestamp to datetime
events['timestamp_dt'] = pd.to_datetime(events['timestamp'], unit='us')
# Convert to string for better plotting
events['rain_str'] = events['rain'].astype(str) # Required for correct colouring of scatter plots
meteo['position_str'] = meteo['position'].astype(str)
# Store meteo index separately for correct axis labelling
meteo['timestamp_dt'] = meteo.index
# Compute logarithmic values for event duration and event offsets
events['evt_duration_log'] = np.log10(events['evt_duration'])
events['delta_start_log'] = np.log10(events['delta_start'])
# Color mapping for consistent display of rain data
plot_for_paper=True
if plot_for_paper:
color_rain_map = {'True': px.colors.qualitative.G10[1], 'False': px.colors.qualitative.G10[1]}
events['rain_str'] = 'False' # Make sure that all points are in the same class (due to timezone issues, this is not the case for a small minority)
else:
color_rain_map = {'True': 'red', 'False': 'green'}
def format_figure(figure, two_column_figure=False, half_column_figure=False, color_scale=px.colors.qualitative.G10, height_pt=None, show_legend=False, figure_name=None, to_zero=True):
# Figure width
yaxis_offset = 20
xaxis_offset = 20
fig_width = column_width_pt * pt_to_px
fig_height = height_pt * pt_to_px if height_pt is not None else None
if two_column_figure:
fig_width = page_width_pt * pt_to_px
elif half_column_figure:
yaxis_offset = 50
xaxis_offset = 70
# Color scale
if color_scale is not None:
figure.update_layout(colorscale_sequential=color_scale)
# Height
if fig_height is not None:
figure.update_layout(height=fig_height)
# Legend
if not show_legend:
# Put the legend inside the figure so it does not change the margins
figure.update_layout(showlegend=False,
legend=dict(yanchor='top', y=0.99, xanchor='right', x=1.0))
else:
figure.update_layout(showlegend=True,
legend_bordercolor='Grey',
legend_borderwidth=1 if half_column_figure else 0.5,
legend_title_text='',
legend_font_color='Grey')
# Remove margins
figure.update_layout(margin=dict(l=yaxis_offset, r=0, t=0, b=xaxis_offset, autoexpand=True))
# Update the default parameters
figure.update_layout(plot_bgcolor='white',
font_family="Linux Libertine O",
font_size=14 if half_column_figure else 7,
title_font_color='white',
width=fig_width)
# Axes
figure.update_yaxes(showline=True,
color='black',
linecolor='black',
linewidth=1 if half_column_figure else 0.5,
tickwidth=1 if half_column_figure else 0.5,
title_standoff=5,
automargin=False,
ticks='outside',
rangemode='tozero' if to_zero else 'normal')
figure.update_xaxes(showline=True,
color='black',
linecolor='black',
linewidth=1 if half_column_figure else 0.5,
tickwidth=1 if half_column_figure else 0.5,
title_standoff=10,
automargin=False,
ticks='outside')
# Show figure
figure.show()
# Store figure
if figure_name is not None:
directory_name = 'figures'
# Create a directory for the figures
if not os.path.exists(directory_name):
os.mkdir(directory_name)
# Create file
figure.write_image(directory_name + '/' + figure_name)
```
%% Cell type:code id: tags:
``` python
# Number of nodes per co-detection
tick_array = [2, 4, 6, 8, 11, 13]
fig = px.scatter(events, x='timestamp_dt', y='nr_nodes', labels={'timestamp_dt': 'Time', 'nr_nodes': 'Co-detection degree [#]', 'rain_str': 'Day included rain'}, color='rain_str', color_discrete_map=color_rain_map, render_mode='svg')
fig.update_yaxes(tickmode='array',
tickvals=tick_array,
ticktext=tick_array)
format_figure(fig, figure_name='fig_nodes_per_codet_timeline.pdf', height_pt=column_width_pt, half_column_figure=True)
# Duration
fig = px.scatter(events, x='timestamp_dt', y='evt_duration', labels={'timestamp_dt': 'Time', 'evt_duration': 'Event duration [s]', 'rain_str': 'Day included rain'}, color='rain_str', color_discrete_map=color_rain_map)
format_figure(fig, figure_name='fig_duration_timeline.pdf', height_pt=column_width_pt, half_column_figure=True)
# CDF of event duration
tick_array = [0.1, 0.25, 0.5, 1, 2, 5, 10, 15, 30, 60]
fig = px.histogram(events, x='evt_duration_log', histnorm='percent', cumulative=True, labels={'evt_duration': 'Duration [s]', 'evt_duration_log': 'Duration [s]', 'rain_str': 'Day included rain'}, color='rain_str', color_discrete_map=color_rain_map, nbins=10000)
fig.update_xaxes(tickmode='array',
tickvals=np.log10(tick_array),
ticktext=tick_array,
tickangle=-60)
fig.update_layout(yaxis_title='CDF')
format_figure(fig, figure_name='fig_cdf_duration.pdf', height_pt=column_width_pt, half_column_figure=True)
# Delta to start of last event in previous co-detection
events['freq_evts'] = events['delta_start'].rdiv(1)
fig = px.scatter(events, x='timestamp_dt', y='freq_evts', labels={'timestamp_dt': 'Time', 'freq_evts': 'Event frequency [Hz]', 'rain_str': 'Day included rain'}, color='rain_str', color_discrete_map=color_rain_map)
format_figure(fig, figure_name='fig_delta_timeline.pdf', height_pt=column_width_pt, half_column_figure=True)
# CDF of start of last event in previous co-detection
tick_array = [0.5, 1, 2, 5, 15, 30, 60, 300, 900, 3600, 7200]
fig = px.histogram(events, x='delta_start_log', histnorm='percent', cumulative=True, labels={'delta_start': 'Event interval [s]', 'delta_start_log': 'Event interval [s]', 'rain_str': 'Day included rain'}, color='rain_str', color_discrete_map=color_rain_map, nbins=10000)
fig.update_xaxes(tickmode='array',
tickvals=np.log10(tick_array),
ticktext=tick_array,
tickangle=-60)
fig.update_layout(yaxis_title='CDF')
format_figure(fig, figure_name='fig_cdf_delta.pdf', height_pt=column_width_pt, half_column_figure=True)
```
%% Cell type:code id: tags:
``` python
# Rain intensity
fig = px.scatter(meteo, x='timestamp_dt', y='rain_intensity', labels={'timestamp_dt': 'Time', 'rain_intensity': 'Rain intensity [mm/h]'}, color='position_str', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
# Hail intensity
fig = px.scatter(meteo, x='timestamp_dt', y='hail_intensity', labels={'timestamp_dt': 'Time', 'hail_intensity': 'Hail intensity [mm/h]'}, color='position_str', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
```
......
%% Cell type:markdown id: tags:
**Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)**
# STeC Deployment Evaluation
This script visualizes the deployment data of the STeC deployments for both on DH and ETZ (test deployment).
%% Cell type:code id: tags:
``` python
import sys
import os
import datetime as dt
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
sys.path.append('../') # FIXME: Work around if not built as a package
sys.path.append('../../system_model')
from data_management.data_manager import DataManager
from data_management.data_processing import DataProcessor
from des.des_eval import load_metrics
# Settings
TEST_SERVER = 'http://tpbl.permasense.ethz.ch/'
WEATHER_STATION_GRUENGARTEN_POS = 13 # Dirruhorn Gruengarten - still existing
DEPLOYMENT_OUTDOOR = 'dirruhorn'
DEPLOYMENT_TEST = 'etz'
VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_AGGR = '_dpp_geophone_acq_min_aggr__mapped'
VS_ACOUSTIC_DATA = '_dpp_geophone_adcData__conv'
VS_HEALTH_MIN_DATA = '_dpp_health_min__conv'
VS_ACOUSTIC_AGGR_LOG = '_dpp_geophone_acq_logs__conv'
DIRRU_START_TIME = dt.datetime.strptime("20/10/2020", "%d/%m/%Y")
DIRRU_END_TIME = dt.datetime.strptime("20/05/2021", "%d/%m/%Y")
ETZ_START_TIME = dt.datetime.strptime("03/02/2021", "%d/%m/%Y")
ETZ_END_TIME = dt.datetime.strptime("17/05/2021", "%d/%m/%Y")
LOGS_START_TIME = dt.datetime.strptime("19/10/2020", "%d/%m/%Y")
LOGS_END_TIME = dt.datetime.strptime("17/05/2021", "%d/%m/%Y")
S_TO_US = 1000 * 1000
FALLBACK_BS_IDS_OUTDOOR = [106, 110, 111, 112]
FALLBACK_BS_IDS_TEST = [103, 107, 109]
INDOORS_NODES = [21012, 21018, 21019]
# Paper constants - use font size 14 if using 1/2 column
column_width_pt = 241.14749
page_width_pt = 506.295
pt_to_px = 1.3281472327365
pt_to_inch = 0.0138889
# Print settings
print('Going to fetch STeC Health data at deployment site {0:9s} from {1:s} - {2:s}'.format(DEPLOYMENT_OUTDOOR.capitalize(), DIRRU_START_TIME.strftime('%d/%m/%Y'), DIRRU_END_TIME.strftime('%d/%m/%Y')))
print('Going to fetch STeC Health data at deployment site {0:9s} from {1:s} - {2:s}'.format(DEPLOYMENT_TEST.capitalize(), ETZ_START_TIME.strftime('%d/%m/%Y'), ETZ_END_TIME.strftime('%d/%m/%Y')))
# Adjust display width for variables
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
```
%% Cell type:code id: tags:
``` python
# Create necessary objects
DataMgr_outdoor = DataManager(deployment=DEPLOYMENT_OUTDOOR, config_file='../stec.conf', project_name='stec', start_time=DIRRU_START_TIME, end_time=DIRRU_END_TIME)
DataMgr_test = DataManager(deployment=DEPLOYMENT_TEST, config_file='../stec.conf', project_name='stec', start_time=ETZ_START_TIME, end_time=ETZ_END_TIME)
DataMgr_logs = DataManager(deployment=DEPLOYMENT_TEST, config_file='../stec.conf', project_name='stec', start_time=LOGS_START_TIME, end_time=LOGS_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf')
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')
# Fetch data
# Create URL including conditions
url_health_outdoor = DataMgr_outdoor.assemble_gsn_url(VS_HEALTH_MIN_DATA)
url_aggr_outdoor = DataMgr_outdoor.assemble_gsn_url(VS_ACOUSTIC_AGGR)
cond_ids_test = [{'join': 'and', 'field': 'device_id', 'min': 21005, 'max': 21021}]
url_health_test = DataMgr_test.assemble_gsn_url(VS_HEALTH_MIN_DATA, server_url=TEST_SERVER, conditions=cond_ids_test)
url_aggr_test = DataMgr_test.assemble_gsn_url(VS_ACOUSTIC_AGGR, server_url=TEST_SERVER, conditions=cond_ids_test)
# Fetch data
df_health_outdoor = DataMgr_outdoor.fetch_csv_data(url_health_outdoor, description='Health')
df_aggr_outdoor = DataMgr_outdoor.fetch_csv_data(url_aggr_outdoor, description='Aggregation')
df_health_test = DataMgr_test.fetch_csv_data( url_health_test, description='Health')
df_aggr_test = DataMgr_test.fetch_csv_data( url_aggr_test, description='Aggregation')
df_meteo_outdoor = DataMgr_outdoor.fetch_precipitation_data(WEATHER_STATION_GRUENGARTEN_POS)
# Get logged data and generate dataframe of co-detections
url_logs_test = DataMgr_logs.assemble_gsn_url(VS_ACOUSTIC_AGGR_LOG)
df_logs_test = DataMgr_logs.fetch_csv_data(url_logs_test, description='Logs')
logs_codet_trace = DataProc.generate_codetection_trace(df_logs_test)
df_aggr_logs = DataProc.convert_codets_from_trace(logs_codet_trace)
# Get metrics from simulation based on logs
metrics_sim_test = load_metrics(load_ebc=True, postfix='_meta_sd_15000_ad_etz_all')
df_aggr_sim = DataProc.convert_codets_from_metrics(metrics_sim_test)
```
%% Cell type:code id: tags:
``` python
# Prepare data for plotting
# Formatting timestamps, add column indicating whether the packet was received over SF7 or SF10
if df_health_outdoor is not None:
df_health_outdoor['generation_time'] = pd.to_datetime(df_health_outdoor['generation_time_microsec'], unit='us')
df_health_outdoor['device_id_str'] = df_health_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_outdoor['fallback'] = df_health_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_health_outdoor['fallback_str'] = df_health_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_outdoor is not None:
df_aggr_outdoor['generation_time'] = pd.to_datetime(df_aggr_outdoor['generation_time_microsec'], unit='us')
df_aggr_outdoor['device_id_str'] = df_aggr_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_outdoor['fallback'] = df_aggr_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_aggr_outdoor['fallback_str'] = df_aggr_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_meteo_outdoor is not None:
df_meteo_outdoor['timestamp_dt'] = df_meteo_outdoor.index # Store meteo index separately for correct axis labelling
if df_health_test is not None:
df_health_test['generation_time'] = pd.to_datetime(df_health_test['generation_time_microsec'], unit='us')
df_health_test['device_id_str'] = df_health_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_test['fallback'] = df_health_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_health_test['fallback_str'] = df_health_test['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_test is not None:
df_aggr_test['generation_time'] = pd.to_datetime(df_aggr_test['generation_time_microsec'], unit='us')
df_aggr_test['device_id_str'] = df_aggr_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_test['fallback'] = df_aggr_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_aggr_test['fallback_str'] = df_aggr_test['fallback'].astype(str) # Required for correct colouring of scatter plots
# Color mapping for consistent display of fallback data
color_fallback_map = {'True': 'red', 'False': 'green'}
```
%% Cell type:code id: tags:
``` python
# Post processing
if df_health_outdoor is not None:
# Adjust sequence number after reset
reset_date_21025_1 = dt.datetime.strptime("09/03/2021 01:00:10", "%d/%m/%Y %H:%M:%S")
reset_date_21025_2 = dt.datetime.strptime("12/03/2021 04:00:10", "%d/%m/%Y %H:%M:%S")
reset_date_21025_3 = dt.datetime.strptime("14/03/2021 04:00:10", "%d/%m/%Y %H:%M:%S")
reset_date_21034 = dt.datetime.strptime("25/04/2021 07:02:00", "%d/%m/%Y %H:%M:%S")
# 21025
id_mask = df_health_outdoor['device_id'] == 21025
time_mask_1 = df_health_outdoor['generation_time'] < reset_date_21025_1
time_mask_2 = df_health_outdoor['generation_time'] < reset_date_21025_2
time_mask_3 = df_health_outdoor['generation_time'] < reset_date_21025_3
last_seqnr_1 = df_health_outdoor.loc[id_mask & time_mask_1, 'seqnr'].iat[-1]
last_seqnr_2 = df_health_outdoor.loc[id_mask & time_mask_2, 'seqnr'].iat[-1]
last_seqnr_3 = df_health_outdoor.loc[id_mask & time_mask_3, 'seqnr'].iat[-1]
df_health_outdoor.loc[id_mask & ~time_mask_1, 'seqnr'] = df_health_outdoor.loc[id_mask & ~time_mask_1, 'seqnr'].add(last_seqnr_1)
df_health_outdoor.loc[id_mask & ~time_mask_2, 'seqnr'] = df_health_outdoor.loc[id_mask & ~time_mask_2, 'seqnr'].add(last_seqnr_2)
df_health_outdoor.loc[id_mask & ~time_mask_3, 'seqnr'] = df_health_outdoor.loc[id_mask & ~time_mask_3, 'seqnr'].add(last_seqnr_3)
# 21034
id_mask = df_health_outdoor['device_id'] == 21034
time_mask = df_health_outdoor['generation_time'] < reset_date_21034
last_seqnr = df_health_outdoor.loc[id_mask & time_mask, 'seqnr'].iat[-1]
df_health_outdoor.loc[id_mask & ~time_mask, 'seqnr'] = df_health_outdoor.loc[id_mask & ~time_mask, 'seqnr'].add(last_seqnr)
if df_meteo_outdoor is not None:
# Drop rows with null due to faulty data from Vaisala sensor
df_meteo_outdoor.dropna(inplace=True)
# Drop days where both rain and hail is non-existent
no_precipitation = df_meteo_outdoor[(df_meteo_outdoor['rain_intensity'] == 0) & (df_meteo_outdoor['hail_intensity'] == 0)].index.tolist()
df_meteo_outdoor.drop(no_precipitation, inplace=True)
```
%% Cell type:markdown id: tags:
## Printing
We print some key metrics of our deployment.
%% Cell type:code id: tags:
``` python
if df_aggr_outdoor is not None:
# Remove duplicates from sniffers
df_printing = df_aggr_outdoor.copy()
df_printing.drop_duplicates(inplace=True, subset=['device_id', 'seqnr'])
print('Number of events: %i; Number of co-detections: %i; Number of co-detections with 3+ nodes: %i' % (df_printing['block_cnt'].sum(), df_printing.shape[0], df_printing.loc[df_printing['block_cnt'] > 2].shape[0]))
df_printing = DataProc.extract_nr_codets(df_aggr_outdoor)
print('Number of co-detections: %i; Number of co-detections with 3+ nodes: %i' % (df_printing['nr_codets_hourly'].sum(), df_printing.loc[df_printing['codet_nodes'] == '3+', 'nr_codets_hourly'].sum()))
print('Number of co-detections: %i; Number of co-detections with 3+ nodes: %i' % (df_printing['nr_codets_daily'].sum(), df_printing.loc[df_printing['codet_nodes'] == '3+', 'nr_codets_daily'].sum()))
```
%% Cell type:markdown id: tags:
## Plotting
After having pre-processed the data, we now plot the data for visual inspection.
%% Cell type:code id: tags:
``` python
LINE_STYLES_EXP = [None, None, 'dash', 'dash']
LINE_STYLES_EXP_ALL = [None, 'dash', 'dot']
def format_figure(figure, two_column_figure=False, half_column_figure=False, color_scale=px.colors.qualitative.G10, height_pt=None, show_legend=False, legend_names=None, figure_name=None, to_zero=True, is_line_plot=True, secondary_title_standoff=0):
# Figure width
yaxis_offset = 30
xaxis_offset = 30
fig_width = column_width_pt * pt_to_px
fig_height = height_pt * pt_to_px if height_pt is not None else None
if two_column_figure:
fig_width = page_width_pt * pt_to_px
elif half_column_figure:
yaxis_offset = 60
xaxis_offset = 50
# Color scale
if color_scale is not None:
figure.update_layout(colorscale_sequential=color_scale)
# Height
if fig_height is not None:
figure.update_layout(height=fig_height)
# Legend
if not show_legend:
# Put the legend inside the figure so it does not change the margins
figure.update_layout(showlegend=False,
legend=dict(yanchor='top', y=0.99, xanchor='right', x=1.0))
else:
figure.update_layout(showlegend=True,
legend_bordercolor='Grey',
legend_borderwidth=1 if half_column_figure else 0.5,
legend_title_text='',
legend_font_color='Grey')
# Re-name legend variables if desired
if legend_names is not None:
for i, dat in enumerate(figure.data):
for elem in dat:
if elem == 'name':
fig.data[i].name = legend_names[i]
# Remove margins
figure.update_layout(margin=dict(l=yaxis_offset, r=secondary_title_standoff, t=0, b=xaxis_offset, autoexpand=True))
# Update the default parameters
figure.update_layout(plot_bgcolor='white',
font_family='Linux Libertine O',
font_size=14 if half_column_figure else 7,
title_font_color='white',
width=fig_width)
# Line
if is_line_plot:
figure.update_traces(line=dict(width=0.5))
# Axes
figure.update_yaxes(showline=True,
color='black',
linecolor='black',
linewidth=1 if half_column_figure else 0.5,
tickwidth=1 if half_column_figure else 0.5,
title_standoff=5,
automargin=False,
ticks='outside',
rangemode='tozero' if to_zero else 'normal')
figure.update_xaxes(showline=True,
color='black',
linecolor='black',
linewidth=1 if half_column_figure else 0.5,
tickwidth=1 if half_column_figure else 0.5,
title_standoff=10,
automargin=False,
ticks='outside')
# Show figure
figure.show()
# Store figure
if figure_name is not None:
directory_name = 'figures'
# Create a directory for the figures
if not os.path.exists(directory_name):
os.mkdir(directory_name)
# Create file
figure.write_image(directory_name + '/' + figure_name)
```
%% Cell type:code id: tags:
``` python
# Plotting Outdoor
if df_aggr_outdoor is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
# Number of Co-Detections
df_codets = DataProc.extract_nr_codets(df_aggr_outdoor)
fig = px.line(df_codets, x='generation_time', y='nr_codets_daily', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'nr_codets_hourly': 'Number of co-detections per hour [#]', 'nr_codets_daily': 'Number of co-detections per day [#]', 'codet_nodes': 'Number of co-detecting nodes'}, color='codet_nodes', color_discrete_sequence=px.colors.qualitative.G10, render_mode='svg')
fig.update_layout(legend_yanchor='top',
legend_y=1.0,
legend_xanchor='right',
legend_x=1)
format_figure(fig, figure_name='fig_codet_outdoor.pdf', height_pt=column_width_pt/2, show_legend=False)
if df_meteo_outdoor is not None:
# Combined plot: Number of Co-Detections vs. Rain Intensity
fig = make_subplots(specs=[[{'secondary_y': True}]])
fig.add_trace(go.Bar(x=df_meteo_outdoor['timestamp_dt'], y=df_meteo_outdoor['rain_intensity'], showlegend=False), secondary_y=False)
codet_strings = ['2+']
for codet in codet_strings:
fig.add_trace(go.Scatter(x=df_codets.loc[df_codets['codet_nodes'] == codet, 'generation_time'], y=df_codets.loc[df_codets['codet_nodes'] == codet, 'nr_codets_daily'], mode='lines', line=dict(width=0.5, color=px.colors.qualitative.G10[codet_strings.index(codet)]), name=codet), secondary_y=True)
fig.update_xaxes(title_text='Time')
fig.update_yaxes(title_text='Rain intensity [mm/h]', secondary_y=False)
fig.update_yaxes(title_text='Number of co-detections per day [#]', secondary_y=True)
fig.update_layout(legend_yanchor='top',
legend_y=1.0,
legend_xanchor='right',
legend_x=0.9)
format_figure(fig, figure_name='fig_codet_outdoor_combined.pdf', height_pt=column_width_pt/2, is_line_plot=False, secondary_title_standoff=10, show_legend=False)
if df_health_outdoor is not None:
# Health messages over time
fig = px.scatter(df_health_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id'])
fig.update_yaxes(type='category')
fig.show()
# Uptime
fig = px.line(df_health_outdoor, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
# Sequence number
fig = px.line(df_health_outdoor, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
# Sequence number - Deltas
# df_seqnr = DataProc.extract_transmitted_packets(df_health_outdoor)
# fig = px.line(df_seqnr, x='generation_time', y='nr_packets_hourly', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'nr_packets_hourly': 'Number of packets per hour [#]', 'nr_packets_daily': 'Number of packets per day [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.qualitative.G10, render_mode='svg')
# format_figure(fig, figure_name='fig_packets_outdoor.pdf', height_pt=column_width_pt/2)
# Temperature
fig = px.line(df_health_outdoor, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
# Aggregated Temperature - available options: 'temp_avg', 'temp_median', 'temp_max', 'temp_min'
df_temperature = DataProc.extract_avg_temperature(df_health_outdoor)
fig = px.line(df_temperature, x='generation_time', y='temp_max', labels={'generation_time': 'Time', 'temp_max': 'Temperature [°C]', 'variable': 'Aggregation method', 'value': 'Temperature [°C]'}, color_discrete_sequence=px.colors.qualitative.G10, render_mode='svg')
format_figure(fig, figure_name='fig_temperature_outdoor.pdf', height_pt=column_width_pt/2)
# Base station stats
# bs_data = DataProc.extract_bs_data(df_health_outdoor)
# fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.qualitative.G10)
# fig.update_yaxes(dtick=1)
# fig.show()
if df_meteo_outdoor is not None:
# Rain intensity
fig = px.bar(df_meteo_outdoor, x='timestamp_dt', y='rain_intensity', labels={'timestamp_dt': 'Time', 'rain_intensity': 'Rain intensity [mm/h]'}, color_discrete_sequence=px.colors.qualitative.G10)
format_figure(fig, figure_name='fig_precipitation_outdoor.pdf', height_pt=column_width_pt/2, is_line_plot=False)
```
%% Cell type:code id: tags:
``` python
# Plotting Test Deployment
if df_aggr_test is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
# Number of Co-Detections - GSN vs Logs
df_codets_gsn = DataProc.extract_nr_codets(df_aggr_test, start_date=ETZ_START_TIME, end_date=ETZ_END_TIME)
df_codets_logs = DataProc.extract_nr_codets(df_aggr_logs, start_date=ETZ_START_TIME, end_date=ETZ_END_TIME)
df_codets_gsn['codet_nodes'] = 'Report' # df_codets_gsn['codet_nodes'] + ' Rep'
df_codets_logs['codet_nodes'] = 'GT' # df_codets_logs['codet_nodes'] + ' GT'
df_codets = df_codets_gsn.append(df_codets_logs)
fig = px.line(df_codets, x='generation_time', y='nr_codets_daily', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'nr_codets_hourly': 'Number of co-detections per hour [#]', 'nr_codets_daily': 'Number of co-detections per day [#]', 'codet_nodes': 'Number of co-detecting nodes'}, color='codet_nodes', color_discrete_sequence=px.colors.qualitative.G10, line_dash='codet_nodes', line_dash_sequence=LINE_STYLES_EXP, render_mode='svg')
fig.update_layout(legend_yanchor='top',
legend_y=1.0,
legend_xanchor='right',
legend_x=1,
legend_orientation='h')
# fig.update_yaxes(range=[0, 900])
format_figure(fig, figure_name='fig_codet_test.pdf', height_pt=column_width_pt/2, show_legend=True)
# Number of Co-Detections - GSN vs Simulation
df_codets_sim = DataProc.extract_nr_codets(df_aggr_sim, start_date=ETZ_START_TIME, end_date=ETZ_END_TIME)
df_codets_sim['codet_nodes'] = 'Sim' # df_codets_sim['codet_nodes'] + ' Sim'
df_codets = df_codets_gsn.append(df_codets_sim)
fig = px.line(df_codets, x='generation_time', y='nr_codets_daily', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'nr_codets_hourly': 'Number of co-detections per hour [#]', 'nr_codets_daily': 'Number of co-detections per day [#]', 'codet_nodes': 'Number of co-detecting nodes'}, color='codet_nodes', color_discrete_sequence=px.colors.qualitative.G10, line_dash='codet_nodes', line_dash_sequence=LINE_STYLES_EXP, render_mode='svg')
fig.update_layout(legend_yanchor='top',
legend_y=1.0,
legend_xanchor='right',
legend_x=1,
legend_orientation='h')
# fig.update_yaxes(range=[0, 900])
format_figure(fig, figure_name='fig_codet_sim.pdf', height_pt=column_width_pt/2, show_legend=True)
# Number of Co-Detections - all
df_codets = df_codets_gsn.append(df_codets_sim).append(df_codets_logs)
fig = px.line(df_codets, x='generation_time', y='nr_codets_daily', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'nr_codets_hourly': 'Number of co-detections per hour [#]', 'nr_codets_daily': 'Number of co-detections per day [#]', 'codet_nodes': 'Number of co-detecting nodes'}, color='codet_nodes', color_discrete_sequence=px.colors.qualitative.G10, line_dash='codet_nodes', line_dash_sequence=LINE_STYLES_EXP_ALL, render_mode='svg')
fig.update_layout(legend_yanchor='top',
legend_y=1.0,
legend_xanchor='right',
legend_x=1,
legend_orientation='h')
# fig.update_yaxes(range=[0, 900])
format_figure(fig, figure_name='fig_codet_etz.pdf', height_pt=column_width_pt/2, show_legend=True)
# Number of Co-Detections - GSN vs Simulation
df_codets = df_codets_logs.append(df_codets_sim)
fig = px.line(df_codets, x='generation_time', y='nr_codets_daily', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'nr_codets_hourly': 'Number of co-detections per hour [#]', 'nr_codets_daily': 'Number of co-detections per day [#]', 'codet_nodes': 'Number of co-detecting nodes'}, color='codet_nodes', color_discrete_sequence=px.colors.qualitative.G10, line_dash='codet_nodes', line_dash_sequence=LINE_STYLES_EXP, render_mode='svg')
format_figure(fig, figure_name='fig_codet_logs.pdf', height_pt=column_width_pt/2)
if df_health_test is not None:
# Health messages over time
fig = px.scatter(df_health_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id'])
fig.update_yaxes(type='category')
fig.show()
# Uptime
fig = px.line(df_health_test, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
# Sequence number
fig = px.line(df_health_test, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
# Temperature
fig = px.line(df_health_test, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.qualitative.G10)
fig.show()
# Base station stats
# bs_data = DataProc.extract_bs_data(df_health_test)
# fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.qualitative.G10)
# fig.update_yaxes(dtick=1)
# fig.show()
if df_health_test is None and df_aggr_test is None:
print('Skipping test data as not able to access from this machine (no connection to ETH-internal network)')
```
%% Cell type:markdown id: tags:
## Statistics
After having pre-processed the data, we now compute statistics over the gathered data.
%% Cell type:code id: tags:
``` python
node_statistics_outdoor = DataProc.extract_node_data(df_health_outdoor, df_aggr_outdoor, html_output=True)
node_statistics_test = DataProc.extract_node_data(df_health_test, df_aggr_test, html_output=True)
```
......
%% Cell type:markdown id: tags:
**Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)**
# STeC Health Visualisation
This script visualizes the health data of the STeC deployments both on DH and ETZ (test deployment).
%% Cell type:code id: tags:
``` python
import datetime as dt
import pandas as pd
import sys
import plotly.express as px
sys.path.append('../') # FIXME: Work around if not built as a package
from data_management.data_manager import DataManager
from data_management.data_processing import DataProcessor
# Settings
TEST_SERVER = 'http://tpbl.permasense.ethz.ch/'
DEPLOYMENT_OUTDOOR = 'dirruhorn'
DEPLOYMENT_TEST = 'etz'
VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_AGGR = '_dpp_geophone_acq_min_aggr__mapped'
VS_ACOUSTIC_DATA = '_dpp_geophone_adcData__conv'
VS_HEALTH_MIN_DATA = '_dpp_health_min__conv'
DATA_START_TIME = dt.datetime.strptime('19/10/2020', '%d/%m/%Y')
DATA_END_TIME = dt.datetime.today()
S_TO_US = 1000 * 1000
FALLBACK_BS_IDS_OUTDOOR = [106, 110, 111, 112]
FALLBACK_BS_IDS_TEST = [103, 107, 109]
# Print settings
print('Going to fetch STeC Health data at deployment site {0:s} and {1:s} from {2:s} - {3:s}'.format(DEPLOYMENT_OUTDOOR.capitalize(), DEPLOYMENT_TEST.capitalize(), DATA_START_TIME.strftime('%d/%m/%Y'), DATA_END_TIME.strftime('%d/%m/%Y')))
# Adjust display width for variables
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
```
%% Cell type:code id: tags:
``` python
# Create necessary objects
DataMgr = DataManager(deployment=DEPLOYMENT_OUTDOOR, config_file='../stec.conf', project_name='stec', start_time=DATA_START_TIME, end_time=DATA_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf')
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')
# Fetch data
# Create URL including conditions
url_health_outdoor = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA)
url_aggr_outdoor = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR)
url_health_test = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
url_aggr_test = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
# Fetch data
df_health_outdoor = DataMgr.fetch_csv_data(url_health_outdoor, description="Health", cache=False)
df_aggr_outdoor = DataMgr.fetch_csv_data(url_aggr_outdoor, description="Aggregation", cache=False)
df_health_test = DataMgr.fetch_csv_data(url_health_test, description="Health", abort=False, cache=False)
df_aggr_test = DataMgr.fetch_csv_data(url_aggr_test, description="Aggregation", abort=False, cache=False)
```
%% Cell type:code id: tags:
``` python
# Prepare data for plotting
# Formatting timestamps, add column indicating whether the packet was received over SF7 or SF10
if df_health_outdoor is not None:
df_health_outdoor['generation_time'] = pd.to_datetime(df_health_outdoor['generation_time_microsec'], unit='us')
df_health_outdoor['device_id_str'] = df_health_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_outdoor['fallback'] = df_health_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_health_outdoor['fallback_str'] = df_health_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_outdoor is not None:
df_aggr_outdoor['generation_time'] = pd.to_datetime(df_aggr_outdoor['generation_time_microsec'], unit='us')
df_aggr_outdoor['device_id_str'] = df_aggr_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_outdoor['fallback'] = df_aggr_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_aggr_outdoor['fallback_str'] = df_aggr_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_health_test is not None:
df_health_test['generation_time'] = pd.to_datetime(df_health_test['generation_time_microsec'], unit='us')
df_health_test['device_id_str'] = df_health_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_test['fallback'] = df_health_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_health_test['fallback_str'] = df_health_test['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_test is not None:
df_aggr_test['generation_time'] = pd.to_datetime(df_aggr_test['generation_time_microsec'], unit='us')
df_aggr_test['device_id_str'] = df_aggr_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_test['fallback'] = df_aggr_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_aggr_test['fallback_str'] = df_aggr_test['fallback'].astype(str) # Required for correct colouring of scatter plots
# Color mapping for consistent display of fallback data
color_fallback_map = {'True': 'red', 'False': 'green'}
```
%% Cell type:markdown id: tags:
## Statistics
After having pre-processed the data, we now compute statistics over the gathered data.
%% Cell type:code id: tags:
``` python
node_statistics_outdoor = DataProc.extract_node_data(df_health_outdoor, df_aggr_outdoor, html_output=True)
node_statistics_test = DataProc.extract_node_data(df_health_test, df_aggr_test, html_output=True)
```
%% Cell type:markdown id: tags:
## Plotting
After having pre-processed the data, we now plot the data for visual inspection.
%% Cell type:code id: tags:
``` python
# Plotting Outdoor
if df_health_outdoor is not None:
# Health messages over time
fig = px.scatter(df_health_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id', 'position'], title='Outdoor Deployment (Dirruhorn)')
fig.update_yaxes(type='category')
fig.show()
if df_aggr_outdoor is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
if df_health_outdoor is not None:
# Uptime
fig = px.line(df_health_outdoor, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Sequence number
fig = px.line(df_health_outdoor, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Temperature
fig = px.line(df_health_outdoor, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Humidity
fig = px.line(df_health_outdoor, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Radio RSSI
fig = px.line(df_health_outdoor, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Clock drift
drift_data = DataProc.extract_clock_drift(df_health_outdoor, min_id=21006, reference_id=112)
# fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
# fig.show()
fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Base station stats
bs_data = DataProc.extract_bs_data(df_health_outdoor, start_date=dt.datetime(year=2020, month=10, day=27))
fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.matter)
fig.update_yaxes(dtick=1)
fig.show()
```
%% Cell type:code id: tags:
``` python
# Plotting Test Deployment
if df_health_test is not None:
# Health messages over time
fig = px.scatter(df_health_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id'], title='Test Deployment (ETZ)')
fig.update_yaxes(type='category')
fig.show()
if df_aggr_test is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
if df_health_test is not None:
# Uptime
fig = px.line(df_health_test, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Sequence number
fig = px.line(df_health_test, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Temperature
fig = px.line(df_health_test, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Humidity
fig = px.line(df_health_test, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Radio RSSI
fig = px.line(df_health_test, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Clock drift
drift_data = DataProc.extract_clock_drift(df_health_test, min_id=21006, reference_id=108)
#fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
#fig.show()
fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Base station stats
bs_data = DataProc.extract_bs_data(df_health_test, start_date=dt.datetime(year=2020, month=10, day=27))
fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.deep)
fig.update_yaxes(dtick=1)
fig.show()
if df_health_test is None and df_aggr_test is None:
print('Skipping test data as not able to access from this machine (no connection to ETH-internal network)')
```
......
%% Cell type:markdown id: tags:
**Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)**
# STeC Health Visualisation (weekly)
This script visualizes the health data of the STeC deployments for the last week both on DH and ETZ (test deployment).
%% Cell type:code id: tags:
``` python
import datetime as dt
import pandas as pd
import sys
import plotly.express as px
sys.path.append('../') # FIXME: Work around if not built as a package
from data_management.data_manager import DataManager
from data_management.data_processing import DataProcessor
# Settings
TEST_SERVER = 'http://tpbl.permasense.ethz.ch/'
DEPLOYMENT_OUTDOOR = 'dirruhorn'
DEPLOYMENT_TEST = 'etz'
VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_AGGR = '_dpp_geophone_acq_min_aggr__mapped'
VS_ACOUSTIC_DATA = '_dpp_geophone_adcData__conv'
VS_HEALTH_MIN_DATA = '_dpp_health_min__conv'
DATA_START_TIME = dt.datetime.today() - dt.timedelta(days=7)
DATA_END_TIME = dt.datetime.today()
S_TO_US = 1000 * 1000
FALLBACK_BS_IDS_OUTDOOR = [106, 110]
FALLBACK_BS_IDS_TEST = [103, 107, 109]
# Print settings
print('Going to fetch STeC Health data at deployment site {0:s} and {1:s} from {2:s} - {3:s}'.format(DEPLOYMENT_OUTDOOR.capitalize(), DEPLOYMENT_TEST.capitalize(), DATA_START_TIME.strftime('%d/%m/%Y'), DATA_END_TIME.strftime('%d/%m/%Y')))
# Adjust display width for variables
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
```
%% Cell type:code id: tags:
``` python
# Create necessary objects
DataMgr = DataManager(deployment=DEPLOYMENT_OUTDOOR, config_file='../stec.conf', project_name='stec', start_time=DATA_START_TIME, end_time=DATA_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf')
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')
# Fetch data
# Create URL including conditions
url_health_outdoor = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA)
url_aggr_outdoor = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR)
url_health_test = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
url_aggr_test = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
# Fetch data
df_health_outdoor = DataMgr.fetch_csv_data(url_health_outdoor, description="Health", cache=False)
df_aggr_outdoor = DataMgr.fetch_csv_data(url_aggr_outdoor, description="Aggregation", cache=False)
df_health_test = DataMgr.fetch_csv_data(url_health_test, description="Health", abort=False, cache=False)
df_aggr_test = DataMgr.fetch_csv_data(url_aggr_test, description="Aggregation", abort=False, cache=False)
```
%% Cell type:code id: tags:
``` python
# Prepare data for plotting
# Formatting timestamps, add column indicating whether the packet was received over SF7 or SF10
if df_health_outdoor is not None:
df_health_outdoor['generation_time'] = pd.to_datetime(df_health_outdoor['generation_time_microsec'], unit='us')
df_health_outdoor['device_id_str'] = df_health_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_outdoor['fallback'] = df_health_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_health_outdoor['fallback_str'] = df_health_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_outdoor is not None:
df_aggr_outdoor['generation_time'] = pd.to_datetime(df_aggr_outdoor['generation_time_microsec'], unit='us')
df_aggr_outdoor['device_id_str'] = df_aggr_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_outdoor['fallback'] = df_aggr_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_aggr_outdoor['fallback_str'] = df_aggr_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_health_test is not None:
df_health_test['generation_time'] = pd.to_datetime(df_health_test['generation_time_microsec'], unit='us')
df_health_test['device_id_str'] = df_health_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_test['fallback'] = df_health_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_health_test['fallback_str'] = df_health_test['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_test is not None:
df_aggr_test['generation_time'] = pd.to_datetime(df_aggr_test['generation_time_microsec'], unit='us')
df_aggr_test['device_id_str'] = df_aggr_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_test['fallback'] = df_aggr_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_aggr_test['fallback_str'] = df_aggr_test['fallback'].astype(str) # Required for correct colouring of scatter plots
# Color mapping for consistent display of fallback data
color_fallback_map = {'True': 'red', 'False': 'green'}
```
%% Cell type:markdown id: tags:
## Plotting
After having pre-processed the data, we now plot the data for visual inspection.
%% Cell type:code id: tags:
``` python
# Plotting Outdoor
if df_health_outdoor is not None:
# Health messages over time
fig = px.scatter(df_health_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id', 'position'], title='Outdoor Deployment (Dirruhorn)')
fig.update_yaxes(type='category')
fig.show()
if df_aggr_outdoor is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
else:
print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')
if df_health_outdoor is not None:
# Uptime
fig = px.line(df_health_outdoor, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Sequence number
fig = px.line(df_health_outdoor, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Temperature
fig = px.line(df_health_outdoor, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Humidity
fig = px.line(df_health_outdoor, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Radio RSSI
fig = px.line(df_health_outdoor, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Clock drift
drift_data = DataProc.extract_clock_drift(df_health_outdoor, min_id=21006, reference_id=112)
# fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
# fig.show()
fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Base station stats
bs_data = DataProc.extract_bs_data(df_health_outdoor)
fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.matter)
fig.update_yaxes(dtick=1)
fig.show()
```
%% Cell type:code id: tags:
``` python
# Plotting Test Deployment
if df_health_test is not None:
# Health messages over time
fig = px.scatter(df_health_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id'], title='Test Deployment (ETZ)')
fig.update_yaxes(type='category')
fig.show()
if df_aggr_test is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
else:
print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')
if df_health_test is not None:
# Uptime
fig = px.line(df_health_test, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Sequence number
fig = px.line(df_health_test, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Temperature
fig = px.line(df_health_test, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Humidity
fig = px.line(df_health_test, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Radio RSSI
fig = px.line(df_health_test, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Clock drift
drift_data = DataProc.extract_clock_drift(df_health_test, min_id=21006, reference_id=108)
#fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
#fig.show()
fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Base station stats
bs_data = DataProc.extract_bs_data(df_health_test)
fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.deep)
fig.update_yaxes(dtick=1)
fig.show()
if df_health_test is None and df_aggr_test is None:
print('Skipping test data as not able to access from this machine (no connection to ETH-internal network)')
```
......
......@@ -400,7 +400,7 @@ class Simulator:
if use_trace_data or use_real_data:
DataMgr = DataManager(deployment='dirruhorn', config_file='../../analysis/stec.conf', project_name='stec', start_time=dt.datetime.strptime("01/06/2018", "%d/%m/%Y"), end_time=dt.datetime.strptime("01/01/2020", "%d/%m/%Y"))
DataMgr_logs = DataManager(deployment='etz', config_file='../../analysis/stec.conf', project_name='stec', start_time=dt.datetime.strptime("19/10/2020", "%d/%m/%Y"), end_time=dt.datetime.strptime("17/05/2021", "%d/%m/%Y"))
DataProc = DataProcessor(config_file='../../analysis/stec.conf')
DataProc = DataProcessor(config_file='../../analysis/stec.conf', project_name='stec')
if use_real_data:
if use_trace_data:
......
%% Cell type:markdown id: tags:
**Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)**
# Discrete Event Simulation Evaluation
This script visualizes the evaluation plots.
%% Cell type:code id: tags:
``` python
import sys
import os
import pandas as pd
import numpy as np
import datetime as dt
import plotly.express as px
sys.path.append("../")
sys.path.append("../../analysis")
from des.des_eval import load_metrics, generate_postfix
from data_management.data_manager import DataManager
from data_management.data_processing import DataProcessor
```
%% Cell type:code id: tags:
``` python
METRIC_PATH = '/output/'
EXP_METRIC_PATH = '/../../analysis/bin/'
# General constants
MS_TO_US = 1000
S_TO_MS = 1000
S_TO_US = 1000 * 1000
# Parameters
DEFAULT_SPREADING_FACTOR = 11
DEFAULT_CODET_THRESHOLD = 2
DEFAULT_PERIOD_S = 15
DEFAULT_SAMPLE_DURATION_MS = 100
SPREADING_FACTORS = [11]
CODET_THRESHOLDS = range(2, 15)
ELWB_PERIODS_S = [5, 10, 15, 30, 60, 120, 300]
SAMPLE_DURATIONS_MS = [100, 250, 500, 1000, 2000, 5000, 10000, 15000]
EVT_INTERVAL_S = [5, 15, 30, 60, 300, 900, 1800, 3600, 7200, 21600, 86400]
EVT_INTERVAL_TEXT = ['5s', '15s', '30s', '1min', '5min', '15min', '30min', '1h', '2h', '6h', '1d']
NETWORK_SIZES = range(2,10)
# Paper constants - use font size 14 if using 1/2 column
column_width_pt = 241.14749
page_width_pt = 506.295
pt_to_px = 1.3281472327365
ebcIDs_to_paperIDs = {21037: 1, 21031: 2, 21034: 3, 21029: 4, 21024: 5, 21027: 6, 21028: 7, 21032: 8, 21025: 9}
simIDs_to_paperIDs = {21004: 1, 21005: 2, 21006: 3, 21007: 4, 21008: 5, 21009: 6, 21010: 7, 21011: 8, 21012: 9, 21013: 10, 21014: 11, 21015: 12, 21016: 13, 21017: 14, 21018: 15, 21019: 16, 21020: 17, 21021: 18, 21022: 19, 21023: 20, 21024: 21, 21025: 22, 21026: 23, 21027: 24, 21028: 25, 21029: 26, 21030: 27, 21031: 28, 21032: 29, 21034: 30, 21035: 31, 21037: 32}
```
%% Cell type:code id: tags:
``` python
# Generate dataframe out of final metrics
rows_list = []
for spreading_factor in SPREADING_FACTORS:
for codet_threshold in CODET_THRESHOLDS:
# Load metrics from file
metrics_ebc = load_metrics(load_ebc=True, relative_path=METRIC_PATH, postfix=generate_postfix(sf=spreading_factor, cd=codet_threshold, sd=DEFAULT_SAMPLE_DURATION_MS))
metrics_elwb = load_metrics(load_elwb=True, relative_path=METRIC_PATH, postfix=generate_postfix(sf=spreading_factor, cd=codet_threshold, sd=DEFAULT_SAMPLE_DURATION_MS, p=DEFAULT_PERIOD_S))
metrics_aloha = load_metrics(load_aloha=True, relative_path=METRIC_PATH, postfix=generate_postfix(sf=spreading_factor, cd=codet_threshold, sd=DEFAULT_SAMPLE_DURATION_MS))
# Add current info
nr_nodes = len(metrics_ebc['node_id'].unique())
nr_evts_total = metrics_ebc['nr_evts_tot'].sum()
nr_evts_reported_ebc = metrics_ebc['nr_evts_rep'].sum()
energy_com_avg_ebc = metrics_ebc['energy_com_tot'].sum() / nr_evts_total
energy_com_wc_ebc = metrics_ebc['energy_com_tot'].max() / (nr_evts_total / nr_nodes)
latency_com_avg_ebc = metrics_ebc['latency_tot'].sum() / nr_evts_reported_ebc
latency_com_wc_ebc = metrics_ebc['latency_tot'].max() / (nr_evts_reported_ebc / nr_nodes)
rel_evts_rep_ebc = metrics_ebc.iloc[0].at['nr_codet_evts_rep'] / metrics_ebc.iloc[0].at['nr_codet_evts_tot'] * 100
rel_evts_extra_ebc = metrics_ebc.iloc[0].at['nr_evts_superfluous'] / metrics_ebc.iloc[0].at['nr_codet_evts_tot'] * 100
nr_evts_reported_elwb = metrics_elwb['nr_evts_rep'].sum() if metrics_elwb is not None else None
energy_com_avg_elwb = metrics_elwb['energy_com_tot'].sum() / nr_evts_total if metrics_elwb is not None else None
energy_com_wc_elwb = metrics_elwb['energy_com_tot'].max() / (nr_evts_total / nr_nodes) if metrics_elwb is not None else None
latency_com_avg_elwb = metrics_elwb['latency_tot'].sum() / nr_evts_reported_elwb if metrics_elwb is not None else None
latency_com_wc_elwb = metrics_elwb['latency_tot'].max() / (nr_evts_reported_elwb / nr_nodes) if metrics_elwb is not None else None
rel_evts_rep_elwb = metrics_elwb.iloc[0].at['nr_codet_evts_rep'] / metrics_elwb.iloc[0].at['nr_codet_evts_tot'] * 100 if metrics_elwb is not None else None
rel_evts_extra_elwb = metrics_elwb.iloc[0].at['nr_evts_superfluous'] / metrics_elwb.iloc[0].at['nr_codet_evts_tot'] * 100 if metrics_elwb is not None else None
nr_evts_reported_aloha = metrics_aloha['nr_evts_rep'].sum()
energy_com_avg_aloha = metrics_aloha['energy_com_tot'].sum() / nr_evts_total
energy_com_wc_aloha = metrics_aloha['energy_com_tot'].max() / (nr_evts_total / nr_nodes)
latency_com_avg_aloha = metrics_aloha['latency_tot'].sum() / nr_evts_reported_aloha
latency_com_wc_aloha = metrics_aloha['latency_tot'].max() / (nr_evts_reported_aloha / nr_nodes)
rel_evts_rep_aloha = metrics_aloha.iloc[0].at['nr_codet_evts_rep'] / metrics_aloha.iloc[0].at['nr_codet_evts_tot'] * 1
rel_evts_extra_aloha = metrics_aloha.iloc[0].at['nr_evts_superfluous'] / metrics_aloha.iloc[0].at['nr_codet_evts_tot'] * 1
row = {'nr_nodes': nr_nodes, 'nr_evts_tot': nr_evts_total, 'spreading_factor': str(spreading_factor), 'codet_thres': codet_threshold,
'nr_evts_rep_ebc': nr_evts_reported_ebc, 'energy_ebc': energy_com_avg_ebc, 'latency_ebc': latency_com_avg_ebc, 'rel_evts_rep_ebc': rel_evts_rep_ebc, 'rel_evts_extra_ebc': rel_evts_extra_ebc,
'nr_evts_rep_elwb': nr_evts_reported_elwb, 'energy_elwb': energy_com_avg_elwb, 'latency_elwb': latency_com_avg_elwb, 'rel_evts_rep_elwb': rel_evts_rep_elwb, 'rel_evts_extra_elwb': rel_evts_extra_elwb,
'nr_evts_rep_aloha': nr_evts_reported_aloha, 'energy_aloha': energy_com_avg_aloha, 'latency_aloha': latency_com_avg_aloha, 'rel_evts_rep_aloha': rel_evts_rep_aloha, 'rel_evts_extra_aloha': rel_evts_extra_aloha}
rows_list.append(row)
# Add worst case as well
row = {'nr_nodes': nr_nodes, 'nr_evts_tot': nr_evts_total, 'spreading_factor': str(spreading_factor) + ' WC', 'codet_thres': codet_threshold,
'nr_evts_rep_ebc': nr_evts_reported_ebc, 'energy_ebc': energy_com_wc_ebc, 'latency_ebc': latency_com_wc_ebc,
'nr_evts_rep_elwb': nr_evts_reported_elwb, 'energy_elwb': energy_com_wc_elwb, 'latency_elwb': latency_com_wc_elwb,
'nr_evts_rep_aloha': nr_evts_reported_aloha, 'energy_aloha': energy_com_wc_aloha, 'latency_aloha': latency_com_wc_aloha}
rows_list.append(row)
metrics_final = pd.DataFrame(rows_list)
# Generate dataframe out of all eLWB metrics with different round periods
rows_list.clear()
for spreading_factor in SPREADING_FACTORS:
for elwb_period in ELWB_PERIODS_S:
# Load metrics from file
metrics_elwb = load_metrics(load_elwb=True, relative_path=METRIC_PATH, postfix=generate_postfix(sf=spreading_factor, cd=DEFAULT_CODET_THRESHOLD, sd=DEFAULT_SAMPLE_DURATION_MS, p=elwb_period))
# Exclude non-existing files (due to round duration being longer than period, invalidating the simulation)
if metrics_elwb is None:
continue
# Add current info
nr_nodes = len(metrics_elwb['node_id'].unique())
nr_evts_total = metrics_elwb['nr_evts_tot'].sum()
nr_evts_reported_elwb = metrics_elwb['nr_evts_rep'].sum()
energy_com_avg_elwb = metrics_elwb['energy_com_tot'].sum() / nr_evts_total
energy_com_wc_elwb = metrics_elwb['energy_com_tot'].max() / (nr_evts_total / nr_nodes)
latency_com_avg_elwb = metrics_elwb['latency_tot'].sum() / nr_evts_reported_elwb
latency_com_wc_elwb = metrics_elwb['latency_tot'].max() / (nr_evts_reported_elwb / nr_nodes)
rel_evts_rep_elwb = metrics_elwb.iloc[0].at['nr_codet_evts_rep'] / metrics_elwb.iloc[0].at['nr_codet_evts_tot'] * 100
rel_evts_extra_elwb = metrics_elwb.iloc[0].at['nr_evts_superfluous'] / metrics_elwb.iloc[0].at['nr_codet_evts_tot'] * 100
row = {'nr_nodes': nr_nodes, 'nr_evts_tot': nr_evts_total, 'spreading_factor': str(spreading_factor), 'elwb_period': elwb_period,
'nr_evts_rep_elwb': nr_evts_reported_elwb, 'energy_elwb': energy_com_avg_elwb, 'latency_elwb': latency_com_avg_elwb, 'rel_evts_rep_elwb': rel_evts_rep_elwb, 'rel_evts_extra_elwb': rel_evts_extra_elwb}
rows_list.append(row)
# Add worst case as well
row = {'nr_nodes': nr_nodes, 'nr_evts_tot': nr_evts_total, 'spreading_factor': str(spreading_factor) + ' WC', 'elwb_period': elwb_period,
'nr_evts_rep_elwb': nr_evts_reported_elwb, 'energy_elwb': energy_com_wc_elwb, 'latency_elwb': latency_com_wc_elwb}
rows_list.append(row)
metrics_elwb_period = pd.DataFrame(rows_list)
# Generate dataframe out of metrics with different sample duration for STeC and ALOHA
rows_list.clear()
for spreading_factor in SPREADING_FACTORS:
for duration in SAMPLE_DURATIONS_MS:
# Load metrics from file
metrics_ebc = load_metrics(load_ebc=True, relative_path=METRIC_PATH, postfix=generate_postfix(sf=spreading_factor, cd=DEFAULT_CODET_THRESHOLD, sd=duration))
metrics_aloha = load_metrics(load_aloha=True, relative_path=METRIC_PATH, postfix=generate_postfix(sf=spreading_factor, cd=DEFAULT_CODET_THRESHOLD, sd=duration))
# Add current info
nr_nodes = len(metrics_ebc['node_id'].unique())
nr_evts_total = metrics_ebc['nr_evts_tot'].sum()
nr_evts_reported_ebc = metrics_ebc['nr_evts_rep'].sum()
energy_com_avg_ebc = metrics_ebc['energy_com_tot'].sum() / nr_evts_total
energy_com_wc_ebc = metrics_ebc['energy_com_tot'].max() / (nr_evts_total / nr_nodes)
latency_com_avg_ebc = metrics_ebc['latency_tot'].sum() / nr_evts_reported_ebc
latency_com_wc_ebc = metrics_ebc['latency_tot'].max() / (nr_evts_reported_ebc / nr_nodes)
rel_evts_rep_ebc = metrics_ebc.iloc[0].at['nr_codet_evts_rep'] / metrics_ebc.iloc[0].at['nr_codet_evts_tot'] * 100
rel_evts_extra_ebc = metrics_ebc.iloc[0].at['nr_evts_superfluous'] / metrics_ebc.iloc[0].at['nr_codet_evts_tot'] * 100
nr_evts_reported_aloha = metrics_aloha['nr_evts_rep'].sum()
energy_com_avg_aloha = metrics_aloha['energy_com_tot'].sum() / nr_evts_total
energy_com_wc_aloha = metrics_aloha['energy_com_tot'].max() / (nr_evts_total / nr_nodes)
latency_com_avg_aloha = metrics_aloha['latency_tot'].sum() / nr_evts_reported_aloha
latency_com_wc_aloha = metrics_aloha['latency_tot'].max() / (nr_evts_reported_aloha / nr_nodes)
rel_evts_rep_aloha = metrics_aloha.iloc[0].at['nr_codet_evts_rep'] / metrics_aloha.iloc[0].at['nr_codet_evts_tot'] * 1
rel_evts_extra_aloha = metrics_aloha.iloc[0].at['nr_evts_superfluous'] / metrics_aloha.iloc[0].at['nr_codet_evts_tot'] * 1
row = {'nr_nodes': nr_nodes, 'nr_evts_tot': nr_evts_total, 'spreading_factor': str(spreading_factor), 'sample_duration': duration,
'nr_evts_rep_ebc': nr_evts_reported_ebc, 'energy_ebc': energy_com_avg_ebc, 'latency_ebc': latency_com_avg_ebc, 'rel_evts_rep_ebc': rel_evts_rep_ebc, 'rel_evts_extra_ebc': rel_evts_extra_ebc,
'nr_evts_rep_aloha': nr_evts_reported_aloha, 'energy_aloha': energy_com_avg_aloha, 'latency_aloha': latency_com_avg_aloha, 'rel_evts_rep_aloha': rel_evts_rep_aloha, 'rel_evts_extra_aloha': rel_evts_extra_aloha}
rows_list.append(row)
# Add worst case as well
row = {'nr_nodes': nr_nodes, 'nr_evts_tot': nr_evts_total, 'spreading_factor': str(spreading_factor) + ' WC', 'sample_duration': duration,
'nr_evts_rep_ebc': nr_evts_reported_ebc, 'energy_ebc': energy_com_wc_ebc, 'latency_ebc': latency_com_wc_ebc,
'nr_evts_rep_aloha': nr_evts_reported_aloha, 'energy_aloha': energy_com_wc_aloha, 'latency_aloha': latency_com_wc_aloha}
rows_list.append(row)
metrics_sampling = pd.DataFrame(rows_list)
# Generate dataframe out of metrics with synthetic traces of different event intervals
DEFAULT_NR_CODETS = 500
PERIODS_S = [15, 60, 300, 900]
rows_list.clear()
for events_per_codet in [1, 5]:
for event_interval_s in EVT_INTERVAL_S:
# Load metrics from file
curr_postfix_ebc = generate_postfix(sf=DEFAULT_SPREADING_FACTOR, cd=DEFAULT_CODET_THRESHOLD, sd