Commit 43a909bd authored by matthmey's avatar matthmey

added kwargs to csv_with_store

parent b22376df
......@@ -46,16 +46,22 @@ def dat(x):
return dask.delayed(x)
def to_csv_with_store(store, filename, dataframe):
def to_csv_with_store(store, filename, dataframe, pandas_kwargs=None):
if pandas_kwargs is None:
pandas_kwargs = dict()
StreamWriter = codecs.getwriter("utf-8")
bytes_buffer = io.BytesIO()
string_buffer = StreamWriter(bytes_buffer)
dataframe.to_csv(string_buffer, index=False)
dataframe.to_csv(string_buffer, **pandas_kwargs)
store[filename] = bytes_buffer.getvalue()
def read_csv_with_store(store, filename):
def read_csv_with_store(store, filename, pandas_kwargs=None):
if pandas_kwargs is None:
pandas_kwargs = dict()
bytes_buffer = io.BytesIO(store[str(filename)])
StreamReader = codecs.getreader("utf-8")
string_buffer = StreamReader(bytes_buffer)
return pd.read_csv(string_buffer)
return pd.read_csv(string_buffer,**pandas_kwargs)
......@@ -680,7 +680,7 @@ class MHDSLRFilenames(DataSource):
# try to reload it and write to remote
imglist_df = self.image_integrity_store(config["store"])
try:
to_csv_with_store(config["store"], filename, imglist_df)
to_csv_with_store(config["store"], filename, imglist_df, dict(index=False))
success = True
except Exception as e:
print(e)
......@@ -1361,7 +1361,7 @@ class SegmentedDataset(Dataset):
label_coords, label_slices = get_label_slices(l)
print(label_slices, slices)
# print(label_slices, slices)
# print(len(l),len(label_slices))
# filter out where we do not get data for the slice
......@@ -1408,98 +1408,7 @@ class SegmentedDataset(Dataset):
}
label_list = [label_dict[key] for key in label_dict]
print(label_list)
def check():
from plotly.subplots import make_subplots
import plotly.graph_objects as go
fig = go.Figure(
layout=dict(
title=dict(text="A Bar Chart"),
xaxis={"type": "date"},
xaxis_range=[
pd.to_datetime("2017-08-01"),
pd.to_datetime("2017-08-06"),
],
)
)
# fig = make_subplots(rows=3, cols=1, shared_xaxes=True, vertical_spacing = 0.)
# fig.update_xaxes(range=[pd.to_datetime('2017-08-01'), pd.to_datetime('2017-08-03')])
# fig.update_yaxes(range=[0, 2])
for i, sl in enumerate([slices, label_slices, label_list]):
for item in sl:
label = "None"
if "indexers" in item:
label = item["labels"]
item = item["indexers"]
points = np.array(
[
[pd.to_datetime(item["time"].start), 1],
[pd.to_datetime(item["time"].stop), 0],
]
)
# fig.add_shape(
# # Line reference to the axes
# go.layout.Shape(
# type="rect",
# xref="x",
# yref="paper",
# x0=pd.to_datetime(item['time'].start),
# y0=0,
# x1=pd.to_datetime(item['time'].stop),
# y1=1,
# fillcolor="LightSalmon",
# opacity=0.5,
# layer="below",
# line_width=0,
# # line=dict(
# # color="LightSeaGreen",
# # width=3,
# # ),
# ))
fig.add_trace(
go.Scatter(
x=[
pd.to_datetime(item["time"].start),
pd.to_datetime(item["time"].stop),
pd.to_datetime(item["time"].stop),
pd.to_datetime(item["time"].start),
],
y=[0, 0, 1, 1],
fill="toself",
fillcolor="darkviolet",
# marker={'size':0},
mode="lines",
hoveron="points+fills", # select where hover is active
line_color="darkviolet",
showlegend=False,
# line_width=0,
opacity=0.5,
text=str(label),
hoverinfo="text+x+y",
)
)
# fig.add_trace(go.Scatter(x=points[:,0], y=points[:,1],
# fill=None,
# mode='lines',
# line_color=None, line_width=0,
# name="hv", line_shape='hv',
# showlegend=False,
# hovertext=str(label),
# hoveron = 'points+fills',
# ))
# fig.add_trace(go.Scatter(
# x=points[:,0],
# y=np.zeros_like(points[:,1]),
# showlegend=False,
# hovertext=str(label),
# hoveron = 'points+fills',
# fill='tonexty', # fill area between trace0 and trace1
# mode='lines', line_width=0))
fig.show()
# print(label_list)
self.label_list = label_list
self.data = data
......@@ -1531,11 +1440,10 @@ class SegmentedDataset(Dataset):
x = data.compute()
torch.Tensor(x.values)
print(x)
# print(x)
return segment
class PytorchDataset(DataSource): # TODO: extends pytorch dataset
def __init__(self, source=None):
""" Creates a pytorch like dataset from a data source and a label source.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment