To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit e2e35c66 authored by matthmey's avatar matthmey
Browse files

fix bugs in freeze

parent 2bfd459b
......@@ -227,10 +227,11 @@ class GSNDataSource(DataSource):
pass
# Remove columns with only 'null'
df = df.replace(r"null", np.nan, regex=True)
# df = df.replace(r"null", np.nan, regex=True)
isnull = df.isnull().all()
[df.drop([col_name], axis=1, inplace=True) for col_name in df.columns[isnull]]
df['time'] = pd.to_datetime(df['time'])
df = df.set_index("time")
df = df.sort_index(axis=1)
......@@ -1199,10 +1200,10 @@ class Freezer(StuettNode):
x = data.to_dataset(name="frozen")
# x = x.chunk({name: x[name].shape for name in list(x.dims)})
# zarr_dataset = zarr.open(self.store, mode='r')
x.to_zarr(request['store'], append_dim="time")
x.to_zarr(request['store'],group=self.config['groupname'], append_dim="time")
def open_zarr(self, request):
ds_zarr = xr.open_zarr(request['store'])
ds_zarr = xr.open_zarr(request['store'],group=self.config['groupname'])
if f"{self.config['groupname']}_indexers" in request:
ds_zarr = ds_zarr.sortby('time')
indexers = request[f"{self.config['groupname']}_indexers"]
......@@ -1226,14 +1227,17 @@ class Freezer(StuettNode):
offset = self.config['offset']
meta_list = []
while current_chunk <= end_chunk:
print(current_chunk, end_chunk)
# print(current_chunk, end_chunk)
if current_chunk not in meta[dim]:
meta_list.append(current_chunk)
# avoid having data points twice
off = offset - pd.to_timedelta('1 ns')
current_indexers = {'time':slice(current_chunk,current_chunk+off)}
# TODO: Issue here is that we call a dask computation within a dask computation
self.to_zarr(data.sel(current_indexers),request)
# print(current_indexers)
# print(data)
data_sel = data.sel(current_indexers)
self.to_zarr(data_sel,request)
current_chunk = current_chunk + offset
meta_update = pd.DataFrame(meta_list,columns=[dim])
......
......@@ -55,11 +55,14 @@ class MinMaxDownsampling(StuettNode):
x_min = rolling.min().dropna(dim)
x_max = rolling.max().dropna(dim)
# TODO: better interleave instead of concat
x_ds = xr.concat(
[x_min, x_max], dim
) # TODO: better interleave instead of concat
)
x_ds = x_ds.sortby(dim) # TODO: try to avoid this by using interleaving
# TODO: try to avoid this by using interleaving
x_ds = x_ds.sortby(dim)
return x_ds
......@@ -74,7 +77,7 @@ class LTTBDownsampling(StuettNode):
dim = self.dim
if dim is None:
dim = data.dims[-1]
""" The following tries to re-implement the LTTB for xarray
""" The following tries to re-implement the LTTB algorithm for xarray
There are several issues:
1. We cannot use the LTTB package
it expects the index and the data in one numpy array. Since our index is
......@@ -216,7 +219,6 @@ class LTTBDownsampling(StuettNode):
array.attrs = data.attrs
return array
class Downsampling(StuettNode):
def __init__(self):
raise NotImplementedError()
......@@ -224,7 +226,6 @@ class Downsampling(StuettNode):
# classes depending on the user request
pass
class Spectrogram(StuettNode):
def __init__(
self,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment