Commit 932054cd authored by matthmey's avatar matthmey

enable zarr storage for more nodes

parent 6605aa7d
......@@ -141,7 +141,7 @@ description = "Parallel PyData with Task Scheduling"
name = "dask"
optional = false
python-versions = ">=3.6"
version = "2.8.0"
version = "2.8.1"
[package.dependencies]
[package.dependencies.PyYaml]
......@@ -204,7 +204,7 @@ marker = "extra == \"complete\""
name = "distributed"
optional = false
python-versions = ">=3.6"
version = "2.8.0"
version = "2.8.1"
[package.dependencies]
click = ">=6.6"
......@@ -558,8 +558,8 @@ description = "Appendable key-value storage"
marker = "extra == \"complete\""
name = "partd"
optional = false
python-versions = "*"
version = "1.0.0"
python-versions = ">=3.5"
version = "1.1.0"
[package.dependencies]
locket = "*"
......@@ -855,6 +855,35 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.10.0"
[[package]]
category = "main"
description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration"
name = "torch"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "1.3.1"
[package.dependencies]
future = "*"
numpy = "*"
[[package]]
category = "main"
description = "image and video datasets and models for torch deep learning"
name = "torchvision"
optional = false
python-versions = "*"
version = "0.4.2"
[package.dependencies]
numpy = "*"
pillow = ">=4.1.1"
six = "*"
torch = "1.3.1"
[package.extras]
scipy = ["scipy"]
[[package]]
category = "main"
description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed."
......@@ -960,7 +989,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pathlib2", "contextlib2", "unittest2"]
[metadata]
content-hash = "0f3e0df37d1c21229e8ebab1f12f22174ae9a3b3686d0767960f4058f808f018"
content-hash = "1b36ce916a66d5e8985eb9590c11d79b454bd45e6cb4e17755b03c973093565c"
python-versions = "^3.7" # Compatible python versions must be declared here
[metadata.files]
......@@ -1045,16 +1074,16 @@ cycler = [
{file = "cycler-0.10.0.tar.gz", hash = "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8"},
]
dask = [
{file = "dask-2.8.0-py3-none-any.whl", hash = "sha256:d3cf6f11abafb3087337f410d34977c1a773fcae51667e74df5044884fd791f6"},
{file = "dask-2.8.0.tar.gz", hash = "sha256:000f1d8cea21e73d4691718d9224903e9ba37fbbe756c8e7d11d4067ef9e0609"},
{file = "dask-2.8.1-py3-none-any.whl", hash = "sha256:78b8a7187ba613ce6ca72d6b14229be72a732d432415c18265c71f00f4927d63"},
{file = "dask-2.8.1.tar.gz", hash = "sha256:1232925dcc197290aec7ca39f58bb94799c6980220831d3db3eb14d96a8f5b84"},
]
decorator = [
{file = "decorator-4.4.1-py2.py3-none-any.whl", hash = "sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"},
{file = "decorator-4.4.1.tar.gz", hash = "sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce"},
]
distributed = [
{file = "distributed-2.8.0-py3-none-any.whl", hash = "sha256:87c29bc33613b653e703d3214bd6b3a3fce677a51b8482b71173de29e6942cea"},
{file = "distributed-2.8.0.tar.gz", hash = "sha256:37f8a89bb499b7858a2396e3fdd2e5997dece543725d3791ce239d960a647710"},
{file = "distributed-2.8.1-py3-none-any.whl", hash = "sha256:b740c0e24cb287be2edc1eced674fc0a037f381c2f099a0c409b1dda14ab6e35"},
{file = "distributed-2.8.1.tar.gz", hash = "sha256:146ee0b75878a07020f8a37a4533aef5ed15a500d2f0ce9f021d6b09b1f3d9ed"},
]
entrypoints = [
{file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"},
......@@ -1387,8 +1416,8 @@ pandas = [
{file = "pandas-0.25.3.tar.gz", hash = "sha256:52da74df8a9c9a103af0a72c9d5fdc8e0183a90884278db7f386b5692a2220a4"},
]
partd = [
{file = "partd-1.0.0-py2.py3-none-any.whl", hash = "sha256:f278ded3a62560db4a0d1529664fedc440585c520407a53b071fdbfb043187b9"},
{file = "partd-1.0.0.tar.gz", hash = "sha256:54fd91bc3b9c38159c790cd16950dbca6b019a2ead4c51dee4f9efc884f8ce0e"},
{file = "partd-1.1.0-py3-none-any.whl", hash = "sha256:7a491cf254e5ab09e9e6a40d80195e5e0e5e169115bfb8287225cb0c207536d2"},
{file = "partd-1.1.0.tar.gz", hash = "sha256:6e258bf0810701407ad1410d63d1a15cfd7b773fd9efe555dac6bb82cc8832b0"},
]
pathspec = [
{file = "pathspec-0.6.0.tar.gz", hash = "sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c"},
......@@ -1605,6 +1634,28 @@ toml = [
toolz = [
{file = "toolz-0.10.0.tar.gz", hash = "sha256:08fdd5ef7c96480ad11c12d472de21acd32359996f69a5259299b540feba4560"},
]
torch = [
{file = "torch-1.3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:d8e1d904a6193ed14a4fed220b00503b2baa576e71471286d1ebba899c851fae"},
{file = "torch-1.3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:b6f01d851d1c5989d4a99b50ae0187762b15b7718dcd1a33704b665daa2402f9"},
{file = "torch-1.3.1-cp27-none-macosx_10_7_x86_64.whl", hash = "sha256:31062923ac2e60eac676f6a0ae14702b051c158bbcf7f440eaba266b0defa197"},
{file = "torch-1.3.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:458f1d87e5b7064b2c39e36675d84e163be3143dd2fc806057b7878880c461bc"},
{file = "torch-1.3.1-cp35-none-macosx_10_6_x86_64.whl", hash = "sha256:3b05233481b51bb636cee63dc761bb7f602e198178782ff4159d385d1759608b"},
{file = "torch-1.3.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0cec2e13a2e95c24c34f17d437f354ee2a40902e8d515a524556b350e12555dd"},
{file = "torch-1.3.1-cp36-none-macosx_10_7_x86_64.whl", hash = "sha256:77fd8866c0bf529861ffd850a5dada2190a8d9c5167719fb0cfa89163e23b143"},
{file = "torch-1.3.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:72a1c85bffd2154f085bc0a1d378d8a54e55a57d49664b874fe7c949022bf071"},
{file = "torch-1.3.1-cp37-none-macosx_10_7_x86_64.whl", hash = "sha256:134e8291a97151b1ffeea09cb9ddde5238beb4e6d9dfb66657143d6990bfb865"},
]
torchvision = [
{file = "torchvision-0.4.2-cp27-cp27m-macosx_10_7_x86_64.whl", hash = "sha256:dda25ce304978bba19e6543f7dcfee4f37d2f128ec83d4ab0c7e8f991d64865f"},
{file = "torchvision-0.4.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:8ff715c2323d9eca89126824ebfa74b282a95d6f64a4743fbe9b738d2de21c77"},
{file = "torchvision-0.4.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1ad7593d94f6612ccb84a59467f0d10cdc213fb3e2bb91f1e773eb844787fa4c"},
{file = "torchvision-0.4.2-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:2553405b9afe3cedb410873b9877eb18b1526f8b01cb7c2747e51b69a936e0b5"},
{file = "torchvision-0.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:0f8245d6378acc86917f58492675f93df5279abae8bc5f832e3510722191f6c9"},
{file = "torchvision-0.4.2-cp36-cp36m-macosx_10_7_x86_64.whl", hash = "sha256:7a458330e4efcd66f9f70127ab21fcf8cfea84acda8e707322fd2843aa6dd396"},
{file = "torchvision-0.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:66deba9c577e36f4f071decdd894bf7ba794ac133dae64b3fd02fc3f0c6b989d"},
{file = "torchvision-0.4.2-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:dca4aadc12a123730957b501f9c5c2870d2f6727a2c28552cb7907b68b0ea10c"},
{file = "torchvision-0.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:276a385f2f5fe484bf08467b5d081d9144b97eb458ba5b4a11e4640389e53149"},
]
tornado = [
{file = "tornado-6.0.3-cp35-cp35m-win32.whl", hash = "sha256:c9399267c926a4e7c418baa5cbe91c7d1cf362d505a1ef898fde44a07c9dd8a5"},
{file = "tornado-6.0.3-cp35-cp35m-win_amd64.whl", hash = "sha256:398e0d35e086ba38a0427c3b37f4337327231942e731edaa6e9fd1865bbd6f60"},
......
......@@ -34,6 +34,8 @@ pillow = "^6.2.1"
xarray-extras = "^0.4.2"
lttb = "^0.2.0"
pyarrow = "^0.15.1"
torch = "^1.3.1"
torchvision = "^0.4.2"
# Optional dependencies (extras)
......
......@@ -20,10 +20,12 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.'''
from pandas import to_timedelta, to_timedelta
from pandas import to_datetime, to_timedelta
from zarr import DirectoryStore, ABSStore
import dask
import pandas as pd
import io, codecs
# TODO: make it a proper decorator with arguments etc
def dat(x):
......@@ -41,3 +43,17 @@ def dat(x):
"""
return dask.delayed(x)
def to_csv_with_store(store,filename,dataframe):
StreamWriter = codecs.getwriter('utf-8')
bytes_buffer = io.BytesIO()
string_buffer = StreamWriter(bytes_buffer)
dataframe.to_csv(string_buffer,index=False)
store[filename] = bytes_buffer.getvalue()
def read_csv_with_store(store,filename):
bytes_buffer = io.BytesIO(store[str(filename)])
StreamReader = codecs.getreader('utf-8')
string_buffer = StreamReader(bytes_buffer)
return pd.read_csv(string_buffer)
......@@ -296,7 +296,7 @@ def configuration(delayed, request, keys=None, default_merge=None):
out[k] = out[k][:2] + (input_requests[k],)
# convert to delayed object
from dask.delayed import Delayed
from dask.delayed import Delayed # TODO: move somewhere else
in_keys = list(flatten(keys))
# print(in_keys)
......
This diff is collapsed.
......@@ -261,11 +261,15 @@ def test_datasets():
"timeseries", "MH30_temperature_rock_2017.csv"
)
data = stuett.data.CsvSource(filename)
dataset = stuett.data.SegmentedDataset(
data,
label,
dataset_slice={"time": slice("2017-08-01", "2017-08-03")},
batch_dims={"time": pd.to_timedelta(24, "m")},
)
# test_datasets()
x = dataset[0]
test_datasets()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment