To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit b22376df authored by matthmey's avatar matthmey
Browse files

black

parent d397d46b
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,13 +19,13 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,13 +19,13 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
from pandas import to_datetime, to_timedelta from pandas import to_datetime, to_timedelta
from zarr import DirectoryStore, ABSStore from zarr import DirectoryStore, ABSStore
import dask import dask
import pandas as pd import pandas as pd
import io, codecs import io, codecs
# TODO: make it a proper decorator with arguments etc # TODO: make it a proper decorator with arguments etc
...@@ -46,15 +46,16 @@ def dat(x): ...@@ -46,15 +46,16 @@ def dat(x):
return dask.delayed(x) return dask.delayed(x)
def to_csv_with_store(store,filename,dataframe): def to_csv_with_store(store, filename, dataframe):
StreamWriter = codecs.getwriter('utf-8') StreamWriter = codecs.getwriter("utf-8")
bytes_buffer = io.BytesIO() bytes_buffer = io.BytesIO()
string_buffer = StreamWriter(bytes_buffer) string_buffer = StreamWriter(bytes_buffer)
dataframe.to_csv(string_buffer,index=False) dataframe.to_csv(string_buffer, index=False)
store[filename] = bytes_buffer.getvalue() store[filename] = bytes_buffer.getvalue()
def read_csv_with_store(store,filename):
def read_csv_with_store(store, filename):
bytes_buffer = io.BytesIO(store[str(filename)]) bytes_buffer = io.BytesIO(store[str(filename)])
StreamReader = codecs.getreader('utf-8') StreamReader = codecs.getreader("utf-8")
string_buffer = StreamReader(bytes_buffer) string_buffer = StreamReader(bytes_buffer)
return pd.read_csv(string_buffer) return pd.read_csv(string_buffer)
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import dask import dask
from dask.core import get_dependencies, flatten from dask.core import get_dependencies, flatten
...@@ -297,7 +297,7 @@ def configuration(delayed, request, keys=None, default_merge=None): ...@@ -297,7 +297,7 @@ def configuration(delayed, request, keys=None, default_merge=None):
out[k] = out[k][:2] + (input_requests[k],) out[k] = out[k][:2] + (input_requests[k],)
# convert to delayed object # convert to delayed object
from dask.delayed import Delayed # TODO: move somewhere else from dask.delayed import Delayed # TODO: move somewhere else
in_keys = list(flatten(keys)) in_keys = list(flatten(keys))
# print(in_keys) # print(in_keys)
......
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import numpy as np import numpy as np
import pandas as pd import pandas as pd
......
This diff is collapsed.
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
from ..global_config import get_setting from ..global_config import get_setting
......
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import yaml import yaml
from os.path import dirname, abspath, join from os.path import dirname, abspath, join
......
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import datetime as dt import datetime as dt
......
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import stuett import stuett
from pathlib import Path from pathlib import Path
......
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import stuett import stuett
import datetime as dt import datetime as dt
......
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import stuett import stuett
from tests.stuett.sample_data import * from tests.stuett.sample_data import *
...@@ -217,28 +217,29 @@ def test_csv(): ...@@ -217,28 +217,29 @@ def test_csv():
assert len(x) == 8760 assert len(x) == 8760
assert x.mean() == 0.16201109532441677 assert x.mean() == 0.16201109532441677
# Test with store # Test with store
directory = Path(test_data_dir).joinpath( directory = Path(test_data_dir).joinpath("timeseries")
"timeseries")
store = stuett.DirectoryStore(directory) store = stuett.DirectoryStore(directory)
filename = "MH30_temperature_rock_2017.csv" filename = "MH30_temperature_rock_2017.csv"
node = stuett.data.CsvSource(filename=filename,store=store) node = stuett.data.CsvSource(filename=filename, store=store)
x = node() x = node()
assert len(x) == 8760 assert len(x) == 8760
assert x.mean() == 0.16201109532441677 assert x.mean() == 0.16201109532441677
# TODO: test with start and end time # TODO: test with start and end time
test_csv() test_csv()
def test_annotations(): def test_annotations():
filename = Path(test_data_dir).joinpath("annotations", "boundingbox_timeseries.csv") filename = Path(test_data_dir).joinpath("annotations", "boundingbox_timeseries.csv")
node = stuett.data.BoundingBoxAnnotation(filename) node = stuett.data.BoundingBoxAnnotation(filename)
targets = node() targets = node()
assert targets[0] == 'no_visibility' assert targets[0] == "no_visibility"
filename = Path(test_data_dir).joinpath("annotations", "boundingbox_images.csv") filename = Path(test_data_dir).joinpath("annotations", "boundingbox_images.csv")
node = stuett.data.BoundingBoxAnnotation(filename) node = stuett.data.BoundingBoxAnnotation(filename)
...@@ -248,8 +249,9 @@ def test_annotations(): ...@@ -248,8 +249,9 @@ def test_annotations():
targets = targets.swap_dims({"index": "start_time"}) targets = targets.swap_dims({"index": "start_time"})
targets = targets.sortby("start_time") targets = targets.sortby("start_time")
assert targets.sel(start_time='2017-08-04T08:12:11') == 'mountaineer' assert targets.sel(start_time="2017-08-04T08:12:11") == "mountaineer"
assert targets['start_y'][1] == 16 assert targets["start_y"][1] == 16
# test_annotations() # test_annotations()
...@@ -262,7 +264,7 @@ def test_datasets(): ...@@ -262,7 +264,7 @@ def test_datasets():
"timeseries", "MH30_temperature_rock_2017.csv" "timeseries", "MH30_temperature_rock_2017.csv"
) )
data = stuett.data.CsvSource(filename) data = stuett.data.CsvSource(filename)
dataset = stuett.data.SegmentedDataset( dataset = stuett.data.SegmentedDataset(
data, data,
label, label,
......
'''MIT License """MIT License
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich), Matthias Meyer
...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE ...@@ -19,7 +19,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.''' SOFTWARE."""
import stuett import stuett
import datetime as dt import datetime as dt
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment