To receive notifications about scheduled maintenance, please subscribe to the mailing-list gitlab-operations@sympa.ethz.ch. You can subscribe to the mailing-list at https://sympa.ethz.ch

Commit 39bf3d51 authored by matthmey's avatar matthmey
Browse files

towards zarr_freeze and updated arclink config

parent b24aef98
......@@ -6,6 +6,14 @@ optional = false
python-versions = "*"
version = "1.4.3"
[[package]]
category = "main"
description = "Draws ASCII trees."
name = "asciitree"
optional = false
python-versions = "*"
version = "0.3.3"
[[package]]
category = "dev"
description = "Atomic file writes."
......@@ -207,6 +215,18 @@ toolz = ">=0.7.4"
tornado = ">=5"
zict = ">=0.1.3"
[[package]]
category = "main"
description = "A python package that provides useful locks."
name = "fasteners"
optional = false
python-versions = "*"
version = "0.15"
[package.dependencies]
monotonic = ">=0.1"
six = "*"
[[package]]
category = "main"
description = "File-system specification"
......@@ -330,6 +350,14 @@ numpy = ">=1.11"
pyparsing = ">=2.0.1,<2.0.4 || >2.0.4,<2.1.2 || >2.1.2,<2.1.6 || >2.1.6"
python-dateutil = ">=2.1"
[[package]]
category = "main"
description = "An implementation of time.monotonic() for Python 2 & < 3.3"
name = "monotonic"
optional = false
python-versions = "*"
version = "1.5"
[[package]]
category = "dev"
description = "More routines for operating on iterables, beyond itertools"
......@@ -347,6 +375,20 @@ optional = false
python-versions = "*"
version = "0.6.2"
[[package]]
category = "main"
description = "A Python package providing buffer compression and transformation codecs for use in data storage and communication applications."
name = "numcodecs"
optional = false
python-versions = "*"
version = "0.6.4"
[package.dependencies]
numpy = ">=1.7"
[package.extras]
msgpack = ["msgpack"]
[[package]]
category = "main"
description = "Fast numerical expression evaluator for NumPy"
......@@ -759,6 +801,20 @@ version = "0.14.0"
numpy = ">=1.14"
pandas = ">=0.24"
[[package]]
category = "main"
description = "An implementation of chunked, compressed, N-dimensional arrays for Python."
name = "zarr"
optional = false
python-versions = "*"
version = "2.3.2"
[package.dependencies]
asciitree = "*"
fasteners = "*"
numcodecs = ">=0.6.2"
numpy = ">=1.7"
[[package]]
category = "main"
description = "Mutable mapping tools"
......@@ -788,7 +844,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pathlib2", "contextlib2", "unittest2"]
[metadata]
content-hash = "1d9445716aee36dfbf181cdaa7f4dcf1fe54822cf572c4a95640d56b75e4c293"
content-hash = "3976a3b0a911dcd5dfd5a9dafbab4dbe37df04dd167ec6bd92365642ea162b1a"
python-versions = "^3.7" # Compatible python versions must be declared here
[metadata.files]
......@@ -796,6 +852,9 @@ appdirs = [
{file = "appdirs-1.4.3-py2.py3-none-any.whl", hash = "sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"},
{file = "appdirs-1.4.3.tar.gz", hash = "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92"},
]
asciitree = [
{file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"},
]
atomicwrites = [
{file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"},
{file = "atomicwrites-1.3.0.tar.gz", hash = "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"},
......@@ -881,6 +940,10 @@ distributed = [
{file = "distributed-2.6.0-py3-none-any.whl", hash = "sha256:5f1082c158e976e23b05fa4d22566fec16c013c9bba12a428ea7be3fd47be4e0"},
{file = "distributed-2.6.0.tar.gz", hash = "sha256:30b0ca195ace1e39bdd278bf1ad257f7674b3e2b8e7a2a37ce7e2ade4aecccf3"},
]
fasteners = [
{file = "fasteners-0.15-py2.py3-none-any.whl", hash = "sha256:007e4d2b2d4a10093f67e932e5166722d2eab83b77724156e92ad013c6226574"},
{file = "fasteners-0.15.tar.gz", hash = "sha256:3a176da6b70df9bb88498e1a18a9e4a8579ed5b9141207762368a1017bf8f5ef"},
]
fsspec = [
{file = "fsspec-0.5.2.tar.gz", hash = "sha256:6531a5fa9ea6bf27a5180d225558a8a7aa5d7c3cbf7e8b146dd37ac699017937"},
]
......@@ -1014,6 +1077,10 @@ matplotlib = [
{file = "matplotlib-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4442ce720907f67a79d45de9ada47be81ce17e6c2f448b3c64765af93f6829c9"},
{file = "matplotlib-3.1.1.tar.gz", hash = "sha256:1febd22afe1489b13c6749ea059d392c03261b2950d1d45c17e3aed812080c93"},
]
monotonic = [
{file = "monotonic-1.5-py2.py3-none-any.whl", hash = "sha256:552a91f381532e33cbd07c6a2655a21908088962bb8fa7239ecbcc6ad1140cc7"},
{file = "monotonic-1.5.tar.gz", hash = "sha256:23953d55076df038541e648a53676fb24980f7a1be290cdda21300b3bc21dfb0"},
]
more-itertools = [
{file = "more-itertools-7.2.0.tar.gz", hash = "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832"},
{file = "more_itertools-7.2.0-py3-none-any.whl", hash = "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"},
......@@ -1041,6 +1108,9 @@ msgpack = [
{file = "msgpack-0.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:1904b7cb65342d0998b75908304a03cb004c63ef31e16c8c43fee6b989d7f0d7"},
{file = "msgpack-0.6.2.tar.gz", hash = "sha256:ea3c2f859346fcd55fc46e96885301d9c2f7a36d453f5d8f2967840efa1e1830"},
]
numcodecs = [
{file = "numcodecs-0.6.4.tar.gz", hash = "sha256:ef4843d5db4d074e607e9b85156835c10d006afc10e175bda62ff5412fca6e4d"},
]
numexpr = [
{file = "numexpr-2.7.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:49f97366ed8e0e970ae7f96d569d4aee6372dfe61951cd08b142edada2c459d7"},
{file = "numexpr-2.7.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:08bdae334855b19c55137b5a3c0d1bb1b05f23f7e6df24e47dfc8d5b3de4e10a"},
......@@ -1359,6 +1429,9 @@ xarray = [
{file = "xarray-0.14.0-py3-none-any.whl", hash = "sha256:9a4f97c6a7fdf9a6dd873ac679a86abfa1910263a85774d69bc3c0fa1e7967f5"},
{file = "xarray-0.14.0.tar.gz", hash = "sha256:a8b93e1b0af27fa7de199a2d36933f1f5acc9854783646b0f1b37fed9b4da091"},
]
zarr = [
{file = "zarr-2.3.2.tar.gz", hash = "sha256:c62d0158fb287151c978904935a177b3d2d318dea3057cfbeac8541915dfa105"},
]
zict = [
{file = "zict-1.0.0-py2.py3-none-any.whl", hash = "sha256:be8c7a24e3e78f871b72bfff16245105d1f0448606b1decdae054a14bfdf5996"},
{file = "zict-1.0.0.tar.gz", hash = "sha256:e34dd25ea97def518fb4c77f2c27078f3a7d6c965b0a3ac8fe5bdb0a8011a310"},
......
......@@ -28,6 +28,7 @@ toolz = "^0.10.0"
obspy = "^1.1.1"
appdirs = "^1.4.3"
obsplus = "^0.0.2"
zarr = "^2.3.2"
# Optional dependencies (extras)
......
......@@ -10,6 +10,8 @@ from obspy.clients.fdsn import Client
from obspy import UTCDateTime
from obsplus import obspy_to_array
import zarr
#TODO: revisit the following packages
import numpy as np
import pandas as pd
......@@ -36,8 +38,8 @@ class DataSource(StuettNode):
return requests
class Freezer(StuettNode):
def __init__(self):
pass
def __init__(self,store):
self.store = store
def configure(self,requests):
"""
......@@ -76,6 +78,10 @@ class Freezer(StuettNode):
#TODO: maybe a warning
return x
zarr_dataset = zarr.open(self.store, mode='r')
x_ds = x.to_dataset()
#TODO: check request start_time and load the data which is available, store the data which is not available
......@@ -98,45 +104,34 @@ class SeismicSource(DataSource):
if 'source' not in self.config:
self.config['source'] = None
# TODO: make fallback to permasense_vault
if use_arclink:
arclink_user = get_setting('arclink_user')
arclink_password = get_setting('arclink_password')
arclink = get_setting('arclink')
arclink_user = arclink['user']
arclink_password = arclink['password']
self.fdsn_client = Client(base_url='http://arclink.ethz.ch',user=arclink_user,password=arclink_password)
@dask.delayed
def __call__(self,request=None):
# logging.info('Loading seismic with fdsn')
print('loading with seismic')
# TODO: check if request contains settings
config = self.config.copy()
if request is not None:
config.update(request)
print('config',config)
if self.use_arclink:
# logging.info('Loading seismic with fdsn')
st = self.fdsn_client.get_waveforms(network='4D', station=config['station'], location='A',
channel=config['channel'], starttime=UTCDateTime(config['start_time']), endtime=UTCDateTime(config['end_time']), attach_response=True)
else: #20180914 is last full day available in permasense_vault
# logging.info('Loading seismic with fdsn')
st = self.get_obspy_stream(config['start_time'],config['end_time'],config['station'],config['channel'])
print(st)
x = obspy_to_array(st)
# print(x)
# change time coords from relative to absolute time
starttime = obspy.UTCDateTime(x.starttime.values).datetime
starttime = pd.to_datetime(starttime,utc=True)
# print('starttime',starttime)
# print(x['time'].values)
starttime = pd.to_datetime(starttime,utc=True)
timedeltas = pd.to_timedelta(x['time'].values,unit='seconds')
# print('timedeltas',timedeltas)
# print(type(starttime),type(timedeltas))
xt = starttime + timedeltas
# print(type(xt),xt)
x['time'] = pd.to_datetime(xt,utc=True)
# print('xtime',x['time'])
return x
......
......@@ -9,8 +9,8 @@ stations = ['MH36', 'MH44', 'MH48', 'MH52', 'MH54']
start_time = dt.datetime(2019,7,14,7,7,0,tzinfo=dt.timezone.utc)
end_time = dt.datetime(2019,7,14,7,7,7,tzinfo=dt.timezone.utc)
start_time = dt.datetime(2017,7,14,7,7,0,tzinfo=dt.timezone.utc)
end_time = dt.datetime(2017,7,14,7,7,7,tzinfo=dt.timezone.utc)
# start_time = dt.datetime(2017,7,14,7,7,0,tzinfo=dt.timezone.utc)
# end_time = dt.datetime(2017,7,14,7,7,7,tzinfo=dt.timezone.utc)
offset = dt.timedelta(days=1)
config = {'channel':channels[0],
......
......@@ -17,14 +17,14 @@ class TestSeismicSource(object):
@pytest.mark.slow
def test_call(self):
# first test without config
seismic_source = stuett.data.SeismicSource()
seismic_source = stuett.data.SeismicSource(use_arclink=True)
x = seismic_source(config)
x = x.compute()
assert x.mean() == -52.102427184466016
# with config
seismic_source = stuett.data.SeismicSource(config)
seismic_source = stuett.data.SeismicSource(config,use_arclink=True)
request = {'start_time':start_time+offset, 'end_time':end_time+offset}
x = seismic_source(request)
......@@ -32,3 +32,17 @@ class TestSeismicSource(object):
assert x.mean() == -91.36310679611651
# TODO: make test to compare start_times
# class TestFreezer(object):
# @pytest.mark.slow
# def test_seismic_source_freeze(self):
# # first test without config
# seismic_source = stuett.data.SeismicSource()
# # with config
# seismic_source = stuett.data.SeismicSource(config)
# freezer = stuett.data.Freezer()
# x = freezer(seismic_source)
......@@ -22,13 +22,11 @@ class TestMinMaxDownsampling(object):
x = x.compute()
print(da)
print(x)
# TODO: proper test
def test_seismic(self):
seismic_source = stuett.data.SeismicSource()
seismic_source = stuett.data.SeismicSource(use_arclink=True)
minmax_rate2 = stuett.data.MinMaxDownsampling(2)
x = seismic_source(config)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment