Commit 2e775481 authored by matthmey's avatar matthmey
Browse files

Initial commit

parents
Debug
.settings
*.hex
*.o
*.su
*.list
*.map
*.elf
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
*.swp
# PyBuilder
target/
# Rep Ignores
.vscode/
data/*
!data/ReadMe.md
!data/microseismic_test_events.csv
!data/microseismic_train_events.csv
\ No newline at end of file
# Code for IPSN2019 submission
Please see the respective subfolders ReadMe for more information
## mhcnn
Contains the python code to train the convolutional neural network and generate parameter files for MCU implementation.
## mhmcu
Contains the C Code which implements the trained convolutional neural network on a MCU. The parameter files are pregenerated. The ReadMe in *mhcnn* explains how to generate your own parameter files. The implementation has been developed with Atollic TrueSTUDIO for STM32 (Version 9.0.1).
## tools
Helper tools to evaluate the MCU implementation.
## data
Please follow the ReadMe in this folder to download the data correctly.
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich).
All rights reserved.
\ No newline at end of file
# Matterhorn dataset
Download and extract the dataset from https://zenodo.org/record/1320835 or via DOI: 10.5281/zenodo.1320835
Place the *dataset*, *event_dataset* and *secondary_data* folders into this folder.
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
# Translations
*.mo
*.pot
# Django stuff:
*.log
# Sphinx documentation
docs/_build/
*.swp
# PyBuilder
target/
# Rep Ignores
.vscode/
data/
tmp/*
!tmp/.gitkeep
results/*
!results/runs/
results/runs/*
!results/runs/final/
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
"""
Copyright (c) 2019, Swiss Federal Institute of Technology (ETH Zurich)
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
import keras
from keras import backend as K
from keras.models import Sequential
import time
class INQ_Model(Sequential):
# def __init__(self, steps):
# self.steps = steps
# super(INQ_Model, self)
def partition_weights(self, step, strategy, seed=None):
ii = 1
print("Partitioning weights...")
for layer in self.layers:
print("Processing layer:", ii, "of", len(self.layers))
ii += 1
if hasattr(layer, 'is_inq_layer'):
t0 = time.time()
layer.partition_weights(step, strategy, seed=seed)
t1 = time.time()
print('Took %f seconds' %(t1 - t0))
else:
pass
def quantize(self, bits):
ii = 1
print("Quantizing weights...")
for layer in self.layers:
print("Processing layer", ii, "of", len(self.layers))
ii += 1
if hasattr(layer, 'is_inq_layer'):
t0 = time.time()
layer.quantize(bits)
t1 = time.time()
print('Took %f seconds' %(t1 - t0))
else:
pass
def print_kernels(self):
ii = 0
print('Printing all kernels/weights:')
for layer in self.layers:
ii += 1
if hasattr(layer, 'is_inq_layer'):
#layer.print_kernel()
print('Layer {}'.format(ii))
print(layer.get_weights()[0])
def print_biases(self):
ii = 0
print('Printing all biases:')
for layer in self.layers:
ii += 1
if hasattr(layer, 'is_inq_layer'):
print('Layer {}'.format(ii))
print(layer.get_weights()[1])
\ No newline at end of file
# mhcnn repository
## Requirements
Python and additional modules are required. Using anaconda you can install the requirements by executing the following command from this directory and activating the newly created environment
```
conda env create -f environment.yml
source activate mhcnn
```
Besides the training data, the scripts need additional disk space to store the preprocessed data. This can more than 60GB. Make sure you have enough disk space available. Alternativly, you can specify a location with enough storage. See **Changing paths** section.
## Getting Started
Install the requirements.
Download and extract the dataset. See ../data/ReadMe.md for more information.
To test all the pretrained networks run from the *mhcnn/* directory:
```
python run_experiment.py -id final -r test
python run_experiment.py -id final -r inq
```
This will load and preprocess all the test set files for each experiment, which takes some time the first time (unfortunately without progress bar at the moment). Then, it will display the test results for each classifier.
If the script fails. Run the script to test only one of the classifiers (e.g. microseismicCNN) to check functionality:
```
python test.py -m microseismic_events
```
To train all classifiers you can run
```
python run_experiment.py
```
To train and test an individual classifier run
```
python train.py -m classifier_name
python test.py -m classifier_name
```
`classifier_name` should be 'microseismic_events'
## Prepare for MCU implementation
If you retrain the network the CNN must be converted to be used on the MCU. First, it must be quantized with INQ. Then, it must be converted to C code.
### INQ
The train.py script copies the generated files to the results/microseismic_events/test/ folder. The INQ.py script takes these results and quantizes them. Run the script as follows
```
python INQ.py -m microseismic_events
```
### Convert to C code
The MCU implementation requires the weights of the INQ model. These are converted with the following script
```
python convert_to_c_header.py -m microseismic_events
```
The scripts creates a new file *model_weights.h* in the tmp folder. This file can then be copied to mhmcu/Inc/.
## Changing paths
If you run the script from a different folder than where your data is stored you need to update some paths.
Open the file *toolbox.py* change the variable `data_dir` at the beginning of the file to the path where you extracted the datasets (the folder containing dataset/ and event_dataset/). In the same file change the `tmp_dir` variable to a path with enough storage to create temporary training/testing files (60GB or more, if all classifiers are to be trained).
\ No newline at end of file
"""
Script for the paper "Systematic Identification of External Influences in Multi-Year Micro-Seismic Recordings Using Convolutional Neural Networks"
Custom keras callback function to save the best instance
Copyright (c) 2018, Swiss Federal Institute of Technology (ETH Zurich)
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
from aednn.toolbox import evaluate
import keras
class MetricPlot(keras.callbacks.Callback):
def __init__(self,filename,names=['loss']):
self.filename = filename
self.names = names
self.fig, self.ax = plt.subplots(len(self.names),1)
def on_train_begin(self, logs={}):
self.logs = {}
def on_batch_end(self, batch, logs={}):
pass
def append_log(self,name,logs):
if name not in self.logs:
self.logs[name] = []
self.logs[name].append(logs.get(name))
def on_epoch_end(self, batch, logs={}):
plt.figure(self.fig.number)
for idx,name in enumerate(self.names):
self.append_log(name,logs)
self.append_log('val_' + name,logs)
self.ax[idx].plot(self.logs[name],'r')
self.ax[idx].plot(self.logs['val_'+name],'b')
self.ax[idx].set_title('%s (train:red val:blue)'%name)
plt.savefig(self.filename,dpi=300)
class SaveBest(keras.callbacks.Callback):
def __init__(self,filename,X_val,y_val,indices=None):
"""Performs evaluation and saves the model if it exceeds the best value
Arguments:
keras {[type]} -- [description]
filename {[type]} -- [description]
X_val {[type]} -- [description]
y_val {[type]} -- [description]
indices {list or None} -- List of indices. If None the scores for all labels will be compared, else only labels given by the indices
"""
self.X_val = X_val
self.y_val = y_val
self.filename = filename
self.best_eval = 0
if indices is None:
self.indices = np.arange(y_val.shape[1])
else:
self.indices = np.array(indices)
def on_train_begin(self, logs={}):
self.logs = {}
def on_batch_end(self, batch, logs={}):
pass
def on_epoch_end(self, batch, logs={}):
y_pred = self.model.predict(self.X_val,batch_size=16)
eval_results, thresholds = evaluate(self.y_val[:,self.indices],y_pred[:,self.indices],eval_best='f1score',return_thresholds=True)
if eval_results['f1score'] > self.best_eval:
self.best_eval = eval_results['f1score']
self.model.save(self.filename, overwrite=True)
print(eval_results['f1score'],thresholds,self.best_eval)
class myCallbackPlot(keras.callbacks.Callback):
def __init__(self, filename,names=['loss']):
self.filename = filename
self.names = names
self.fig, self.ax = plt.subplots(len(self.names), 1)
def on_train_begin(self, logs={}):
self.logs = {}
def on_batch_end(self, batch, logs={}):
pass
def append_log(self, name, logs):
if name not in self.logs:
self.logs[name] = []
self.logs[name].append(logs.get(name))
def on_epoch_end(self, batch, logs={}):
plt.figure(self.fig.number)
for idx, name in enumerate(self.names):
self.append_log(name, logs)
self.append_log('val_' + name, logs)
self.ax[idx].plot(self.logs[name], 'r')
self.ax[idx].plot(self.logs['val_'+ name],'b')
self.ax[idx].set_title('%s (train:red val:blue)'%name)
plt.savefig(self.filename, fdpi=300, format='eps')
This diff is collapsed.
"""
Script for the paper "Systematic Identification of External Influences in Multi-Year Micro-Seismic Recordings Using Convolutional Neural Networks"
Custom metrics for evaluation and for use with keras
Copyright (c) 2018, Swiss Federal Institute of Technology (ETH Zurich)
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
import keras
from keras import backend as K
from keras.utils.generic_utils import get_custom_objects
def precision(y_true, y_pred):
"""Precision metric.
Only computes a batch-wise average of precision.
Computes the precision, a metric for multi-label classification of
how many selected items are relevant.
"""
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
return precision
def recall(y_true, y_pred):
"""Recall metric.
Only computes a batch-wise average of recall.
Computes the recall, a metric for multi-label classification of
how many relevant items are selected.
"""
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
def specificity(y_true, y_pred):
"""specificity metric.
Only computes a batch-wise average of specificity.
Computes the specificity
"""
y_pred_inverse = - K.round(K.clip(y_pred, 0, 1)) + 1
y_true_inv = -y_true + 1
true_negatives = K.sum(K.round(K.clip(y_true_inv * y_pred_inverse, 0, 1)))
possible_negatives = K.sum(K.round(K.clip(y_true_inv, 0, 1)))
specificity = true_negatives / (possible_negatives + K.epsilon())
return specificity
def f1score(y_true, y_pred):
"""F1 score metric.
Only computes a batch-wise average of F1 score.
Computes the F1 score
"""
prec = precision(y_true,y_pred) + K.epsilon()
rec = recall(y_true,y_pred) + K.epsilon()
return 2 * prec*rec / (prec + rec)
get_custom_objects().update({"precision": precision})
get_custom_objects().update({"recall": recall})
get_custom_objects().update({"specificity": specificity})
get_custom_objects().update({"f1score": f1score})
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment