Merge branch '58-gitlab-pipeline-fails-to-build' into 'develop'

Resolve "gitlab pipeline fails to build"

See merge request !88

.gitlab-ci.yml

 test:
-   script:
-   - python3 --version
-   - java -version
-   - mvn -version
-   - python3 -m synpp config_gitlab.yml
+  script:
+    - source environment/activate.sh env
+    - rm -rf output cache
+    - mkdir output cache
+    - which conda
+    - conda list
+    - which python
+    - python --version
+    - python -u -v -m synpp config_gitlab.yml
 
 before_script:
-  # Anaconda
-  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - source "$HOME/miniconda/etc/profile.d/conda.sh"
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
-  - conda env create -f environment.yml
-  - conda activate switzerland
-
-  # Java
-  - wget https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.7_10.tar.gz -O java.tar.gz
-  - tar xf java.tar.gz
-  - export PATH=$HOME/jdk-11.0.7%2B10/bin:$PATH
-
-  # Maven
-  - wget http://mirror.easyname.ch/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -O maven.tar.gz
-  - tar xf maven.tar.gz
-  - export PATH=$HOME/apache-maven-3.6.3/bin:$PATH
-
-  # eqasim-python
-  - git clone https://github.com/eqasim-org/eqasim-python.git $HOME/eqasim-python
-  - sh -c "cd $HOME/eqasim-python; git pull origin master; python3 setup.py install"
+  - rm -rf env
+  - sh environment/setup.sh env
+  - sh environment/activate.sh env
 
 cache:
-    key: environment_v2
-    paths:
-        - env/
+  key: environment_v2
+  paths:
+    - env/

CHANGELOG.md

 **3.0.0**
+- Update environment set-up on servers
 - Refactor IPU fitting
 - Impute canton id directly from shapefile
 - Added config elements for generating flowchart

config_gitlab.yml

 # General pipeline settings
-working_directory: /nas/ivtmatsim/scenarios/switzerland/data
-flowchart_path: /path/to/flowchart.json
+working_directory: cache
 dryrun: false
 
 # Requested stages

environment/activate.sh

@@ -6,8 +6,7 @@ if [ ! -f ${environment_directory}/miniconda_installed ]; then
     echo "Miniconda is not installed properly."
     exit 1
 else
-    PATH=${environment_directory}/miniconda3/bin:$PATH
-
+    source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
     echo "Testing Miniconda ..."
     conda -V
 fi
@@ -16,10 +15,12 @@ if [ ! -f ${environment_directory}/python_installed ]; then
     echo "Python environment is not installed properly."
     exit 1
 else
-    source activate ${environment_directory}/venv
+
+    source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
+    conda activate venv
 
     echo "Testing Python ..."
-    python3 --version
+    python --version
 fi
 
 if [ ! -f ${environment_directory}/jdk_installed ]; then

environment/setup.sh

 #!/bin/bash
 set -e
 
-# Define Miniconda3
+# Define Miniconda
 miniconda_version="4.6.14"
 miniconda_url="https://repo.anaconda.com/miniconda/Miniconda3-${miniconda_version}-Linux-x86_64.sh"
 miniconda_md5="718259965f234088d785cad1fbd7de03"
 
-python_version="3.6"
-
-jdk_version="8u212"
-jdk_url="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u212-b03/OpenJDK8U-jdk_x64_linux_hotspot_8u212b03.tar.gz"
-jdk_sha256="dd28d6d2cde2b931caf94ac2422a2ad082ea62f0beee3bf7057317c53093de93"
+jdk_version="11.0.7"
+jdk_url="https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.7_10.tar.gz"
+jdk_sha256="ee60304d782c9d5654bf1a6b3f38c683921c1711045e1db94525a51b7024a2ca"
 
 maven_version="3.6.3"
-maven_url="http://mirror.easyname.ch/apache/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz"
+maven_url="https://downloads.apache.org/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz"
 maven_sha512="c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0"
 
+
 # Define Python requirements
-python_requirements=$(cat <<EOF
-tqdm==4.30.0
-pyyaml==3.13
-pandas==0.24.1
-pyproj==1.9.6
-geopandas==0.4.0
-scikit-learn==0.20.2
-requests==2.21.0
-rtree==0.8.3
-xlrd==1.2.0
-jinja2==2.9.5
+environment_yaml=$(cat <<EOF
+name: venv
+
+channels:
+  - defaults
+
+dependencies:
+  - matplotlib=3.1.3
+  - pandas=1.0.3
+  - scipy=1.4.1
+  - numpy=1.18.1
+  - geopandas=0.6.1
+  - numba=0.49.0
+  - palettable=3.3.0
+  - scikit-learn=0.22.1
+  - shapely=1.7.0
+  - tqdm=4.46.0
+  - pytables=3.6.1
+  - xlrd=1.2.0
+  - pip=20.0.2
+
+  - pip:
+    - simpledbf==0.2.6
+    - synpp==1.3.1
+    - python-Levenshtein==0.12.0
+
+  # For testing
+  - pytest==5.4.2
+  - xlwt==1.3.0
+  - pysal==1.14.4.post1
 EOF
 )
 
+
 # Miniconda update script to avoid too long paths in interpreter path
 miniconda_update_script=$(cat <<EOF
 import sys
@@ -38,7 +57,7 @@ import re
 
 with open(sys.argv[1]) as f:
     content = f.read()
-    content = re.sub(r'#!(.+)/miniconda3/bin/python', '#!/usr/bin/env python', content)
+    content = re.sub(r'#!(.+)/miniconda/bin/python', '#!/usr/bin/env python', content)
 
 with open(sys.argv[1], "w+") as f:
     f.write(content)
@@ -63,7 +82,7 @@ cd ${environment_directory}
 if [ "$(md5sum miniconda.sh)" == "${miniconda_md5}  miniconda.sh" ]; then
     echo "Miniconda 3 ${miniconda_version} already downloaded."
 else
-    echo "Downloading Miniconda3 ${miniconda_version} ..."
+    echo "Downloading Miniconda ${miniconda_version} ..."
     rm -rf miniconda_installed
     rm -rf python_installed
     curl -o miniconda.sh ${miniconda_url}
@@ -91,19 +110,22 @@ fi
 
 # III.1) Install Miniconda
 if [ -f miniconda_installed ]; then
-    echo "Miniconda3 ${miniconda_version} already installed."
+    echo "Miniconda ${miniconda_version} already installed."
 else
-    echo "Installing Miniconda3 ${miniconda_version} ..."
+    echo "Installing Miniconda ${miniconda_version} ..."
 
-    rm -rf miniconda3
-    sh miniconda.sh -b -u -p miniconda3
+    rm -rf miniconda
+    sh miniconda.sh -b -u -p miniconda
 
     cat <<< "${miniconda_update_script}" > fix_conda.py
 
-    PATH=${environment_directory}/miniconda3/bin:$PATH
-    python fix_conda.py miniconda3/bin/conda
-    python fix_conda.py miniconda3/bin/conda-env
-    conda update -y conda
+    PATH=${environment_directory}/miniconda/bin:$PATH
+    python fix_conda.py miniconda/bin/conda
+    python fix_conda.py miniconda/bin/conda-env
+
+    source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
+    conda config --set always_yes yes --set changeps1 no
+    conda update -q conda
 
     touch miniconda_installed
 fi
@@ -114,8 +136,10 @@ if [ -f python_installed ]; then
 else
     echo "Setting up Python environment ..."
 
-    cat <<< "${python_requirements}" > requirements.txt
-    conda create -p venv python=${python_version} --no-default-packages --channel conda-forge --file requirements.txt -y
+    cat <<< "${environment_yaml}" > environment.yml
+
+    source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
+    conda env create -f environment.yml
 
     touch python_installed
 fi