Commit 891aa595 authored by tchervec's avatar tchervec
Browse files

Resolve "gitlab pipeline fails to build"

parent 2d137b1f
test: test:
script: script:
- python3 --version - source environment/activate.sh env
- java -version - rm -rf output cache
- mvn -version - mkdir output cache
- python3 -m synpp config_gitlab.yml - which conda
- conda list
- which python
- python --version
- python -u -v -m synpp config_gitlab.yml
before_script: before_script:
# Anaconda - rm -rf env
- wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - sh environment/setup.sh env
- bash miniconda.sh -b -p $HOME/miniconda - sh environment/activate.sh env
- source "$HOME/miniconda/etc/profile.d/conda.sh"
- conda config --set always_yes yes --set changeps1 no
- conda update -q conda
- conda env create -f environment.yml
- conda activate switzerland
# Java
- wget https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.7_10.tar.gz -O java.tar.gz
- tar xf java.tar.gz
- export PATH=$HOME/jdk-11.0.7%2B10/bin:$PATH
# Maven
- wget http://mirror.easyname.ch/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz -O maven.tar.gz
- tar xf maven.tar.gz
- export PATH=$HOME/apache-maven-3.6.3/bin:$PATH
# eqasim-python
- git clone https://github.com/eqasim-org/eqasim-python.git $HOME/eqasim-python
- sh -c "cd $HOME/eqasim-python; git pull origin master; python3 setup.py install"
cache: cache:
key: environment_v2 key: environment_v2
paths: paths:
- env/ - env/
**3.0.0** **3.0.0**
- Update environment set-up on servers
- Refactor IPU fitting - Refactor IPU fitting
- Impute canton id directly from shapefile - Impute canton id directly from shapefile
- Added config elements for generating flowchart - Added config elements for generating flowchart
......
# General pipeline settings # General pipeline settings
working_directory: /nas/ivtmatsim/scenarios/switzerland/data working_directory: cache
flowchart_path: /path/to/flowchart.json
dryrun: false dryrun: false
# Requested stages # Requested stages
......
...@@ -6,8 +6,7 @@ if [ ! -f ${environment_directory}/miniconda_installed ]; then ...@@ -6,8 +6,7 @@ if [ ! -f ${environment_directory}/miniconda_installed ]; then
echo "Miniconda is not installed properly." echo "Miniconda is not installed properly."
exit 1 exit 1
else else
PATH=${environment_directory}/miniconda3/bin:$PATH source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
echo "Testing Miniconda ..." echo "Testing Miniconda ..."
conda -V conda -V
fi fi
...@@ -16,10 +15,12 @@ if [ ! -f ${environment_directory}/python_installed ]; then ...@@ -16,10 +15,12 @@ if [ ! -f ${environment_directory}/python_installed ]; then
echo "Python environment is not installed properly." echo "Python environment is not installed properly."
exit 1 exit 1
else else
source activate ${environment_directory}/venv
source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
conda activate venv
echo "Testing Python ..." echo "Testing Python ..."
python3 --version python --version
fi fi
if [ ! -f ${environment_directory}/jdk_installed ]; then if [ ! -f ${environment_directory}/jdk_installed ]; then
......
#!/bin/bash #!/bin/bash
set -e set -e
# Define Miniconda3 # Define Miniconda
miniconda_version="4.6.14" miniconda_version="4.6.14"
miniconda_url="https://repo.anaconda.com/miniconda/Miniconda3-${miniconda_version}-Linux-x86_64.sh" miniconda_url="https://repo.anaconda.com/miniconda/Miniconda3-${miniconda_version}-Linux-x86_64.sh"
miniconda_md5="718259965f234088d785cad1fbd7de03" miniconda_md5="718259965f234088d785cad1fbd7de03"
python_version="3.6" jdk_version="11.0.7"
jdk_url="https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.7_10.tar.gz"
jdk_version="8u212" jdk_sha256="ee60304d782c9d5654bf1a6b3f38c683921c1711045e1db94525a51b7024a2ca"
jdk_url="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u212-b03/OpenJDK8U-jdk_x64_linux_hotspot_8u212b03.tar.gz"
jdk_sha256="dd28d6d2cde2b931caf94ac2422a2ad082ea62f0beee3bf7057317c53093de93"
maven_version="3.6.3" maven_version="3.6.3"
maven_url="http://mirror.easyname.ch/apache/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz" maven_url="https://downloads.apache.org/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz"
maven_sha512="c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0" maven_sha512="c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0"
# Define Python requirements # Define Python requirements
python_requirements=$(cat <<EOF environment_yaml=$(cat <<EOF
tqdm==4.30.0 name: venv
pyyaml==3.13
pandas==0.24.1 channels:
pyproj==1.9.6 - defaults
geopandas==0.4.0
scikit-learn==0.20.2 dependencies:
requests==2.21.0 - matplotlib=3.1.3
rtree==0.8.3 - pandas=1.0.3
xlrd==1.2.0 - scipy=1.4.1
jinja2==2.9.5 - numpy=1.18.1
- geopandas=0.6.1
- numba=0.49.0
- palettable=3.3.0
- scikit-learn=0.22.1
- shapely=1.7.0
- tqdm=4.46.0
- pytables=3.6.1
- xlrd=1.2.0
- pip=20.0.2
- pip:
- simpledbf==0.2.6
- synpp==1.3.1
- python-Levenshtein==0.12.0
# For testing
- pytest==5.4.2
- xlwt==1.3.0
- pysal==1.14.4.post1
EOF EOF
) )
# Miniconda update script to avoid too long paths in interpreter path # Miniconda update script to avoid too long paths in interpreter path
miniconda_update_script=$(cat <<EOF miniconda_update_script=$(cat <<EOF
import sys import sys
...@@ -38,7 +57,7 @@ import re ...@@ -38,7 +57,7 @@ import re
with open(sys.argv[1]) as f: with open(sys.argv[1]) as f:
content = f.read() content = f.read()
content = re.sub(r'#!(.+)/miniconda3/bin/python', '#!/usr/bin/env python', content) content = re.sub(r'#!(.+)/miniconda/bin/python', '#!/usr/bin/env python', content)
with open(sys.argv[1], "w+") as f: with open(sys.argv[1], "w+") as f:
f.write(content) f.write(content)
...@@ -63,7 +82,7 @@ cd ${environment_directory} ...@@ -63,7 +82,7 @@ cd ${environment_directory}
if [ "$(md5sum miniconda.sh)" == "${miniconda_md5} miniconda.sh" ]; then if [ "$(md5sum miniconda.sh)" == "${miniconda_md5} miniconda.sh" ]; then
echo "Miniconda 3 ${miniconda_version} already downloaded." echo "Miniconda 3 ${miniconda_version} already downloaded."
else else
echo "Downloading Miniconda3 ${miniconda_version} ..." echo "Downloading Miniconda ${miniconda_version} ..."
rm -rf miniconda_installed rm -rf miniconda_installed
rm -rf python_installed rm -rf python_installed
curl -o miniconda.sh ${miniconda_url} curl -o miniconda.sh ${miniconda_url}
...@@ -91,19 +110,22 @@ fi ...@@ -91,19 +110,22 @@ fi
# III.1) Install Miniconda # III.1) Install Miniconda
if [ -f miniconda_installed ]; then if [ -f miniconda_installed ]; then
echo "Miniconda3 ${miniconda_version} already installed." echo "Miniconda ${miniconda_version} already installed."
else else
echo "Installing Miniconda3 ${miniconda_version} ..." echo "Installing Miniconda ${miniconda_version} ..."
rm -rf miniconda3 rm -rf miniconda
sh miniconda.sh -b -u -p miniconda3 sh miniconda.sh -b -u -p miniconda
cat <<< "${miniconda_update_script}" > fix_conda.py cat <<< "${miniconda_update_script}" > fix_conda.py
PATH=${environment_directory}/miniconda3/bin:$PATH PATH=${environment_directory}/miniconda/bin:$PATH
python fix_conda.py miniconda3/bin/conda python fix_conda.py miniconda/bin/conda
python fix_conda.py miniconda3/bin/conda-env python fix_conda.py miniconda/bin/conda-env
conda update -y conda
source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
conda config --set always_yes yes --set changeps1 no
conda update -q conda
touch miniconda_installed touch miniconda_installed
fi fi
...@@ -114,8 +136,10 @@ if [ -f python_installed ]; then ...@@ -114,8 +136,10 @@ if [ -f python_installed ]; then
else else
echo "Setting up Python environment ..." echo "Setting up Python environment ..."
cat <<< "${python_requirements}" > requirements.txt cat <<< "${environment_yaml}" > environment.yml
conda create -p venv python=${python_version} --no-default-packages --channel conda-forge --file requirements.txt -y
source "${environment_directory}/miniconda/etc/profile.d/conda.sh"
conda env create -f environment.yml
touch python_installed touch python_installed
fi fi
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment