Skip to content
Snippets Groups Projects
Commit 21536ec3 authored by Yaman Umuroglu's avatar Yaman Umuroglu
Browse files

Merge branch 'feature/docker_reorg' into dev

parents af828f3f d7323d21
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitignore
+ 3
− 0
View file @ 21536ec3
......@@ -83,3 +83,6 @@ MANIFEST
/finn-hlslib/
/pyverilator/
/PYNQ-HelloWorld/
# Jenkins cfg dir
/docker/jenkins_home
docker/Dockerfile.finn_ci 0 → 100644
+ 93
− 0
View file @ 21536ec3
# Copyright (c) 2020, Xilinx
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of FINN nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
FROM pytorch/pytorch:1.1.0-cuda10.0-cudnn7.5-devel
MAINTAINER Yaman Umuroglu <yamanu@xilinx.com>
ARG PYTHON_VERSION=3.6
ARG BUILD_PATH
ARG FINN_CI_BRANCH
WORKDIR /workspace
RUN apt-get update
RUN apt-get -y upgrade
RUN apt-get install -y build-essential libglib2.0-0 libsm6 libxext6 libxrender-dev
RUN apt install verilator
RUN apt-get -y install sshpass
RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
# cloning dependency repos
# Brevitas
RUN git clone --branch feature/finn_onnx_export https://github.com/Xilinx/brevitas.git /workspace/brevitas
RUN git -C /workspace/brevitas checkout 215cf44c76d562339fca368c8c3afee3110033e8
# Brevitas examples
RUN git clone --branch feature/rework_scaling_clipping https://github.com/maltanar/brevitas_cnv_lfc.git /workspace/brevitas_cnv_lfc
RUN git -C /workspace/brevitas_cnv_lfc checkout 2059f96bd576bf71f32c757e7f92617a70190c90
# CNPY
RUN git clone https://github.com/rogersce/cnpy.git /workspace/cnpy
RUN git -C /workspace/cnpy checkout 4e8810b1a8637695171ed346ce68f6984e585ef4
# FINN hlslib
RUN git clone https://github.com/maltanar/finn-hlslib.git /workspace/finn-hlslib
RUN git -C /workspace/finn-hlslib checkout b139bf051ac8f8e0a3625509247f714127cf3317
# PyVerilator
RUN git clone https://github.com/maltanar/pyverilator /workspace/pyverilator
RUN git -C /workspace/pyverilator checkout 307fc5c82db748620836307a2002fdc9fe170226
# PYNQ-HelloWorld
RUN git clone --branch feature/synth_rpt https://github.com/maltanar/PYNQ-HelloWorld.git /workspace/PYNQ-HelloWorld
RUN git -C /workspace/PYNQ-HelloWorld checkout db7e418767ce2a8e08fe732ddb3aa56ee79b7560
# FINN
RUN git clone --branch $FINN_CI_BRANCH https://github.com/Xilinx/finn /workspace/finn
RUN pip install -r /workspace/finn/requirements.txt
RUN apt update; apt install nano
RUN pip install pytest-dependency
ENV PYTHONPATH "${PYTHONPATH}:/workspace/finn/src"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas_cnv_lfc/training_scripts"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/pyverilator"
ENV PYNQSHELL_PATH "/workspace/PYNQ-HelloWorld/boards"
ENV VIVADO_IP_CACHE "$BUILD_PATH/vivado_ip_cache"
# colorful terminal output
RUN echo "PS1='\[\033[1;36m\]\u\[\033[1;31m\]@\[\033[1;32m\]\h:\[\033[1;35m\]\w\[\033[1;31m\]\$\[\033[0m\] '" >> /root/.bashrc
RUN mkdir -p $BUILD_PATH
RUN mkdir -p $VIVADO_IP_CACHE
WORKDIR /workspace/finn
COPY finn_entrypoint.sh /usr/local/bin/
RUN chmod 755 /usr/local/bin/finn_entrypoint.sh
ENTRYPOINT ["finn_entrypoint.sh"]
CMD ["bash"]
Dockerfile → docker/Dockerfile.finn_dev
+ 37
− 31
View file @ 21536ec3
......@@ -29,27 +29,52 @@
FROM pytorch/pytorch:1.1.0-cuda10.0-cudnn7.5-devel
MAINTAINER Yaman Umuroglu <yamanu@xilinx.com>
ARG PYTHON_VERSION=3.6
ARG GID
ARG GNAME
ARG UNAME
ARG UID
ARG PASSWD
ARG JUPYTER_PORT
ARG NETRON_PORT
EXPOSE $JUPYTER_PORT
EXPOSE $NETRON_PORT
WORKDIR /workspace
RUN apt-get update
RUN apt-get -y upgrade
RUN apt-get install -y build-essential libglib2.0-0 libsm6 libxext6 libxrender-dev
RUN apt-get install verilator
RUN apt-get install nano
RUN apt-get -y install sshpass
RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
COPY requirements.txt .
RUN pip install -r requirements.txt
RUN rm requirements.txt
RUN apt update; apt install nano
RUN pip install jupyter
RUN pip install netron
RUN pip install matplotlib
RUN pip install pytest-dependency
RUN apt-get update
RUN apt-get -y upgrade
RUN apt-get install -y build-essential libglib2.0-0 libsm6 libxext6 libxrender-dev
RUN apt install verilator
RUN apt-get -y install sshpass
RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
RUN pip install sphinx
RUN pip install sphinx_rtd_theme
# cloning dependency repos
# copy entrypoint script
COPY docker/finn_entrypoint.sh /usr/local/bin/
RUN chmod 755 /usr/local/bin/finn_entrypoint.sh
# switch user
RUN groupadd -g $GID $GNAME
RUN useradd -M -u $UID $UNAME -g $GNAME
RUN usermod -aG sudo $UNAME
RUN echo "$UNAME:$PASSWD" | chpasswd
RUN echo "root:$PASSWD" | chpasswd
RUN ln -s /workspace /home/$UNAME
RUN chown -R $UNAME:$GNAME /home/$UNAME
USER $UNAME
# cloning dependency repos (as user)
# Brevitas
RUN git clone --branch feature/finn_onnx_export https://github.com/Xilinx/brevitas.git /workspace/brevitas
RUN git -C /workspace/brevitas checkout 215cf44c76d562339fca368c8c3afee3110033e8
......@@ -77,33 +102,14 @@ RUN git -C /workspace/PYNQ-HelloWorld checkout db7e418767ce2a8e08fe732ddb3aa56ee
# Note that we expect the cloned finn directory on the host to be
# mounted on /workspace/finn -- see run-docker.sh for an example
# of how to do this.
# This branch assumes the same for brevitas and brevitas_cnv_lfc for easier
# co-development.
ENV PYTHONPATH "${PYTHONPATH}:/workspace/finn/src"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas_cnv_lfc/training_scripts"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/pyverilator"
ENV PYNQSHELL_PATH "/workspace/PYNQ-HelloWorld/boards"
ARG GID
ARG GNAME
ARG UNAME
ARG UID
ARG PASSWD
ARG JUPYTER_PORT
ARG NETRON_PORT
RUN groupadd -g $GID $GNAME
RUN useradd -M -u $UID $UNAME -g $GNAME
RUN usermod -aG sudo $UNAME
RUN echo "$UNAME:$PASSWD" | chpasswd
RUN echo "root:$PASSWD" | chpasswd
RUN ln -s /workspace /home/$UNAME
RUN chown -R $UNAME:$GNAME /home/$UNAME
USER $UNAME
RUN echo "source \$VIVADO_PATH/settings64.sh" >> /home/$UNAME/.bashrc
RUN echo "PS1='\[\033[1;36m\]\u\[\033[1;31m\]@\[\033[1;32m\]\h:\[\033[1;35m\]\w\[\033[1;31m\]\$\[\033[0m\] '" >> /home/$UNAME/.bashrc
EXPOSE $JUPYTER_PORT
EXPOSE $NETRON_PORT
WORKDIR /home/$UNAME/finn
RUN echo "PS1='\[\033[1;36m\]\u\[\033[1;31m\]@\[\033[1;32m\]\h:\[\033[1;35m\]\w\[\033[1;31m\]\$\[\033[0m\] '" >> /home/$UNAME/.bashrc
ENTRYPOINT ["finn_entrypoint.sh"]
CMD ["bash"]
docker/Dockerfile.jenkins 0 → 100644
+ 11
− 0
View file @ 21536ec3
FROM jenkins/jenkins:lts
# if we want to install via apt
USER root
RUN apt-get update
RUN apt-get install -y gnupg-agent curl ca-certificates apt-transport-https software-properties-common
RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable"
RUN apt-get update
RUN apt-get install -y docker-ce-cli
# drop back to the regular jenkins user - good practice
USER jenkins
docker/Jenkinsfile 0 → 100644
+ 52
− 0
View file @ 21536ec3
pipeline {
agent any
parameters {
string(name: 'FINN_CI_BRANCH', defaultValue: '', description: 'FINN branch to build')
string(name: 'VIVADO_PATH', defaultValue: '', description: 'Path to Vivado installation')
string(name: 'PYNQ_BOARD', defaultValue: 'Pynq-Z1', description: 'PYNQ board type')
string(name: 'PYNQ_IP', defaultValue: '', description: 'PYNQ board IP address')
string(name: 'PYNQ_USERNAME', defaultValue: 'xilinx', description: 'PYNQ board username')
string(name: 'PYNQ_PASSWORD', defaultValue: 'xilinx', description: 'PYNQ board password')
string(name: 'PYNQ_TARGET_DIR', defaultValue: '/home/xilinx/finn', description: 'PYNQ board target deployment directory')
}
environment {
DOCKER_TAG='finn_ci:$BUILD_ID'
DOCKER_INST_NAME='finn_ci_$BUILD_ID'
BUILD_PATH='/tmp/finn_ci_$BUILD_ID'
DOCKER_CMD="python setup.py test"
}
stages {
stage("Clone") {
steps {
git branch: "${params.FINN_CI_BRANCH}", url: 'https://github.com/Xilinx/finn.git'
}
}
stage('Build') {
steps {
sh """
docker build -t $DOCKER_TAG -f docker/Dockerfile.finn_ci \
--build-arg BUILD_PATH=$BUILD_PATH \
--build-arg FINN_CI_BRANCH=${params.FINN_CI_BRANCH} \
docker/
"""
}
}
stage('Test') {
steps {
sh """
docker run --name $DOCKER_INST_NAME \
--hostname $DOCKER_INST_NAME \
-v ${params.VIVADO_PATH}:${params.VIVADO_PATH}:ro \
-e FINN_INST_NAME=$DOCKER_INST_NAME \
-e VIVADO_PATH=${params.VIVADO_PATH} \
-e PYNQ_BOARD=${params.PYNQ_BOARD} \
-e PYNQ_IP=${params.PYNQ_IP} \
-e PYNQ_USERNAME=${params.PYNQ_USERNAME} \
-e PYNQ_PASSWORD=${params.PYNQ_PASSWORD} \
-e PYNQ_TARGET_DIR=${params.PYNQ_TARGET_DIR} \
$DOCKER_TAG bash -c "$DOCKER_CMD"
"""
}
}
}
}
docker/finn_entrypoint.sh 0 → 100644
+ 10
− 0
View file @ 21536ec3
#!/bin/bash
export XILINX_VIVADO=$VIVADO_PATH
export SHELL=/bin/bash
export FINN_ROOT=/workspace/finn
# source Vivado env.vars
source $VIVADO_PATH/settings64.sh
exec "$@"
docker/launch-jenkins.sh 0 → 100755
+ 19
− 0
View file @ 21536ec3
#!/bin/bash
# defaults, can be overriden by environment variables
# user to run Jenkins as -- see NOTE below regarding Docker access permissions
: ${JENKINS_USER=jenkins}
# port for Jenkins on host machine
: ${JENKINS_PORT=8080}
# make Jenkins config persistent by mounting into this folder
: ${JENKINS_HOME=$(pwd)/jenkins_home}
mkdir -p $JENKINS_HOME
# build a Jenkins Docker image that also has the Docker CLI installed
docker build -t finn_jenkins -f Dockerfile.jenkins .
# launch Docker container mounted to local Docker socket
# NOTE: we allow customizing the user (e.g. as root) to work around permission
# issues, may not al
docker run -u $JENKINS_USER -p $JENKINS_PORT:8080 -v /var/run/docker.sock:/var/run/docker.sock -v $JENKINS_HOME:/var/jenkins_home finn_jenkins
run-docker.sh
+ 8
− 6
View file @ 21536ec3
......@@ -44,10 +44,10 @@ DOCKER_PASSWD="finn"
# generate a random number per-run to allow multiple
# containers from the same user
DOCKER_RND=$(shuf -i0-32768 -n1)
DOCKER_TAG="finn_${DOCKER_UNAME}"
DOCKER_TAG="finn_dev_${DOCKER_UNAME}"
# uncomment to run multiple instances with different names
# DOCKER_INST_NAME="finn_${DOCKER_UNAME}_${DOCKER_RND}"
DOCKER_INST_NAME="finn_${DOCKER_UNAME}"
DOCKER_INST_NAME="finn_dev_${DOCKER_UNAME}"
# ensure Docker tag and inst. name are all lowercase
DOCKER_TAG=$(echo "$DOCKER_TAG" | tr '[:upper:]' '[:lower:]')
DOCKER_INST_NAME=$(echo "$DOCKER_INST_NAME" | tr '[:upper:]' '[:lower:]')
......@@ -59,6 +59,7 @@ DOCKER_INST_NAME=$(echo "$DOCKER_INST_NAME" | tr '[:upper:]' '[:lower:]')
: ${PYNQ_PASSWORD="xilinx"}
: ${PYNQ_BOARD="Pynq-Z1"}
: ${PYNQ_TARGET_DIR="/home/xilinx/$DOCKER_INST_NAME"}
: ${NUM_DEFAULT_WORKERS=1}
# Absolute path to this script, e.g. /home/user/bin/foo.sh
SCRIPT=$(readlink -f "$0")
......@@ -83,17 +84,17 @@ echo "Using default PYNQ board $PYNQ_BOARD"
if [ "$1" = "test" ]; then
echo "Running test suite"
DOCKER_CMD="source ~/.bashrc; python setup.py test"
DOCKER_CMD="python setup.py test"
elif [ "$1" = "notebook" ]; then
echo "Running Jupyter notebook server"
DOCKER_CMD="source ~/.bashrc; jupyter notebook --ip=0.0.0.0 --port $JUPYTER_PORT notebooks"
DOCKER_CMD="jupyter notebook --ip=0.0.0.0 --port $JUPYTER_PORT notebooks"
else
echo "Running container only"
DOCKER_CMD="bash"
fi
# Build the FINN Docker image
docker build --tag=$DOCKER_TAG \
docker build -f docker/Dockerfile.finn_dev --tag=$DOCKER_TAG \
--build-arg GID=$DOCKER_GID \
--build-arg GNAME=$DOCKER_GNAME \
--build-arg UNAME=$DOCKER_UNAME \
......@@ -119,6 +120,7 @@ docker run -t --rm --name $DOCKER_INST_NAME -it \
-e PYNQ_USERNAME=$PYNQ_USERNAME \
-e PYNQ_PASSWORD=$PYNQ_PASSWORD \
-e PYNQ_TARGET_DIR=$PYNQ_TARGET_DIR \
-e NUM_DEFAULT_WORKERS=$NUM_DEFAULT_WORKERS \
-p $JUPYTER_PORT:$JUPYTER_PORT \
-p $NETRON_PORT:$NETRON_PORT \
$DOCKER_TAG bash -c "$DOCKER_CMD"
$DOCKER_TAG $DOCKER_CMD
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment