diff --git a/.gitignore b/.gitignore index 91879240b36709b5c827ec951366cc55ad515cce..0411de3941d790fd1668fe2328b248cd3c09be08 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,6 @@ MANIFEST /finn-hlslib/ /pyverilator/ /PYNQ-HelloWorld/ + +# Jenkins cfg dir +/docker/jenkins_home diff --git a/docker/Dockerfile.finn_ci b/docker/Dockerfile.finn_ci new file mode 100644 index 0000000000000000000000000000000000000000..dd0c28da759d31544a68f2a969783174c628c28b --- /dev/null +++ b/docker/Dockerfile.finn_ci @@ -0,0 +1,93 @@ +# Copyright (c) 2020, Xilinx +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of FINN nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM pytorch/pytorch:1.1.0-cuda10.0-cudnn7.5-devel +MAINTAINER Yaman Umuroglu <yamanu@xilinx.com> +ARG PYTHON_VERSION=3.6 +ARG BUILD_PATH +ARG FINN_CI_BRANCH + +WORKDIR /workspace + +RUN apt-get update +RUN apt-get -y upgrade +RUN apt-get install -y build-essential libglib2.0-0 libsm6 libxext6 libxrender-dev +RUN apt install verilator +RUN apt-get -y install sshpass +RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config + +# cloning dependency repos +# Brevitas +RUN git clone --branch feature/finn_onnx_export https://github.com/Xilinx/brevitas.git /workspace/brevitas +RUN git -C /workspace/brevitas checkout 215cf44c76d562339fca368c8c3afee3110033e8 + +# Brevitas examples +RUN git clone --branch feature/rework_scaling_clipping https://github.com/maltanar/brevitas_cnv_lfc.git /workspace/brevitas_cnv_lfc +RUN git -C /workspace/brevitas_cnv_lfc checkout 2059f96bd576bf71f32c757e7f92617a70190c90 + +# CNPY +RUN git clone https://github.com/rogersce/cnpy.git /workspace/cnpy +RUN git -C /workspace/cnpy checkout 4e8810b1a8637695171ed346ce68f6984e585ef4 + +# FINN hlslib +RUN git clone https://github.com/maltanar/finn-hlslib.git /workspace/finn-hlslib +RUN git -C /workspace/finn-hlslib checkout b139bf051ac8f8e0a3625509247f714127cf3317 + +# PyVerilator +RUN git clone https://github.com/maltanar/pyverilator /workspace/pyverilator +RUN git -C /workspace/pyverilator checkout 307fc5c82db748620836307a2002fdc9fe170226 + +# PYNQ-HelloWorld +RUN git clone --branch feature/synth_rpt https://github.com/maltanar/PYNQ-HelloWorld.git /workspace/PYNQ-HelloWorld +RUN git -C /workspace/PYNQ-HelloWorld checkout db7e418767ce2a8e08fe732ddb3aa56ee79b7560 + +# FINN +RUN git clone --branch $FINN_CI_BRANCH https://github.com/Xilinx/finn /workspace/finn + +RUN pip install -r /workspace/finn/requirements.txt +RUN apt update; apt install nano +RUN pip install pytest-dependency + +ENV PYTHONPATH "${PYTHONPATH}:/workspace/finn/src" +ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas_cnv_lfc/training_scripts" +ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas" +ENV PYTHONPATH "${PYTHONPATH}:/workspace/pyverilator" +ENV PYNQSHELL_PATH "/workspace/PYNQ-HelloWorld/boards" +ENV VIVADO_IP_CACHE "$BUILD_PATH/vivado_ip_cache" + +# colorful terminal output +RUN echo "PS1='\[\033[1;36m\]\u\[\033[1;31m\]@\[\033[1;32m\]\h:\[\033[1;35m\]\w\[\033[1;31m\]\$\[\033[0m\] '" >> /root/.bashrc +RUN mkdir -p $BUILD_PATH +RUN mkdir -p $VIVADO_IP_CACHE + +WORKDIR /workspace/finn + +COPY finn_entrypoint.sh /usr/local/bin/ +RUN chmod 755 /usr/local/bin/finn_entrypoint.sh +ENTRYPOINT ["finn_entrypoint.sh"] +CMD ["bash"] diff --git a/Dockerfile b/docker/Dockerfile.finn_dev similarity index 93% rename from Dockerfile rename to docker/Dockerfile.finn_dev index eb0e746df429b6617432b23a9c77ec0b91732372..e28492bd31f3a2115ac566ed06a0125d348208f4 100644 --- a/Dockerfile +++ b/docker/Dockerfile.finn_dev @@ -29,27 +29,52 @@ FROM pytorch/pytorch:1.1.0-cuda10.0-cudnn7.5-devel MAINTAINER Yaman Umuroglu <yamanu@xilinx.com> ARG PYTHON_VERSION=3.6 +ARG GID +ARG GNAME +ARG UNAME +ARG UID +ARG PASSWD +ARG JUPYTER_PORT +ARG NETRON_PORT + +EXPOSE $JUPYTER_PORT +EXPOSE $NETRON_PORT WORKDIR /workspace +RUN apt-get update +RUN apt-get -y upgrade +RUN apt-get install -y build-essential libglib2.0-0 libsm6 libxext6 libxrender-dev +RUN apt-get install verilator +RUN apt-get install nano +RUN apt-get -y install sshpass +RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config + COPY requirements.txt . RUN pip install -r requirements.txt RUN rm requirements.txt -RUN apt update; apt install nano RUN pip install jupyter RUN pip install netron RUN pip install matplotlib RUN pip install pytest-dependency -RUN apt-get update -RUN apt-get -y upgrade -RUN apt-get install -y build-essential libglib2.0-0 libsm6 libxext6 libxrender-dev -RUN apt install verilator -RUN apt-get -y install sshpass -RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config RUN pip install sphinx RUN pip install sphinx_rtd_theme -# cloning dependency repos +# copy entrypoint script +COPY docker/finn_entrypoint.sh /usr/local/bin/ +RUN chmod 755 /usr/local/bin/finn_entrypoint.sh + +# switch user +RUN groupadd -g $GID $GNAME +RUN useradd -M -u $UID $UNAME -g $GNAME +RUN usermod -aG sudo $UNAME +RUN echo "$UNAME:$PASSWD" | chpasswd +RUN echo "root:$PASSWD" | chpasswd +RUN ln -s /workspace /home/$UNAME +RUN chown -R $UNAME:$GNAME /home/$UNAME +USER $UNAME + +# cloning dependency repos (as user) # Brevitas RUN git clone --branch feature/finn_onnx_export https://github.com/Xilinx/brevitas.git /workspace/brevitas RUN git -C /workspace/brevitas checkout 215cf44c76d562339fca368c8c3afee3110033e8 @@ -77,33 +102,14 @@ RUN git -C /workspace/PYNQ-HelloWorld checkout db7e418767ce2a8e08fe732ddb3aa56ee # Note that we expect the cloned finn directory on the host to be # mounted on /workspace/finn -- see run-docker.sh for an example # of how to do this. -# This branch assumes the same for brevitas and brevitas_cnv_lfc for easier -# co-development. ENV PYTHONPATH "${PYTHONPATH}:/workspace/finn/src" ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas_cnv_lfc/training_scripts" ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas" ENV PYTHONPATH "${PYTHONPATH}:/workspace/pyverilator" ENV PYNQSHELL_PATH "/workspace/PYNQ-HelloWorld/boards" -ARG GID -ARG GNAME -ARG UNAME -ARG UID -ARG PASSWD -ARG JUPYTER_PORT -ARG NETRON_PORT - -RUN groupadd -g $GID $GNAME -RUN useradd -M -u $UID $UNAME -g $GNAME -RUN usermod -aG sudo $UNAME -RUN echo "$UNAME:$PASSWD" | chpasswd -RUN echo "root:$PASSWD" | chpasswd -RUN ln -s /workspace /home/$UNAME -RUN chown -R $UNAME:$GNAME /home/$UNAME -USER $UNAME - -RUN echo "source \$VIVADO_PATH/settings64.sh" >> /home/$UNAME/.bashrc -RUN echo "PS1='\[\033[1;36m\]\u\[\033[1;31m\]@\[\033[1;32m\]\h:\[\033[1;35m\]\w\[\033[1;31m\]\$\[\033[0m\] '" >> /home/$UNAME/.bashrc -EXPOSE $JUPYTER_PORT -EXPOSE $NETRON_PORT WORKDIR /home/$UNAME/finn +RUN echo "PS1='\[\033[1;36m\]\u\[\033[1;31m\]@\[\033[1;32m\]\h:\[\033[1;35m\]\w\[\033[1;31m\]\$\[\033[0m\] '" >> /home/$UNAME/.bashrc + +ENTRYPOINT ["finn_entrypoint.sh"] +CMD ["bash"] diff --git a/docker/Dockerfile.jenkins b/docker/Dockerfile.jenkins new file mode 100644 index 0000000000000000000000000000000000000000..e1939b642e1493ee97daf6472009649d3634632f --- /dev/null +++ b/docker/Dockerfile.jenkins @@ -0,0 +1,11 @@ +FROM jenkins/jenkins:lts +# if we want to install via apt +USER root +RUN apt-get update +RUN apt-get install -y gnupg-agent curl ca-certificates apt-transport-https software-properties-common +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" +RUN apt-get update +RUN apt-get install -y docker-ce-cli +# drop back to the regular jenkins user - good practice +USER jenkins diff --git a/docker/Jenkinsfile b/docker/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..c4c50a0434a9f037f71cb293cf0e1e6feb300b39 --- /dev/null +++ b/docker/Jenkinsfile @@ -0,0 +1,52 @@ +pipeline { + agent any + parameters { + string(name: 'FINN_CI_BRANCH', defaultValue: '', description: 'FINN branch to build') + string(name: 'VIVADO_PATH', defaultValue: '', description: 'Path to Vivado installation') + string(name: 'PYNQ_BOARD', defaultValue: 'Pynq-Z1', description: 'PYNQ board type') + string(name: 'PYNQ_IP', defaultValue: '', description: 'PYNQ board IP address') + string(name: 'PYNQ_USERNAME', defaultValue: 'xilinx', description: 'PYNQ board username') + string(name: 'PYNQ_PASSWORD', defaultValue: 'xilinx', description: 'PYNQ board password') + string(name: 'PYNQ_TARGET_DIR', defaultValue: '/home/xilinx/finn', description: 'PYNQ board target deployment directory') + } + environment { + DOCKER_TAG='finn_ci:$BUILD_ID' + DOCKER_INST_NAME='finn_ci_$BUILD_ID' + BUILD_PATH='/tmp/finn_ci_$BUILD_ID' + DOCKER_CMD="python setup.py test" + } + stages { + stage("Clone") { + steps { + git branch: "${params.FINN_CI_BRANCH}", url: 'https://github.com/Xilinx/finn.git' + } + } + stage('Build') { + steps { + sh """ + docker build -t $DOCKER_TAG -f docker/Dockerfile.finn_ci \ + --build-arg BUILD_PATH=$BUILD_PATH \ + --build-arg FINN_CI_BRANCH=${params.FINN_CI_BRANCH} \ + docker/ + """ + } + } + stage('Test') { + steps { + sh """ + docker run --name $DOCKER_INST_NAME \ + --hostname $DOCKER_INST_NAME \ + -v ${params.VIVADO_PATH}:${params.VIVADO_PATH}:ro \ + -e FINN_INST_NAME=$DOCKER_INST_NAME \ + -e VIVADO_PATH=${params.VIVADO_PATH} \ + -e PYNQ_BOARD=${params.PYNQ_BOARD} \ + -e PYNQ_IP=${params.PYNQ_IP} \ + -e PYNQ_USERNAME=${params.PYNQ_USERNAME} \ + -e PYNQ_PASSWORD=${params.PYNQ_PASSWORD} \ + -e PYNQ_TARGET_DIR=${params.PYNQ_TARGET_DIR} \ + $DOCKER_TAG bash -c "$DOCKER_CMD" + """ + } + } + } +} diff --git a/docker/finn_entrypoint.sh b/docker/finn_entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..930218e26eff0b7be541529f452efc2a038160c5 --- /dev/null +++ b/docker/finn_entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export XILINX_VIVADO=$VIVADO_PATH +export SHELL=/bin/bash +export FINN_ROOT=/workspace/finn + +# source Vivado env.vars +source $VIVADO_PATH/settings64.sh + +exec "$@" diff --git a/docker/launch-jenkins.sh b/docker/launch-jenkins.sh new file mode 100755 index 0000000000000000000000000000000000000000..64dc1ec73f68e621cdd737595983b6b9a217f6fe --- /dev/null +++ b/docker/launch-jenkins.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# defaults, can be overriden by environment variables +# user to run Jenkins as -- see NOTE below regarding Docker access permissions +: ${JENKINS_USER=jenkins} +# port for Jenkins on host machine +: ${JENKINS_PORT=8080} +# make Jenkins config persistent by mounting into this folder +: ${JENKINS_HOME=$(pwd)/jenkins_home} + +mkdir -p $JENKINS_HOME + +# build a Jenkins Docker image that also has the Docker CLI installed +docker build -t finn_jenkins -f Dockerfile.jenkins . + +# launch Docker container mounted to local Docker socket +# NOTE: we allow customizing the user (e.g. as root) to work around permission +# issues, may not al +docker run -u $JENKINS_USER -p $JENKINS_PORT:8080 -v /var/run/docker.sock:/var/run/docker.sock -v $JENKINS_HOME:/var/jenkins_home finn_jenkins diff --git a/run-docker.sh b/run-docker.sh index 62ca70c2cbc80f28b12f0b0ff8f9139db0108271..018bd9aa8c39666a1b9c0ef7f426587f265769f7 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -44,10 +44,10 @@ DOCKER_PASSWD="finn" # generate a random number per-run to allow multiple # containers from the same user DOCKER_RND=$(shuf -i0-32768 -n1) -DOCKER_TAG="finn_${DOCKER_UNAME}" +DOCKER_TAG="finn_dev_${DOCKER_UNAME}" # uncomment to run multiple instances with different names # DOCKER_INST_NAME="finn_${DOCKER_UNAME}_${DOCKER_RND}" -DOCKER_INST_NAME="finn_${DOCKER_UNAME}" +DOCKER_INST_NAME="finn_dev_${DOCKER_UNAME}" # ensure Docker tag and inst. name are all lowercase DOCKER_TAG=$(echo "$DOCKER_TAG" | tr '[:upper:]' '[:lower:]') DOCKER_INST_NAME=$(echo "$DOCKER_INST_NAME" | tr '[:upper:]' '[:lower:]') @@ -59,6 +59,7 @@ DOCKER_INST_NAME=$(echo "$DOCKER_INST_NAME" | tr '[:upper:]' '[:lower:]') : ${PYNQ_PASSWORD="xilinx"} : ${PYNQ_BOARD="Pynq-Z1"} : ${PYNQ_TARGET_DIR="/home/xilinx/$DOCKER_INST_NAME"} +: ${NUM_DEFAULT_WORKERS=1} # Absolute path to this script, e.g. /home/user/bin/foo.sh SCRIPT=$(readlink -f "$0") @@ -83,17 +84,17 @@ echo "Using default PYNQ board $PYNQ_BOARD" if [ "$1" = "test" ]; then echo "Running test suite" - DOCKER_CMD="source ~/.bashrc; python setup.py test" + DOCKER_CMD="python setup.py test" elif [ "$1" = "notebook" ]; then echo "Running Jupyter notebook server" - DOCKER_CMD="source ~/.bashrc; jupyter notebook --ip=0.0.0.0 --port $JUPYTER_PORT notebooks" + DOCKER_CMD="jupyter notebook --ip=0.0.0.0 --port $JUPYTER_PORT notebooks" else echo "Running container only" DOCKER_CMD="bash" fi # Build the FINN Docker image -docker build --tag=$DOCKER_TAG \ +docker build -f docker/Dockerfile.finn_dev --tag=$DOCKER_TAG \ --build-arg GID=$DOCKER_GID \ --build-arg GNAME=$DOCKER_GNAME \ --build-arg UNAME=$DOCKER_UNAME \ @@ -119,6 +120,7 @@ docker run -t --rm --name $DOCKER_INST_NAME -it \ -e PYNQ_USERNAME=$PYNQ_USERNAME \ -e PYNQ_PASSWORD=$PYNQ_PASSWORD \ -e PYNQ_TARGET_DIR=$PYNQ_TARGET_DIR \ +-e NUM_DEFAULT_WORKERS=$NUM_DEFAULT_WORKERS \ -p $JUPYTER_PORT:$JUPYTER_PORT \ -p $NETRON_PORT:$NETRON_PORT \ -$DOCKER_TAG bash -c "$DOCKER_CMD" +$DOCKER_TAG $DOCKER_CMD