diff --git a/.gitignore b/.gitignore
index 0411de3941d790fd1668fe2328b248cd3c09be08..f838c1695130d232ac6a2b888aed0cea31aafaa7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -76,13 +76,5 @@ MANIFEST
# Per-project virtualenvs
.venv*/
-# Cloned dependencies for Docker
-/brevitas/
-/brevitas_cnv_lfc/
-/cnpy/
-/finn-hlslib/
-/pyverilator/
-/PYNQ-HelloWorld/
-
# Jenkins cfg dir
/docker/jenkins_home
diff --git a/docker/Dockerfile.finn_ci b/docker/Dockerfile.finn_ci
index fb257b05c7c5e63922fe9c51241c18ab671ec0ba..41f6a88f5dd4c9b0822a74cf4a0e7b4663dce910 100644
--- a/docker/Dockerfile.finn_ci
+++ b/docker/Dockerfile.finn_ci
@@ -44,8 +44,6 @@ RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
# cloning dependency repos
# Brevitas
RUN git clone https://github.com/Xilinx/brevitas.git /workspace/brevitas
-# Brevitas examples
-RUN git clone https://github.com/maltanar/brevitas_cnv_lfc.git /workspace/brevitas_cnv_lfc
# CNPY
RUN git clone https://github.com/rogersce/cnpy.git /workspace/cnpy
# FINN hlslib
@@ -63,8 +61,6 @@ RUN apt update; apt install nano
RUN pip install pytest-dependency
ENV PYTHONPATH "${PYTHONPATH}:/workspace/finn/src"
-ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas_cnv_lfc/training_scripts"
-ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/pyverilator"
ENV PYNQSHELL_PATH "/workspace/PYNQ-HelloWorld/boards"
ENV VIVADO_IP_CACHE "$BUILD_PATH/vivado_ip_cache"
diff --git a/docker/Dockerfile.finn_dev b/docker/Dockerfile.finn_dev
index 48ad00a555aebf678f68036d1f5ec576f4c70c3f..b7cfc299a2999662672225aa5f8912653d189559 100644
--- a/docker/Dockerfile.finn_dev
+++ b/docker/Dockerfile.finn_dev
@@ -73,8 +73,6 @@ USER $UNAME
# cloning dependency repos (as user)
# Brevitas
RUN git clone https://github.com/Xilinx/brevitas.git /workspace/brevitas
-# Brevitas examples
-RUN git clone https://github.com/maltanar/brevitas_cnv_lfc.git /workspace/brevitas_cnv_lfc
# CNPY
RUN git clone https://github.com/rogersce/cnpy.git /workspace/cnpy
# FINN hlslib
@@ -87,8 +85,6 @@ RUN git clone https://github.com/maltanar/PYNQ-HelloWorld.git /workspace/PYNQ-He
# for this developer-oriented Docker container we assume the FINN repo is cloned and mounted from the host
# at /workspace/finn -- see run-docker.sh for an example of how to do this.
ENV PYTHONPATH "${PYTHONPATH}:/workspace/finn/src"
-ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas_cnv_lfc/training_scripts"
-ENV PYTHONPATH "${PYTHONPATH}:/workspace/brevitas"
ENV PYTHONPATH "${PYTHONPATH}:/workspace/pyverilator"
ENV PYNQSHELL_PATH "/workspace/PYNQ-HelloWorld/boards"
diff --git a/docker/finn_entrypoint.sh b/docker/finn_entrypoint.sh
index 7ba0aeabbd9bd83c7f33e54cfde626070dcc5ec5..9cc239319fe94f482a4a6564399943b4dfe6ff53 100644
--- a/docker/finn_entrypoint.sh
+++ b/docker/finn_entrypoint.sh
@@ -13,8 +13,7 @@ gecho () {
# checkout the correct dependency repo commits
# the repos themselves are cloned in the Dockerfile
-BREVITAS_COMMIT=215cf44c76d562339fca368c8c3afee3110033e8
-BREVITAS_EXAMPLES_COMMIT=2059f96bd576bf71f32c757e7f92617a70190c90
+BREVITAS_COMMIT=989cdfdba4700fdd900ba0b25a820591d561c21a
CNPY_COMMIT=4e8810b1a8637695171ed346ce68f6984e585ef4
HLSLIB_COMMIT=6b88db826bb023937506913a23d964775a7606af
PYVERILATOR_COMMIT=1d89cb0d4e0c97469cc6352c611f876ec13edfa6
@@ -26,10 +25,7 @@ gecho "Setting up known-good commit versions for FINN dependencies"
gecho "brevitas @ $BREVITAS_COMMIT"
git -C /workspace/brevitas pull --quiet
git -C /workspace/brevitas checkout $BREVITAS_COMMIT --quiet
-# Brevitas examples
-gecho "brevitas_cnv_lfc @ $BREVITAS_EXAMPLES_COMMIT"
-git -C /workspace/brevitas_cnv_lfc pull --quiet
-git -C /workspace/brevitas_cnv_lfc checkout $BREVITAS_EXAMPLES_COMMIT --quiet
+pip install --user -e /workspace/brevitas
# CNPY
gecho "cnpy @ $CNPY_COMMIT"
git -C /workspace/cnpy pull --quiet
diff --git a/docs/finn/brevitas_export.rst b/docs/finn/brevitas_export.rst
index 443b692a2d05b48b2e395373411c3d5382825c6c..83684ae092609ef0f83a5525508febf4676b2d7a 100644
--- a/docs/finn/brevitas_export.rst
+++ b/docs/finn/brevitas_export.rst
@@ -10,7 +10,7 @@ Brevitas Export
:scale: 70%
:align: center
-FINN expects an ONNX model as input. This can be a model trained with `Brevitas <https://github.com/Xilinx/brevitas>`_. Brevitas is a PyTorch library for quantization-aware training and the FINN Docker image comes with several `example Brevitas networks <https://github.com/maltanar/brevitas_cnv_lfc>`_. Brevitas provides an export of a quantized network in ONNX representation. The resulting model consists only of `ONNX standard nodes <https://github.com/onnx/onnx/blob/master/docs/Operators.md>`_, but also contains additional attributes for the ONNX nodes to represent low precision datatypes. To work with the model it is wrapped into :ref:`modelwrapper` provided by FINN.
+FINN expects an ONNX model as input. This can be a model trained with `Brevitas <https://github.com/Xilinx/brevitas>`_. Brevitas is a PyTorch library for quantization-aware training and the FINN Docker image comes with several `example Brevitas networks <https://github.com/Xilinx/brevitas/tree/master/brevitas_examples/bnn_pynq>`_. Brevitas provides an export of a quantized network in ONNX representation. The resulting model consists only of `ONNX standard nodes <https://github.com/onnx/onnx/blob/master/docs/Operators.md>`_, but also contains additional attributes for the ONNX nodes to represent low precision datatypes. To work with the model it is wrapped into :ref:`modelwrapper` provided by FINN.
At this stage we can already use the functional verification flow to simulate the model using Python, this is marked in the graphic with the dotted arrow. For more details please have look at :ref:`verification`.
diff --git a/docs/finn/example_networks.rst b/docs/finn/example_networks.rst
index 47c9a976cb14a3e175dff6800ad8a5da60b44ecb..9f221871f09bf655db9d81988d6fa83e53473634 100644
--- a/docs/finn/example_networks.rst
+++ b/docs/finn/example_networks.rst
@@ -4,7 +4,7 @@
Example Networks
****************
-FINN uses `several pre-trained QNNs <https://github.com/maltanar/brevitas_cnv_lfc>`_ that serve as examples and testcases.
+FINN uses `several pre-trained QNNs <https://github.com/Xilinx/brevitas/tree/master/brevitas_examples/bnn_pynq>`_ that serve as examples and testcases.
You can find a status summary below for each network.
* TFC, SFC, LFC... are fully-connected networks trained on the MNIST dataset
diff --git a/notebooks/basics/1_brevitas_network_import.ipynb b/notebooks/basics/1_brevitas_network_import.ipynb
index 0abf671a57ddeebb9e93745d2dfafb19e5a8373e..3c9cad615e168e19c7f5dfef45e7c7c60965d1e3 100644
--- a/notebooks/basics/1_brevitas_network_import.ipynb
+++ b/notebooks/basics/1_brevitas_network_import.ipynb
@@ -31,29 +31,64 @@
"source": [
"## 1. Load up the trained PyTorch model\n",
"\n",
- "The FINN Docker image comes with several [example Brevitas networks](https://github.com/maltanar/brevitas_cnv_lfc), and we'll use the LFC-w1a1 model as the example network here. This is a binarized fully connected network trained on the MNIST dataset. Let's start by looking at what the PyTorch network definition looks like:"
+ "The FINN Docker image comes with several [example Brevitas networks](https://github.com/Xilinx/brevitas/tree/master/brevitas_examples/bnn_pynq), and we'll use the LFC-w1a1 model as the example network here. This is a binarized fully connected network trained on the MNIST dataset. Let's start by looking at what the PyTorch network definition looks like:"
]
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
+ "# MIT License\n",
+ "#\n",
+ "# Copyright (c) 2019 Xilinx\n",
+ "#\n",
+ "# Permission is hereby granted, free of charge, to any person obtaining a copy\n",
+ "# of this software and associated documentation files (the \"Software\"), to deal\n",
+ "# in the Software without restriction, including without limitation the rights\n",
+ "# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
+ "# copies of the Software, and to permit persons to whom the Software is\n",
+ "# furnished to do so, subject to the following conditions:\n",
+ "#\n",
+ "# The above copyright notice and this permission notice shall be included in all\n",
+ "# copies or substantial portions of the Software.\n",
+ "#\n",
+ "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
+ "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
+ "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
+ "# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
+ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
+ "# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
+ "# SOFTWARE.\n",
+ "\n",
+ "from functools import reduce\n",
+ "from operator import mul\n",
+ "\n",
+ "from torch.nn import Module, ModuleList, BatchNorm1d, Dropout\n",
+ "import torch\n",
+ "\n",
+ "from .common import get_quant_linear, get_act_quant, get_quant_type, QuantLinear\n",
+ "\n",
+ "FC_OUT_FEATURES = [1024, 1024, 1024]\n",
+ "INTERMEDIATE_FC_PER_OUT_CH_SCALING = True\n",
+ "LAST_FC_PER_OUT_CH_SCALING = False\n",
+ "IN_DROPOUT = 0.2\n",
+ "HIDDEN_DROPOUT = 0.2\n",
+ "\n",
+ "\n",
"class LFC(Module):\n",
"\n",
" def __init__(self, num_classes=10, weight_bit_width=None, act_bit_width=None,\n",
- " in_bit_width=None, in_ch=1, in_features=(28, 28), device=\"cpu\"):\n",
+ " in_bit_width=None, in_ch=1, in_features=(28, 28)):\n",
" super(LFC, self).__init__()\n",
- " self.device = device\n",
"\n",
" weight_quant_type = get_quant_type(weight_bit_width)\n",
" act_quant_type = get_quant_type(act_bit_width)\n",
" in_quant_type = get_quant_type(in_bit_width)\n",
- " stats_op = get_stats_op(weight_quant_type)\n",
"\n",
" self.features = ModuleList()\n",
" self.features.append(get_act_quant(in_bit_width, in_quant_type))\n",
@@ -64,8 +99,7 @@
" out_features=out_features,\n",
" per_out_ch_scaling=INTERMEDIATE_FC_PER_OUT_CH_SCALING,\n",
" bit_width=weight_bit_width,\n",
- " quant_type=weight_quant_type,\n",
- " stats_op=stats_op))\n",
+ " quant_type=weight_quant_type))\n",
" in_features = out_features\n",
" self.features.append(BatchNorm1d(num_features=in_features))\n",
" self.features.append(get_act_quant(act_bit_width, act_quant_type))\n",
@@ -74,8 +108,7 @@
" out_features=num_classes,\n",
" per_out_ch_scaling=LAST_FC_PER_OUT_CH_SCALING,\n",
" bit_width=weight_bit_width,\n",
- " quant_type=weight_quant_type,\n",
- " stats_op=stats_op))\n",
+ " quant_type=weight_quant_type))\n",
" self.features.append(BatchNorm1d(num_features=num_classes))\n",
"\n",
" for m in self.modules():\n",
@@ -89,17 +122,31 @@
" \n",
" def forward(self, x):\n",
" x = x.view(x.shape[0], -1)\n",
- " x = 2.0 * x - torch.tensor([1.0]).to(self.device)\n",
+ " x = 2.0 * x - torch.tensor([1.0], device=x.device)\n",
" for mod in self.features:\n",
" x = mod(x)\n",
" return x\n",
+ "\n",
+ "\n",
+ "def lfc(cfg):\n",
+ " weight_bit_width = cfg.getint('QUANT', 'WEIGHT_BIT_WIDTH')\n",
+ " act_bit_width = cfg.getint('QUANT', 'ACT_BIT_WIDTH')\n",
+ " in_bit_width = cfg.getint('QUANT', 'IN_BIT_WIDTH')\n",
+ " num_classes = cfg.getint('MODEL', 'NUM_CLASSES')\n",
+ " in_channels = cfg.getint('MODEL', 'IN_CHANNELS')\n",
+ " net = LFC(weight_bit_width=weight_bit_width,\n",
+ " act_bit_width=act_bit_width,\n",
+ " in_bit_width=in_bit_width,\n",
+ " num_classes=num_classes,\n",
+ " in_ch=in_channels)\n",
+ " return net\n",
"\n"
]
}
],
"source": [
- "from models.LFC import LFC\n",
- "showSrc(LFC)"
+ "from brevitas_examples import bnn_pynq\n",
+ "showSrc(bnn_pynq.models.LFC)"
]
},
{
@@ -111,7 +158,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 8,
"metadata": {},
"outputs": [
{
@@ -267,18 +314,14 @@
")"
]
},
- "execution_count": 3,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "import torch\n",
- "\n",
- "trained_lfc_w1a1_checkpoint = \"/workspace/brevitas_cnv_lfc/pretrained_models/LFC_1W1A/checkpoints/best.tar\"\n",
- "lfc = LFC(weight_bit_width=1, act_bit_width=1, in_bit_width=1).eval()\n",
- "checkpoint = torch.load(trained_lfc_w1a1_checkpoint, map_location=\"cpu\")\n",
- "lfc.load_state_dict(checkpoint[\"state_dict\"])\n",
+ "from finn.util.test import get_test_model\n",
+ "lfc = get_test_model(netname = \"LFC\", wbits = 1, abits = 1, pretrained = True)\n",
"lfc"
]
},
@@ -291,7 +334,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 12,
"metadata": {},
"outputs": [
{
@@ -308,6 +351,7 @@
}
],
"source": [
+ "import torch\n",
"import matplotlib.pyplot as plt\n",
"from pkgutil import get_data\n",
"import onnx\n",
@@ -321,7 +365,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 13,
"metadata": {},
"outputs": [
{
@@ -331,7 +375,7 @@
" 0.0141])"
]
},
- "execution_count": 6,
+ "execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
@@ -346,7 +390,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
@@ -390,15 +434,15 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/workspace/brevitas_cnv_lfc/training_scripts/models/LFC.py:85: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
- " x = 2.0 * x - torch.tensor([1.0]).to(self.device)\n"
+ "/workspace/brevitas/brevitas_examples/bnn_pynq/models/LFC.py:80: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
+ " x = 2.0 * x - torch.tensor([1.0], device=x.device)\n"
]
}
],
@@ -418,7 +462,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
@@ -442,10 +486,10 @@
" "
],
"text/plain": [
- "<IPython.lib.display.IFrame at 0x7f3d330b6ac8>"
+ "<IPython.lib.display.IFrame at 0x7f3a27be9ac8>"
]
},
- "execution_count": 9,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
@@ -472,19 +516,19 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "input: \"37\"\n",
+ "input: \"40\"\n",
"input: \"38\"\n",
- "output: \"40\"\n",
- "op_type: \"MatMul\""
+ "output: \"41\"\n",
+ "op_type: \"Add\""
]
},
- "execution_count": 10,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
@@ -504,22 +548,16 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "array([[-1., -1., -1., ..., -1., -1., 1.],\n",
- " [-1., 1., -1., ..., 1., -1., -1.],\n",
- " [ 1., -1., 1., ..., -1., -1., -1.],\n",
- " ...,\n",
- " [ 1., 1., -1., ..., 1., 1., 1.],\n",
- " [-1., -1., 1., ..., 1., 1., -1.],\n",
- " [ 1., 1., -1., ..., 1., -1., -1.]], dtype=float32)"
+ "array(-0.5, dtype=float32)"
]
},
- "execution_count": 11,
+ "execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
@@ -537,16 +575,16 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "<DataType.BIPOLAR: 8>"
+ "<DataType.FLOAT32: 16>"
]
},
- "execution_count": 12,
+ "execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
@@ -557,16 +595,16 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "[784, 1024]"
+ "[]"
]
},
- "execution_count": 13,
+ "execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
@@ -584,7 +622,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
@@ -598,7 +636,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -624,10 +662,10 @@
" "
],
"text/plain": [
- "<IPython.lib.display.IFrame at 0x7f3d3380aef0>"
+ "<IPython.lib.display.IFrame at 0x7f3a27b49e10>"
]
},
- "execution_count": 15,
+ "execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
@@ -645,7 +683,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 23,
"metadata": {},
"outputs": [
{
@@ -656,7 +694,7 @@
" dtype=float32)"
]
},
- "execution_count": 16,
+ "execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
diff --git a/notebooks/end2end_example/cnv_end2end_example.ipynb b/notebooks/end2end_example/cnv_end2end_example.ipynb
index adb34f6d12ab9177490c07d67fbabc446eeb46ab..ce8c9decf4aaa6b7be2e556b6053abf380d0d373 100644
--- a/notebooks/end2end_example/cnv_end2end_example.ipynb
+++ b/notebooks/end2end_example/cnv_end2end_example.ipynb
@@ -55,7 +55,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
@@ -71,7 +71,7 @@
"source": [
"## 1. Brevitas Export, FINN Import and Tidy-Up\n",
"\n",
- "Similar to what we did in the TFC-w1a1 end-to-end notebook, we will start by exporting the [pretrained CNV-w1a1 network](https://github.com/maltanar/brevitas_cnv_lfc) to ONNX, importing that into FINN and running the \"tidy-up\" transformations to have a first look at the topology."
+ "Similar to what we did in the TFC-w1a1 end-to-end notebook, we will start by exporting the [pretrained CNV-w1a1 network](https://github.com/Xilinx/brevitas/tree/master/brevitas_examples/bnn_pynq) to ONNX, importing that into FINN and running the \"tidy-up\" transformations to have a first look at the topology."
]
},
{
@@ -83,8 +83,10 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "/workspace/brevitas_cnv_lfc/training_scripts/models/CNV.py:112: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
- " x = 2.0 * x - torch.tensor([1.0]).to(self.device)\n"
+ "Downloading: \"https://github.com/Xilinx/brevitas/releases/download/bnn_pynq-r0/cnv_1w1a-758c8fef.pth\" to /home/maltanar/.cache/torch/checkpoints/cnv_1w1a-758c8fef.pth\n",
+ "100%|██████████| 6227617/6227617 [00:01<00:00, 4429264.32it/s]\n",
+ "/workspace/brevitas/brevitas_examples/bnn_pynq/models/CNV.py:107: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
+ " x = 2.0 * x - torch.tensor([1.0], device=x.device)\n"
]
}
],
@@ -272,7 +274,7 @@
"from finn.transformation.fpgadataflow.create_dataflow_partition import (\n",
" CreateDataflowPartition,\n",
")\n",
- "from finn.transformation.move_reshape import MoveReshape\n",
+ "from finn.transformation.move_reshape import RemoveCNVtoFCFlatten\n",
"from finn.custom_op.registry import getCustomOp\n",
"\n",
"# choose the memory mode for the MVTU units, decoupled or const\n",
@@ -284,7 +286,7 @@
"model = model.transform(to_hls.InferConvInpGen())\n",
"model = model.transform(to_hls.InferStreamingMaxPool())\n",
"# get rid of Reshape(-1, 1) operation between hlslib nodes\n",
- "model = model.transform(MoveReshape())\n",
+ "model = model.transform(RemoveCNVtoFCFlatten())\n",
"parent_model = model.transform(CreateDataflowPartition())\n",
"parent_model.save(build_dir + \"/end2end_cnv_w1a1_dataflow_parent.onnx\")\n",
"sdp_node = parent_model.get_nodes_by_op_type(\"StreamingDataflowPartition\")[0]\n",
@@ -299,7 +301,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Notice the additional `MoveReshape` transformation that was not used for TFC-w1a1. In the last Netron visualization you may have noticed a `Reshape` operation towards the end of the network where the convolutional part of the network ends and the fully-connected layers started. That `Reshape` is essentialy a tensor flattening operation, which we can remove for the purposes of hardware implementation. We can examine the contents of the dataflow partition with Netron, and observe the `ConvolutionInputGenerator`, `StreamingFCLayer_Batch` and `StreamingMaxPool_Batch` nodes that implement the sliding window, matrix multiply and maxpool operations in hlslib. *Note that the StreamingFCLayer instances following the ConvolutionInputGenerator nodes are really implementing the convolutions, despite the name. The final three StreamingFCLayer instances implement actual FC layers.*"
+ "Notice the additional `RemoveCNVtoFCFlatten` transformation that was not used for TFC-w1a1. In the last Netron visualization you may have noticed a `Reshape` operation towards the end of the network where the convolutional part of the network ends and the fully-connected layers started. That `Reshape` is essentialy a tensor flattening operation, which we can remove for the purposes of hardware implementation. We can examine the contents of the dataflow partition with Netron, and observe the `ConvolutionInputGenerator`, `StreamingFCLayer_Batch` and `StreamingMaxPool_Batch` nodes that implement the sliding window, matrix multiply and maxpool operations in hlslib. *Note that the StreamingFCLayer instances following the ConvolutionInputGenerator nodes are really implementing the convolutions, despite the name. The final three StreamingFCLayer instances implement actual FC layers.*"
]
},
{
diff --git a/notebooks/end2end_example/tfc_end2end_example.ipynb b/notebooks/end2end_example/tfc_end2end_example.ipynb
index 6399d85ebdec74374115380d6e81a1228f54b0da..d573061487de204084e0d3242da8ad1b791f44d8 100644
--- a/notebooks/end2end_example/tfc_end2end_example.ipynb
+++ b/notebooks/end2end_example/tfc_end2end_example.ipynb
@@ -70,7 +70,7 @@
"metadata": {},
"source": [
"## 1. Brevitas export <a id='brev_exp'></a>\n",
- "FINN expects an ONNX model as input. This can be a model trained with [Brevitas](https://github.com/Xilinx/brevitas). Brevitas is a PyTorch library for quantization-aware training and the FINN Docker image comes with several [example Brevitas networks](https://github.com/maltanar/brevitas_cnv_lfc). To show the FINN end-to-end flow, we'll use the TFC-w1a1 model as example network.\n",
+ "FINN expects an ONNX model as input. This can be a model trained with [Brevitas](https://github.com/Xilinx/brevitas). Brevitas is a PyTorch library for quantization-aware training and the FINN Docker image comes with several [example Brevitas networks](https://github.com/Xilinx/brevitas/tree/master/brevitas_examples/bnn_pynq). To show the FINN end-to-end flow, we'll use the TFC-w1a1 model as example network.\n",
"\n",
"First a few things have to be imported. Then the model can be loaded with the pretrained weights."
]
@@ -84,8 +84,8 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "/workspace/brevitas_cnv_lfc/training_scripts/models/TFC.py:85: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
- " x = 2.0 * x - torch.tensor([1.0]).to(self.device)\n"
+ "/workspace/brevitas/brevitas_examples/bnn_pynq/models/TFC.py:80: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.\n",
+ " x = 2.0 * x - torch.tensor([1.0], device=x.device)\n"
]
}
],
@@ -132,7 +132,7 @@
" "
],
"text/plain": [
- "<IPython.lib.display.IFrame at 0x7fe1ad0b6e80>"
+ "<IPython.lib.display.IFrame at 0x7f8890385828>"
]
},
"execution_count": 3,
diff --git a/src/finn/transformation/bipolar_to_xnor.py b/src/finn/transformation/bipolar_to_xnor.py
index 4c7ebaf04e35f94e84e52e0b4520ee2369502120..8b65cfee17edd5d89fcca0bd86da12415d38fe78 100644
--- a/src/finn/transformation/bipolar_to_xnor.py
+++ b/src/finn/transformation/bipolar_to_xnor.py
@@ -35,6 +35,7 @@ from finn.transformation import Transformation
from finn.transformation.infer_shapes import InferShapes
from finn.transformation.infer_datatypes import InferDataTypes
from finn.util.basic import get_by_name
+from finn.custom_op.registry import getCustomOp
class ConvertBipolarMatMulToXnorPopcount(Transformation):
@@ -71,10 +72,19 @@ class ConvertBipolarMatMulToXnorPopcount(Transformation):
)
graph_modified = True
mt = mt_chain[-1]
- bin_dt_attr = "BINARY".encode("utf-8")
- get_by_name(mt.attribute, "out_dtype").s = bin_dt_attr
- get_by_name(mt.attribute, "out_scale").f = 1.0
- get_by_name(mt.attribute, "out_bias").f = 0
+ mt_inst = getCustomOp(mt)
+ # ensure old scale/bias were correct for BIPOLAR
+ scale_ok = mt_inst.get_nodeattr("out_scale") == 2.0
+ bias_ok = mt_inst.get_nodeattr("out_bias") == -1.0
+ assert (
+ scale_ok and bias_ok
+ ), """Unexpected scale/bias
+ attributes for BIPOLAR MultiThreshold node."""
+ # start conversion, set MT output to binary
+ # (this is what XnorPopcountMatMul expects)
+ mt_inst.set_nodeattr("out_dtype", "BINARY")
+ mt_inst.set_nodeattr("out_scale", 1.0)
+ mt_inst.set_nodeattr("out_bias", 0.0)
model.set_tensor_datatype(mm_input, DataType.BINARY)
# change node type and domain
n.op_type = "XnorPopcountMatMul"
diff --git a/src/finn/transformation/move_reshape.py b/src/finn/transformation/move_reshape.py
index 6a30fd93cc0bdc322b6ec7d892d42d3c3ca96fd6..2ddaf4f840f449d3f5ec5cb83eaf461d624eb7a2 100644
--- a/src/finn/transformation/move_reshape.py
+++ b/src/finn/transformation/move_reshape.py
@@ -17,7 +17,7 @@ def _is_fpgadataflow_node(node):
return False
-class MoveReshape(Transformation):
+class RemoveCNVtoFCFlatten(Transformation):
"""Removes a node that implements a (1, -1) reshape if it is
between two fpgadataflow nodes"""
@@ -27,13 +27,13 @@ class MoveReshape(Transformation):
graph_modified = False
for n in graph.node:
if n.op_type == "Reshape":
- graph_modified = True
shape = model.get_initializer(n.input[1])
if (shape == [1, -1]).all():
producer = model.find_producer(n.input[0])
if _is_fpgadataflow_node(producer) is True:
consumer = model.find_consumer(n.output[0])
if _is_fpgadataflow_node(consumer) is True:
+ graph_modified = True
consumer.input[0] = n.input[0]
graph.node.remove(n)
diff --git a/src/finn/util/test.py b/src/finn/util/test.py
index 34edc3cacdecc461d1254c35c026c56ff8813549..4cad01b1f7ec58da7ba6d5460c072faa01202c55 100644
--- a/src/finn/util/test.py
+++ b/src/finn/util/test.py
@@ -26,49 +26,38 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import torch
-from models.CNV import CNV
-from models.LFC import LFC
-from models.SFC import SFC
-from models.TFC import TFC
-
-
-def get_trained_checkpoint(netname, wbits, abits):
- """Returns the weights and activations from the FINN Brevitas test networks
- for given netname and the number of bits for weights and activations"""
- # TODO get from config instead, hardcoded to Docker path for now
- nname = "%s_%dW%dA" % (netname, wbits, abits)
- root = "/workspace/brevitas_cnv_lfc/pretrained_models/%s/checkpoints/best.tar"
- return root % nname
-
-
-def get_test_model_def_fxn(netname):
- """Returns the PyTorch model instantation function related to netname."""
- model_def_map = {"LFC": LFC, "SFC": SFC, "TFC": TFC, "CNV": CNV}
- return model_def_map[netname]
+from brevitas_examples import bnn_pynq
+
+# map of (wbits,abits) -> model
+example_map = {
+ ("CNV", 1, 1): bnn_pynq.cnv_1w1a,
+ ("CNV", 1, 2): bnn_pynq.cnv_1w2a,
+ ("CNV", 2, 2): bnn_pynq.cnv_2w2a,
+ ("LFC", 1, 1): bnn_pynq.lfc_1w1a,
+ ("LFC", 1, 2): bnn_pynq.lfc_1w2a,
+ ("SFC", 1, 1): bnn_pynq.sfc_1w1a,
+ ("SFC", 1, 2): bnn_pynq.sfc_1w2a,
+ ("SFC", 2, 2): bnn_pynq.sfc_2w2a,
+ ("TFC", 1, 1): bnn_pynq.tfc_1w1a,
+ ("TFC", 1, 2): bnn_pynq.tfc_1w2a,
+ ("TFC", 2, 2): bnn_pynq.tfc_2w2a,
+}
+
+
+def get_test_model(netname, wbits, abits, pretrained):
+ """Returns the model specified by input arguments from the Brevitas BNN-PYNQ
+ test networks. Pretrained weights loaded if pretrained is True."""
+ model_cfg = (netname, wbits, abits)
+ model_def_fxn = example_map[model_cfg]
+ fc = model_def_fxn(pretrained)
+ return fc.eval()
def get_test_model_trained(netname, wbits, abits):
- """Returns the pretrained model specified by input arguments loaded with weights
- and activations from the FINN Brevitas test networks."""
- model_def_fxn = get_test_model_def_fxn(netname)
- checkpoint_loc = get_trained_checkpoint(netname, wbits, abits)
- if netname == "CNV":
- ibits = 8
- else:
- ibits = abits
- fc = model_def_fxn(weight_bit_width=wbits, act_bit_width=abits, in_bit_width=ibits)
- checkpoint = torch.load(checkpoint_loc, map_location="cpu")
- fc.load_state_dict(checkpoint["state_dict"])
- return fc.eval()
+ "get_test_model with pretrained=True"
+ return get_test_model(netname, wbits, abits, pretrained=True)
def get_test_model_untrained(netname, wbits, abits):
- """Returns untrained model specified by input arguments."""
- model_def_fxn = get_test_model_def_fxn(netname)
- if netname == "CNV":
- ibits = 8
- else:
- ibits = abits
- fc = model_def_fxn(weight_bit_width=wbits, act_bit_width=abits, in_bit_width=ibits)
- return fc.eval()
+ "get_test_model with pretrained=False"
+ return get_test_model(netname, wbits, abits, pretrained=False)
diff --git a/tests/brevitas/test_brevitas_act_export.py b/tests/brevitas/test_brevitas_act_export.py
index 0415d70bfe6a543b4547f07dd99ff2525bef5994..e1cfc9db98a9c9d746b6f66ee071ddfb85cc5dbb 100644
--- a/tests/brevitas/test_brevitas_act_export.py
+++ b/tests/brevitas/test_brevitas_act_export.py
@@ -4,8 +4,8 @@ import torch
import brevitas.onnx as bo
from brevitas.nn import QuantHardTanh
from brevitas.core.restrict_val import RestrictValueType
+from brevitas.core.quant import QuantType
from brevitas.core.scaling import ScalingImplType
-from models.common import get_quant_type
import pytest
from finn.core.modelwrapper import ModelWrapper
import finn.core.onnx_exec as oxe
@@ -18,6 +18,14 @@ export_onnx_path = "test_act.onnx"
@pytest.mark.parametrize("narrow_range", [False, True])
@pytest.mark.parametrize("max_val", [1.0, 1 - 2 ** (-7)])
def test_brevitas_act_export(abits, narrow_range, max_val):
+ def get_quant_type(bit_width):
+ if bit_width is None:
+ return QuantType.FP
+ elif bit_width == 1:
+ return QuantType.BINARY
+ else:
+ return QuantType.INT
+
act_quant_type = get_quant_type(abits)
min_val = -1.0
ishape = (1, 10)
diff --git a/tests/custom_op/test_xnorpopcountmatmul.py b/tests/custom_op/test_xnorpopcountmatmul.py
index 6b59283667ac05f569e5c3d80dbfc1530616d045..37d9b7e5968bdb70023be9b70515410e941f51ce 100644
--- a/tests/custom_op/test_xnorpopcountmatmul.py
+++ b/tests/custom_op/test_xnorpopcountmatmul.py
@@ -48,10 +48,6 @@ from finn.transformation.streamline.sign_to_thres import ConvertSignToThres
from finn.util.test import get_test_model_trained
export_onnx_path = "test_output_lfc.onnx"
-# TODO get from config instead, hardcoded to Docker path for now
-trained_lfc_checkpoint = (
- "/workspace/brevitas_cnv_lfc/pretrained_models/LFC_1W1A/checkpoints/best.tar"
-)
def test_xnorpopcountmatmul():
diff --git a/tests/end2end/test_end2end_cnv_w1a1.py b/tests/end2end/test_end2end_cnv_w1a1.py
index d7f59ef35aaf61891937dcaa105cf1392133e732..703cb7ad92d6f4ec6dd67a05345f323a73ee178d 100644
--- a/tests/end2end/test_end2end_cnv_w1a1.py
+++ b/tests/end2end/test_end2end_cnv_w1a1.py
@@ -41,7 +41,7 @@ from finn.custom_op.registry import getCustomOp
from finn.core.onnx_exec import execute_onnx
from finn.transformation.double_to_single_float import DoubleToSingleFloat
from finn.transformation.infer_shapes import InferShapes
-from finn.transformation.move_reshape import MoveReshape
+from finn.transformation.move_reshape import RemoveCNVtoFCFlatten
from finn.transformation.fold_constants import FoldConstants
from finn.transformation.general import GiveReadableTensorNames, GiveUniqueNodeNames
from finn.transformation.streamline import Streamline
@@ -117,7 +117,7 @@ def test_end2end_cnv_w1a1_convert_to_hls_layers():
model = model.transform(to_hls.InferQuantizedStreamingFCLayer(mem_mode))
model = model.transform(to_hls.InferConvInpGen())
model = model.transform(to_hls.InferStreamingMaxPool())
- model = model.transform(MoveReshape())
+ model = model.transform(RemoveCNVtoFCFlatten())
model.save(build_dir + "/end2end_cnv_w1a1_hls_layers.onnx")
diff --git a/tests/transformation/test_infer_datatypes.py b/tests/transformation/test_infer_datatypes.py
index ae8a52882a9126470dad6ca15d8c35000a8edaff..77b6a94f8ed891a4fe761fe864a6e18d35e84382 100644
--- a/tests/transformation/test_infer_datatypes.py
+++ b/tests/transformation/test_infer_datatypes.py
@@ -39,10 +39,6 @@ from finn.transformation.infer_shapes import InferShapes
from finn.util.test import get_test_model_trained
export_onnx_path = "test_output_lfc.onnx"
-# TODO get from config instead, hardcoded to Docker path for now
-trained_lfc_checkpoint = (
- "/workspace/brevitas_cnv_lfc/pretrained_models/LFC_1W1A/checkpoints/best.tar"
-)
def test_infer_datatypes():
diff --git a/tests/transformation/test_sign_to_thres.py b/tests/transformation/test_sign_to_thres.py
index 1033a313560c714b02e256e5940694868fa41cbf..b10840df37a695986e54c0bdaa68baa0538f90f2 100644
--- a/tests/transformation/test_sign_to_thres.py
+++ b/tests/transformation/test_sign_to_thres.py
@@ -42,10 +42,6 @@ from finn.util.test import get_test_model_trained
export_onnx_path = "test_output_lfc.onnx"
transformed_onnx_path = "test_output_lfc_transformed.onnx"
-# TODO get from config instead, hardcoded to Docker path for now
-trained_lfc_checkpoint = (
- "/workspace/brevitas_cnv_lfc/pretrained_models/LFC_1W1A/checkpoints/best.tar"
-)
def test_sign_to_thres():