diff --git a/notebooks/end2end_example/tfc_end2end_example.ipynb b/notebooks/end2end_example/tfc_end2end_example.ipynb
index d573061487de204084e0d3242da8ad1b791f44d8..c84efc964b1f57b7ed385521fc5214fdc2396590 100644
--- a/notebooks/end2end_example/tfc_end2end_example.ipynb
+++ b/notebooks/end2end_example/tfc_end2end_example.ipynb
@@ -132,7 +132,7 @@
" "
],
"text/plain": [
- "<IPython.lib.display.IFrame at 0x7f8890385828>"
+ "<IPython.lib.display.IFrame at 0x7f7cc4290940>"
]
},
"execution_count": 3,
@@ -293,7 +293,7 @@
" "
],
"text/plain": [
- "<IPython.lib.display.IFrame at 0x7fe1ad0639e8>"
+ "<IPython.lib.display.IFrame at 0x7f7c6c567f28>"
]
},
"execution_count": 6,
@@ -333,9 +333,10 @@
" ConvertDivToMul(),\n",
" BatchNormToAffine(),\n",
" ConvertSignToThres(),\n",
+ " AbsorbSignBiasIntoMultiThreshold(),\n",
" MoveAddPastMul(),\n",
" MoveScalarAddPastMatMul(),\n",
- " MoveScalarAddPastConv(),\n",
+ " MoveAddPastConv(),\n",
" MoveScalarMulPastMatMul(),\n",
" MoveScalarMulPastConv(),\n",
" MoveAddPastMul(),\n",
@@ -350,6 +351,7 @@
" ]\n",
" for trn in streamline_transformations:\n",
" model = model.transform(trn)\n",
+ " model = model.transform(RemoveIdentityOps())\n",
" model = model.transform(GiveUniqueNodeNames())\n",
" model = model.transform(GiveReadableTensorNames())\n",
" model = model.transform(InferDataTypes())\n",
@@ -400,7 +402,7 @@
" "
],
"text/plain": [
- "<IPython.lib.display.IFrame at 0x7fe1346e4ef0>"
+ "<IPython.lib.display.IFrame at 0x7f7c6c0bf898>"
]
},
"execution_count": 8,
@@ -454,7 +456,7 @@
" "
],
"text/plain": [
- "<IPython.lib.display.IFrame at 0x7fe1346f7780>"
+ "<IPython.lib.display.IFrame at 0x7f7c6c0e5c18>"
]
},
"execution_count": 9,
diff --git a/src/finn/transformation/streamline/__init__.py b/src/finn/transformation/streamline/__init__.py
index d9c12a20975084705b801c0ff027d4b99aff9490..d7686eaadcbc800542ab96c5f45145857412b773 100644
--- a/src/finn/transformation/streamline/__init__.py
+++ b/src/finn/transformation/streamline/__init__.py
@@ -53,7 +53,7 @@ from finn.transformation.streamline.reorder import (
MoveAddPastMul,
MoveScalarMulPastMatMul,
MoveScalarAddPastMatMul,
- MoveScalarAddPastConv,
+ MoveAddPastConv,
MoveScalarMulPastConv,
)
@@ -75,7 +75,7 @@ class Streamline(Transformation):
AbsorbSignBiasIntoMultiThreshold(),
MoveAddPastMul(),
MoveScalarAddPastMatMul(),
- MoveScalarAddPastConv(),
+ MoveAddPastConv(),
MoveScalarMulPastMatMul(),
MoveScalarMulPastConv(),
MoveAddPastMul(),
diff --git a/src/finn/transformation/streamline/reorder.py b/src/finn/transformation/streamline/reorder.py
index 253988185977c6b9d28505d46e00b2ab11c3a76b..2b03532ce3ba7d5159e5ae57e61c2af9c8c37fce 100644
--- a/src/finn/transformation/streamline/reorder.py
+++ b/src/finn/transformation/streamline/reorder.py
@@ -225,8 +225,8 @@ class MoveScalarAddPastMatMul(Transformation):
return (model, graph_modified)
-class MoveScalarAddPastConv(Transformation):
- """Move scalar add operations past conv operations. We want to have adds
+class MoveAddPastConv(Transformation):
+ """Move scalar and channelwise add operations past conv operations. We want to have adds
next to each other such that they can be collapsed into a single add."""
def apply(self, model):
@@ -251,6 +251,8 @@ class MoveScalarAddPastConv(Transformation):
add_weight_name = n.input[1]
conv_in_name = consumer.input[0]
conv_in_shape = model.get_tensor_shape(conv_in_name)
+ # assume datalayout to be NCHW
+ channels = conv_in_shape[1]
A = model.get_initializer(add_weight_name)
if A is None:
warnings.warn("Add param is not constant, skipping")
@@ -263,11 +265,17 @@ class MoveScalarAddPastConv(Transformation):
pads = list(get_by_name(consumer.attribute, "pads").ints)
if sum(pads) == 0:
using_padding = False
- if all(x == 1 for x in A.shape) and not using_padding:
+ if (
+ all(x == 1 for x in A.shape) or A.shape == (1, channels, 1, 1)
+ ) and not using_padding:
# create a tensor filled with the add constant, in
# the shape expected by the convolution
conv_in_const = np.zeros(conv_in_shape, dtype=np.float32)
- conv_in_const.fill(A.item())
+ if A.shape == (1, channels, 1, 1):
+ for ch in range(channels):
+ conv_in_const[0][ch].fill(A[0][ch].item())
+ else:
+ conv_in_const.fill(A.item())
# create an execution context and put in const input
exec_ctx = model.make_empty_exec_context()
exec_ctx[conv_in_name] = conv_in_const
diff --git a/tests/transformation/test_move_chw_add_past_conv.py b/tests/transformation/test_move_chw_add_past_conv.py
new file mode 100644
index 0000000000000000000000000000000000000000..b626f7e5b8564739ec383aaddfc262d642bf47cc
--- /dev/null
+++ b/tests/transformation/test_move_chw_add_past_conv.py
@@ -0,0 +1,109 @@
+# Copyright (c) 2020, Xilinx
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# * Neither the name of FINN nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import pytest
+
+import numpy as np
+from onnx import helper, TensorProto
+
+from finn.core.modelwrapper import ModelWrapper
+from finn.transformation.infer_shapes import InferShapes
+from finn.transformation.streamline.reorder import MoveAddPastConv
+from finn.custom_op.im2col import compute_conv_output_dim
+import finn.core.onnx_exec as oxe
+
+
+# input dimension
+@pytest.mark.parametrize("idim", [4, 7])
+# kernel size
+@pytest.mark.parametrize("k", [2, 3])
+# stride
+@pytest.mark.parametrize("s", [1, 2])
+# input channels
+@pytest.mark.parametrize("ich", [2, 4])
+# output channels
+@pytest.mark.parametrize("och", [2, 3])
+def test_move_chw_add_past_conv(idim, k, s, ich, och):
+ odim = compute_conv_output_dim(idim, k, s)
+
+ ishape = [1, ich, idim, idim]
+ oshape = [1, och, odim, odim]
+ add_param_shape = [1, ich, 1, 1]
+ conv_param_shape = [och, ich, k, k]
+
+ inp = helper.make_tensor_value_info("inp", TensorProto.FLOAT, ishape)
+ outp = helper.make_tensor_value_info("outp", TensorProto.FLOAT, oshape)
+ a0 = helper.make_tensor_value_info("a0", TensorProto.FLOAT, add_param_shape)
+ a1 = helper.make_tensor_value_info("a1", TensorProto.FLOAT, conv_param_shape)
+
+ conv_config = {}
+ conv_config["dilations"] = [1, 1]
+ conv_config["group"] = 1
+ conv_config["kernel_shape"] = [k, k]
+ conv_config["pads"] = [0, 0, 0, 0]
+ conv_config["strides"] = [s, s]
+
+ add_node = helper.make_node("Add", ["inp", "a0"], ["add_out"])
+ conv_node = helper.make_node("Conv", ["add_out", "a1"], ["outp"], **conv_config)
+
+ model = helper.make_model(
+ helper.make_graph(
+ nodes=[add_node, conv_node],
+ name="move-add-graph",
+ inputs=[inp],
+ outputs=[outp],
+ value_info=[a0, a1],
+ )
+ )
+
+ model = ModelWrapper(model)
+ # initialize model
+ a0_values = np.random.uniform(low=0, high=1, size=tuple(add_param_shape)).astype(
+ np.float32
+ )
+ model.set_initializer("a0", a0_values)
+ a1_values = np.random.uniform(low=0, high=1, size=tuple(conv_param_shape)).astype(
+ np.float32
+ )
+ model.set_initializer("a1", a1_values)
+
+ model = model.transform(InferShapes())
+
+ # execution before transformation
+ inp_values = np.random.uniform(low=0, high=1, size=tuple(ishape)).astype(np.float32)
+ idict = {model.graph.input[0].name: inp_values}
+ odict = oxe.execute_onnx(model, idict)
+ y_before = odict[model.graph.output[0].name]
+
+ model = model.transform(MoveAddPastConv())
+ odict = oxe.execute_onnx(model, idict)
+ y_after = odict[model.graph.output[0].name]
+
+ assert np.isclose(y_before, y_after).all()
+ assert model.graph.node[0].op_type == "Conv"
+ assert model.graph.node[1].op_type == "Add"
diff --git a/tests/transformation/test_move_scalar_past_conv.py b/tests/transformation/test_move_scalar_past_conv.py
index 0f50642d2b9d1583030630cb4927c2b86667e71a..94fee7907d1ed1cccbf95520e903c7d9b43d8f7d 100644
--- a/tests/transformation/test_move_scalar_past_conv.py
+++ b/tests/transformation/test_move_scalar_past_conv.py
@@ -7,14 +7,14 @@ import finn.core.onnx_exec as ox
from finn.core.modelwrapper import ModelWrapper
from finn.transformation.infer_shapes import InferShapes
from finn.transformation.streamline import (
- MoveScalarAddPastConv,
+ MoveAddPastConv,
MoveScalarMulPastConv,
)
@pytest.mark.parametrize("padding", [False, True])
@pytest.mark.parametrize(
- "test_args", [("Add", MoveScalarAddPastConv()), ("Mul", MoveScalarMulPastConv())],
+ "test_args", [("Add", MoveAddPastConv()), ("Mul", MoveScalarMulPastConv())],
)
def test_move_scalar_past_conv(test_args, padding):
scalar_op = test_args[0]
@@ -83,8 +83,8 @@ def test_move_scalar_past_conv(test_args, padding):
assert new_model.graph.node[2].op_type == "Conv"
else:
assert new_model.graph.node[0].op_type == "Conv"
- assert new_model.graph.node[1].op_type == scalar_op
- assert new_model.graph.node[2].op_type == "Conv"
+ assert new_model.graph.node[1].op_type == "Conv"
+ assert new_model.graph.node[2].op_type == scalar_op
else:
assert new_model.graph.node[0].op_type == "Conv"
assert new_model.graph.node[1].op_type == "Conv"
@@ -92,7 +92,7 @@ def test_move_scalar_past_conv(test_args, padding):
@pytest.mark.parametrize(
- "test_args", [("Add", MoveScalarAddPastConv()), ("Mul", MoveScalarMulPastConv())],
+ "test_args", [("Add", MoveAddPastConv()), ("Mul", MoveScalarMulPastConv())],
)
def test_move_scalar_past_conv_only_if_linear(test_args):
scalar_op = test_args[0]