diff --git a/src/finn/transformation/streamline/reorder.py b/src/finn/transformation/streamline/reorder.py
index cc95d34b784b47c9baeb6c1076915db8b1d09d57..3b5d32b1cb6784fcc22cc8dcff04c2842e729d6d 100644
--- a/src/finn/transformation/streamline/reorder.py
+++ b/src/finn/transformation/streamline/reorder.py
@@ -31,7 +31,9 @@ import warnings
from onnx import helper as oh
from finn.transformation import Transformation
+import finn.core.data_layout as DataLayout
from finn.transformation.infer_shapes import InferShapes
+from finn.transformation.infer_datatypes import InferDataTypes
from finn.transformation.infer_data_layouts import InferDataLayouts
from finn.core.datatype import DataType
from finn.core.onnx_exec import execute_node
@@ -674,7 +676,95 @@ class MoveMaxPoolPastMultiThreshold(Transformation):
model = model.transform(InferShapes())
return (model, graph_modified)
+class MoveFlattenPastAffine(Transformation):
+ """Moves a node that implements a (1, -1) reshape past a MatMul, Mul or Add node."""
+ def apply(self, model):
+ graph = model.graph
+ graph_modified = False
+ node_ind = 0
+ for n in graph.node:
+ node_ind += 1
+ if (
+ n.op_type == "Flatten"
+ and not model.is_fork_node(n)
+ and not model.is_join_node(n)
+ ):
+ consumer = model.find_consumer(n.output[0])
+ if (
+ consumer is not None
+ and (
+ consumer.op_type == "MatMul"
+ or consumer.op_type == "Mul"
+ or consumer.op_type == "Add"
+ )
+ and not model.is_join_node(consumer)
+ ):
+ # move flatten past operation and rewire tensors
+ start_name = n.input[0]
+ # check if datalyout is set to NHWC and H=W=1
+ datalayout = model.get_tensor_layout(start_name)
+ if datalayout == DataLayout.NHWC:
+ (b, h, w, c) = model.get_tensor_shape(start_name)
+ if h != 1 or w != 1:
+ warnings.warn(
+ """The Transformation can only be performed if
+ H=W=1."""
+ )
+ continue
+ else:
+ warnings.warn(
+ """The Transformation can only be performed on
+ operations that operate on data layout NHWC."""
+ )
+ continue
+ middle_name = n.output[0]
+ end_name = consumer.output[0]
+ op_param_name = consumer.input[1]
+ A = model.get_initializer(op_param_name)
+ if A is None:
+ warnings.warn("Param is not constant, skipping")
+ continue
+ op_in_dt = model.get_tensor_datatype(consumer.input[0])
+ op_out_dt = model.get_tensor_datatype(consumer.output[0])
+ start_shape = model.get_tensor_shape(start_name)
+ dummy_in = np.random.uniform(low=0, high=1, size=(start_shape))
+
+ if consumer.op_type == "MatMul":
+ dummy_out = np.matmul(dummy_in, A)
+ elif consumer.op_type == "Mul":
+ dummy_out = dummy_in * A
+ elif consumer.op_type == "Add":
+ dummy_out = dummy_in + A
+
+ new_op = oh.make_node(
+ consumer.op_type,
+ [start_name, op_param_name],
+ [middle_name],
+ name=consumer.name,
+ )
+ new_flatten = oh.make_node("Flatten", [middle_name], [end_name])
+ graph.node.insert(node_ind, new_op)
+ graph.node.insert(node_ind + 1, new_flatten)
+ model.set_tensor_shape(middle_name, dummy_out.shape)
+ # because a flatten node doesn't change the datatype we need
+ # only the datatype of the op node
+ model.set_tensor_datatype(start_name, op_in_dt)
+ model.set_tensor_datatype(middle_name, op_out_dt)
+ model.set_tensor_datatype(end_name, op_out_dt)
+ # set datalayout
+ model.set_tensor_layout(start_name, DataLayout.NHWC)
+ model.set_tensor_layout(middle_name, DataLayout.NHWC)
+ # remove old nodes
+ graph.node.remove(n)
+ graph.node.remove(consumer)
+ graph_modified = True
+
+ model = model.transform(InferShapes())
+ model = model.transform(InferDataTypes())
+ model = model.transform(InferDataLayouts())
+ return (model, graph_modified)
+
class MoveTransposePastScalarMul(Transformation):
"""Moves a Transpose node past a scalar Mul node"""
@@ -736,3 +826,4 @@ class MoveTransposePastScalarMul(Transformation):
model = model.transform(InferDataLayouts())
model = model.transform(InferShapes())
return (model, graph_modified)
+
diff --git a/tests/transformation/test_move_flatten_past_affine.py b/tests/transformation/test_move_flatten_past_affine.py
new file mode 100644
index 0000000000000000000000000000000000000000..b2d5e51613d41f3f2db3dabcef7b982ec2816b19
--- /dev/null
+++ b/tests/transformation/test_move_flatten_past_affine.py
@@ -0,0 +1,106 @@
+# Copyright (c) 2020, Xilinx
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# * Neither the name of FINN nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import pytest
+
+import numpy as np
+from onnx import TensorProto, helper
+
+from finn.core.modelwrapper import ModelWrapper
+from finn.core.datatype import DataType
+import finn.core.data_layout as DataLayout
+from finn.util.basic import gen_finn_dt_tensor
+from finn.transformation.infer_shapes import InferShapes
+from finn.transformation.infer_datatypes import InferDataTypes
+from finn.transformation.infer_data_layouts import InferDataLayouts
+from finn.transformation.general import GiveUniqueNodeNames, GiveReadableTensorNames
+from finn.transformation.streamline.reorder import MoveFlattenPastAffine
+import finn.core.onnx_exec as oxe
+
+# data layout
+@pytest.mark.parametrize("data_layout", [DataLayout.NHWC, DataLayout.NCHW])
+# batch size
+@pytest.mark.parametrize("batch_size", [1, 2])
+def test_move_flatten_past_affine(data_layout, batch_size):
+ if data_layout == DataLayout.NHWC:
+ ishape = [batch_size, 1, 1, 1024]
+ oshape = [batch_size, 1000]
+ else:
+ ishape = [batch_size, 1024, 1, 1]
+ oshape = [batch_size, 1000]
+
+ inp = helper.make_tensor_value_info("inp", TensorProto.FLOAT, ishape)
+ a0 = helper.make_tensor_value_info("a1", TensorProto.FLOAT, [1024, 1000])
+ a1 = helper.make_tensor_value_info("a2", TensorProto.FLOAT, [])
+ a2 = helper.make_tensor_value_info("a3", TensorProto.FLOAT, [1000])
+ outp = helper.make_tensor_value_info("outp", TensorProto.FLOAT, oshape)
+
+ flatten_node = helper.make_node("Flatten", ["inp"], ["flatten_out"])
+ matmul_node = helper.make_node("MatMul", ["flatten_out", "a0"], ["matmul_out"])
+ mul_node = helper.make_node("Mul", ["matmul_out", "a1"], ["mul_out"])
+ add_node = helper.make_node("Add", ["mul_out", "a2"], ["outp"])
+
+ graph = helper.make_graph(
+ nodes=[flatten_node, matmul_node, mul_node, add_node],
+ name="move-reshape-graph",
+ inputs=[inp],
+ outputs=[outp],
+ value_info=[a0, a1, a2],
+ )
+
+ model = helper.make_model(graph, producer_name="move_reshape_model")
+ model = ModelWrapper(model)
+
+ # initialize values
+ a0_values = gen_finn_dt_tensor(DataType.TERNARY, [1024, 1000])
+ model.set_initializer("a0", a0_values)
+ a1_values = np.random.uniform(low=0.1, high=0.99, size=(1)).astype(np.float32)
+ model.set_initializer("a1", a1_values)
+ a2_values = np.random.uniform(low=-1, high=1, size=(1000)).astype(np.float32)
+ model.set_initializer("a2", a2_values)
+
+ model.set_tensor_datatype("inp", DataType.INT2)
+ model.set_tensor_layout("inp", data_layout)
+ model = model.transform(InferShapes())
+ model = model.transform(InferDataTypes())
+ model = model.transform(InferDataLayouts())
+ model = model.transform(GiveUniqueNodeNames())
+ model = model.transform(GiveReadableTensorNames())
+
+ # compare execution before and after transformation
+ inp_values = gen_finn_dt_tensor(DataType.INT2, ishape)
+ idict = {model.graph.input[0].name: inp_values}
+ model_transformed = model.transform(MoveFlattenPastAffine())
+ assert oxe.compare_execution(model, model_transformed, idict)
+
+ # depending on data layout check if graph is transformed or not
+ if data_layout == DataLayout.NHWC:
+ # check if nodes have new order in transformed graph
+ assert model.graph != model_transformed.graph
+ assert model_transformed.graph.node[-1].op_type == "Flatten"
+ else:
+ assert model.graph == model_transformed.graph