Add verification options to dataflow build (#257)

* [Build] add verification-related options to DataflowBuildConfig

* [Build] add verification to build steps

* [Test] build_dataflow: check for verification outputs

* [Docs] add documentation on build verification steps

docs/finn/command_line.rst

@@ -132,6 +132,38 @@ build configuration), and are detailed below.
 
   * ``deploy/`` -- deployment package folder with a bitfile and driver, ready to be copied to target hardware platform
 
+Verification of intermediate steps
+----------------------------------
+
+FINN dataflow builds go through many steps before the bitfile is generated,
+and the flow may produce erronous models due to bugs or unsupported features.
+When running new models throught this process it's a good idea to enable the
+verification features of the dataflow build. In this way, FINN will use the
+input you provide to run through the intermediate models, produce some output
+and compare it against the expected output that you provide.
+
+This is achieved by setting up the following members of the build configuration:
+
+* Set ``verify_steps`` to be a list of :py:mod:`finn.builder.build_dataflow.VerificationStepType`
+  where each element in the list indicates the output of a particular step
+  that will be verified. See the documentation of the ``VerificationStepType``
+  for more information.
+* Set ``verify_input_npy`` to the .npy filename to use as the test input to the
+  verification process. We recommend using a single input example as the
+  verification execution time can be lengthy for rtlsim, especially for larger
+  networks. The shape of the numpy array must match the expected shape by
+  the model.
+* Set ``verify_expected_output_npy`` to the .npy filename to use as the "golden"
+  output that the generated outputs will be compared against. The shape of the
+  numpy array must match the produced output shape of the model.
+
+The output of the verification is twofold:
+
+* A message like ``Verification for folded_hls_cppsim : SUCCESS`` will appear in
+  the build logfile.
+* The output generated by the model at each verified step will be saved as a
+  .npy file under ``verification_output/`` where each file created will indicate
+  the verification step and the result of the verification (FAIL/SUCCESS).
 
 Advanced mode
 --------------

src/finn/builder/build_dataflow_config.py

@@ -32,6 +32,8 @@ from finn.transformation.fpgadataflow.vitis_build import VitisOptStrategy
 from enum import Enum
 from dataclasses import dataclass
 from dataclasses_json import dataclass_json
+import os
+import numpy as np
 
 
 class ShellFlowType(str, Enum):
@@ -82,6 +84,19 @@ class LargeFIFOMemStyle(str, Enum):
     URAM = "ultra"
 
 
+class VerificationStepType(str, Enum):
+    "Steps at which FINN ONNX execution can be launched for verification."
+
+    #: verify after step_tidy_up, using Python execution
+    TIDY_UP_PYTHON = "initial_python"
+    #: verify after step_streamline , using Python execution
+    STREAMLINED_PYTHON = "streamlined_python"
+    #: verify after step_apply_folding_config, using C++ for each HLS node
+    FOLDED_HLS_CPPSIM = "folded_hls_cppsim"
+    #: verify after step_create_stitched_ip, using stitched-ip Verilog
+    STITCHED_IP_RTLSIM = "stitched_ip_rtlsim"
+
+
 #: List of steps that will be run as part of the standard dataflow build, in the
 #: specified order. Use the `steps` as part of build config to restrict which
 #: steps will be run.
@@ -138,6 +153,19 @@ class DataflowBuildConfig:
     #: that will override the target_fps setting here.
     target_fps: Optional[int] = None
 
+    #: (Optional) At which steps the generated intermediate output model
+    #: will be verified. See documentation of VerificationStepType for
+    #: available options.
+    verify_steps: Optional[List[VerificationStepType]] = None
+
+    #: (Optional) Name of .npy file that will be used as the input for
+    #: verification. Only required if verify_steps is not empty.
+    verify_input_npy: Optional[str] = "input.npy"
+
+    #: (Optional) Name of .npy file that will be used as the expected output for
+    #: verification. Only required if verify_steps is not empty.
+    verify_expected_output_npy: Optional[str] = "expected_output.npy"
+
     #: (Optional) Control the maximum width of the per-PE MVAU stream while
     #: exploring the parallelization attributes to reach target_fps
     #: Only relevant if target_fps is specified.
@@ -266,3 +294,24 @@ class DataflowBuildConfig:
             VitisOptStrategyCfg.BUILD_SPEED: VitisOptStrategy.BUILD_SPEED,
         }
         return name_to_strategy[self.vitis_opt_strategy]
+
+    def _resolve_verification_steps(self):
+        if self.verify_steps is None:
+            return []
+        else:
+            return self.verify_steps
+
+    def _resolve_verification_io_pair(self):
+        if self.verify_steps is None:
+            return None
+        else:
+            assert os.path.isfile(self.verify_input_npy), (
+                "verify_input_npy not found: " + self.verify_input_npy
+            )
+            verify_input_npy = np.load(self.verify_input_npy)
+            assert os.path.isfile(self.verify_expected_output_npy), (
+                "verify_expected_output_npy not found: "
+                + self.verify_expected_output_npy
+            )
+            verify_expected_output_npy = np.load(self.verify_expected_output_npy)
+            return (verify_input_npy, verify_expected_output_npy)

src/finn/builder/build_dataflow_steps.py

@@ -88,8 +88,47 @@ from finn.builder.build_dataflow_config import (
     DataflowBuildConfig,
     DataflowOutputType,
     ShellFlowType,
+    VerificationStepType,
 )
 from finn.transformation.fpgadataflow.annotate_cycles import AnnotateCycles
+from finn.core.onnx_exec import execute_onnx
+import numpy as np
+from finn.util.test import execute_parent
+from finn.transformation.fpgadataflow.prepare_cppsim import PrepareCppSim
+from finn.transformation.fpgadataflow.compile_cppsim import CompileCppSim
+from finn.transformation.fpgadataflow.set_exec_mode import SetExecMode
+from finn.transformation.fpgadataflow.prepare_rtlsim import PrepareRTLSim
+from copy import deepcopy
+
+
+def verify_step(
+    model: ModelWrapper, cfg: DataflowBuildConfig, step_name: str, need_parent: bool
+):
+    print("Running verification for " + step_name)
+    verify_out_dir = cfg.output_dir + "/verification_output"
+    intermediate_models_dir = cfg.output_dir + "/intermediate_models"
+    os.makedirs(verify_out_dir, exist_ok=True)
+    (in_npy, exp_out_npy) = cfg._resolve_verification_io_pair()
+    if need_parent:
+        assert (
+            cfg.save_intermediate_models
+        ), "Enable save_intermediate_models for verification"
+        parent_model_fn = intermediate_models_dir + "/dataflow_parent.onnx"
+        child_model_fn = intermediate_models_dir + "/verify_%s.onnx" % step_name
+        model.save(child_model_fn)
+        out_npy = execute_parent(parent_model_fn, child_model_fn, in_npy)
+    else:
+        inp_tensor_name = model.graph.input[0].name
+        out_tensor_name = model.graph.output[0].name
+        inp_dict = {inp_tensor_name: in_npy}
+        out_dict = execute_onnx(model, inp_dict)
+        out_npy = out_dict[out_tensor_name]
+    res = np.isclose(exp_out_npy, out_npy, atol=1e-3).all()
+    res_to_str = {True: "SUCCESS", False: "FAIL"}
+    res_str = res_to_str[res]
+    verification_output_fn = verify_out_dir + "/verify_%s_%s.npy" % (step_name, res_str)
+    np.save(verification_output_fn, out_npy)
+    print("Verification for %s : %s" % (step_name, res_str))
 
 
 def step_tidy_up(model: ModelWrapper, cfg: DataflowBuildConfig):
@@ -103,6 +142,10 @@ def step_tidy_up(model: ModelWrapper, cfg: DataflowBuildConfig):
     model = model.transform(GiveReadableTensorNames())
     model = model.transform(InferDataTypes())
     model = model.transform(RemoveStaticGraphInputs())
+
+    if VerificationStepType.TIDY_UP_PYTHON in cfg._resolve_verification_steps():
+        verify_step(model, cfg, "initial_python", need_parent=False)
+
     return model
 
 
@@ -127,6 +170,10 @@ def step_streamline(model: ModelWrapper, cfg: DataflowBuildConfig):
     model = model.transform(absorb.AbsorbScalarMulAddIntoTopK())
     model = model.transform(InferDataLayouts())
     model = model.transform(RemoveUnusedTensors())
+
+    if VerificationStepType.STREAMLINED_PYTHON in cfg._resolve_verification_steps():
+        verify_step(model, cfg, "streamlined_python", need_parent=False)
+
     return model
 
 
@@ -196,6 +243,13 @@ def step_apply_folding_config(model: ModelWrapper, cfg: DataflowBuildConfig):
     if cfg.folding_config_file is not None:
         model = model.transform(GiveUniqueNodeNames())
         model = model.transform(ApplyConfig(cfg.folding_config_file))
+
+    if VerificationStepType.FOLDED_HLS_CPPSIM in cfg._resolve_verification_steps():
+        # prepare cppsim
+        model = model.transform(PrepareCppSim())
+        model = model.transform(CompileCppSim())
+        model = model.transform(SetExecMode("cppsim"))
+        verify_step(model, cfg, "folded_hls_cppsim", need_parent=True)
     return model
 
 
@@ -323,6 +377,20 @@ def step_create_stitched_ip(model: ModelWrapper, cfg: DataflowBuildConfig):
         # TODO copy all ip sources into output dir? as zip?
         copytree(model.get_metadata_prop("vivado_stitch_proj"), stitched_ip_dir)
         print("Vivado stitched IP written into " + stitched_ip_dir)
+    if VerificationStepType.STITCHED_IP_RTLSIM in cfg._resolve_verification_steps():
+        # prepare ip-stitched rtlsim
+        verify_model = deepcopy(model)
+        # rtlsim only supports impl_style=rtl for StreamingFIFO, ensure that
+        for fifo_layer in verify_model.get_nodes_by_op_type("StreamingFIFO"):
+            getCustomOp(fifo_layer).set_nodeattr("impl_style", "rtl")
+        # similarly for StreamingDataWidthConverter with impl_style=hls
+        for dwc_layer in verify_model.get_nodes_by_op_type(
+            "StreamingDataWidthConverter_Batch"
+        ):
+            getCustomOp(dwc_layer).set_nodeattr("impl_style", "hls")
+        verify_model = verify_model.transform(PrepareRTLSim())
+        verify_model.set_metadata_prop("exec_mode", "rtlsim")
+        verify_step(verify_model, cfg, "stitched_ip_rtlsim", need_parent=True)
     return model
 
 

src/finn/qnn-data/build_dataflow/dataflow_build_config.json

@@ -6,6 +6,12 @@
   "board": "Pynq-Z1",
   "standalone_thresholds": true,
   "shell_flow_type": "vivado_zynq",
+  "verify_steps": [
+    "initial_python",
+    "streamlined_python",
+    "folded_hls_cppsim",
+    "stitched_ip_rtlsim"
+  ],
   "generate_outputs": [
     "estimate_reports",
     "stitched_ip",

tests/util/test_build_dataflow.py

@@ -59,3 +59,9 @@ def test_build_dataflow_directory():
     assert os.path.isfile(output_dir + "/bitfile/finn-accel.hwh")
     assert os.path.isfile(output_dir + "/report/post_synth_resources.xml")
     assert os.path.isfile(output_dir + "/report/post_route_timing.rpt")
+    # verification outputs
+    verify_out_dir = output_dir + "/verification_output"
+    assert os.path.isfile(verify_out_dir + "/verify_initial_python_SUCCESS.npy")
+    assert os.path.isfile(verify_out_dir + "/verify_streamlined_python_SUCCESS.npy")
+    assert os.path.isfile(verify_out_dir + "/verify_folded_hls_cppsim_SUCCESS.npy")
+    assert os.path.isfile(verify_out_dir + "/verify_stitched_ip_rtlsim_SUCCESS.npy")