[Build] add verification to build steps

src/finn/builder/build_dataflow_config.py

@@ -93,8 +93,6 @@ class VerificationStepType(str, Enum):
     STREAMLINED_PYTHON = "streamlined_python"
     #: verify after step_apply_folding_config, using C++ for each HLS node
     FOLDED_HLS_CPPSIM = "folded_hls_cppsim"
-    #: verify after step_hls_ipgen, using Verilog for each HLS node
-    IPGEN_RTLSIM = "ipgen_rtlsim"
     #: verify after step_create_stitched_ip, using stitched-ip Verilog
     STITCHED_IP_RTLSIM = "stitched_ip_rtlsim"
 
@@ -158,15 +156,15 @@ class DataflowBuildConfig:
     #: (Optional) At which steps the generated intermediate output model
     #: will be verified. See documentation of VerificationStepType for
     #: available options.
-    verify_steps: Optional[List[VerificationStepType]] = []
+    verify_steps: Optional[List[VerificationStepType]] = None
 
     #: (Optional) Name of .npy file that will be used as the input for
     #: verification. Only required if verify_steps is not empty.
-    verify_input_npy: Optional[str] = None
+    verify_input_npy: Optional[str] = "input.npy"
 
     #: (Optional) Name of .npy file that will be used as the expected output for
     #: verification. Only required if verify_steps is not empty.
-    verify_expected_output_npy: Optional[str] = None
+    verify_expected_output_npy: Optional[str] = "expected_output.npy"
 
     #: (Optional) Control the maximum width of the per-PE MVAU stream while
     #: exploring the parallelization attributes to reach target_fps
@@ -297,8 +295,14 @@ class DataflowBuildConfig:
         }
         return name_to_strategy[self.vitis_opt_strategy]
 
+    def _resolve_verification_steps(self):
+        if self.verify_steps is None:
+            return []
+        else:
+            return self.verify_steps
+
     def _resolve_verification_io_pair(self):
-        if self.verify_steps == []:
+        if self.verify_steps is None:
             return None
         else:
             assert os.path.isfile(self.verify_input_npy), (

src/finn/builder/build_dataflow_steps.py

@@ -88,8 +88,46 @@ from finn.builder.build_dataflow_config import (
     DataflowBuildConfig,
     DataflowOutputType,
     ShellFlowType,
+    VerificationStepType,
 )
 from finn.transformation.fpgadataflow.annotate_cycles import AnnotateCycles
+from finn.core.onnx_exec import execute_onnx
+import numpy as np
+from finn.util.test import execute_parent
+from finn.transformation.fpgadataflow.prepare_cppsim import PrepareCppSim
+from finn.transformation.fpgadataflow.compile_cppsim import CompileCppSim
+from finn.transformation.fpgadataflow.set_exec_mode import SetExecMode
+from finn.transformation.fpgadataflow.prepare_rtlsim import PrepareRTLSim
+
+
+def verify_step(
+    model: ModelWrapper, cfg: DataflowBuildConfig, step_name: str, need_parent: bool
+):
+    print("Running verification for " + step_name)
+    verify_out_dir = cfg.output_dir + "/verification_output"
+    intermediate_models_dir = cfg.output_dir + "/intermediate_models"
+    os.makedirs(verify_out_dir, exist_ok=True)
+    (in_npy, exp_out_npy) = cfg._resolve_verification_io_pair()
+    if need_parent:
+        assert (
+            cfg.save_intermediate_models
+        ), "Enable save_intermediate_models for verification"
+        parent_model_fn = intermediate_models_dir + "/dataflow_parent.onnx"
+        child_model_fn = intermediate_models_dir + "/verify_%s.onnx" % step_name
+        model.save(child_model_fn)
+        out_npy = execute_parent(parent_model_fn, child_model_fn, in_npy)
+    else:
+        inp_tensor_name = model.graph.input[0].name
+        out_tensor_name = model.graph.output[0].name
+        inp_dict = {inp_tensor_name: in_npy}
+        out_dict = execute_onnx(model, inp_dict)
+        out_npy = out_dict[out_tensor_name]
+    res = np.isclose(exp_out_npy, out_npy, atol=1e-3).all()
+    res_to_str = {True: "SUCCESS", False: "FAIL"}
+    res_str = res_to_str[res]
+    verification_output_fn = verify_out_dir + "/verify_%s_%s.npy" % (step_name, res_str)
+    np.save(verification_output_fn, out_npy)
+    print("Verification for %s : %s" % (step_name, res_str))
 
 
 def step_tidy_up(model: ModelWrapper, cfg: DataflowBuildConfig):
@@ -103,6 +141,10 @@ def step_tidy_up(model: ModelWrapper, cfg: DataflowBuildConfig):
     model = model.transform(GiveReadableTensorNames())
     model = model.transform(InferDataTypes())
     model = model.transform(RemoveStaticGraphInputs())
+
+    if VerificationStepType.TIDY_UP_PYTHON in cfg._resolve_verification_steps():
+        verify_step(model, cfg, "initial_python", need_parent=False)
+
     return model
 
 
@@ -127,6 +169,10 @@ def step_streamline(model: ModelWrapper, cfg: DataflowBuildConfig):
     model = model.transform(absorb.AbsorbScalarMulAddIntoTopK())
     model = model.transform(InferDataLayouts())
     model = model.transform(RemoveUnusedTensors())
+
+    if VerificationStepType.STREAMLINED_PYTHON in cfg._resolve_verification_steps():
+        verify_step(model, cfg, "streamlined_python", need_parent=False)
+
     return model
 
 
@@ -196,6 +242,13 @@ def step_apply_folding_config(model: ModelWrapper, cfg: DataflowBuildConfig):
     if cfg.folding_config_file is not None:
         model = model.transform(GiveUniqueNodeNames())
         model = model.transform(ApplyConfig(cfg.folding_config_file))
+
+    if VerificationStepType.FOLDED_HLS_CPPSIM in cfg._resolve_verification_steps():
+        # prepare cppsim
+        model = model.transform(PrepareCppSim())
+        model = model.transform(CompileCppSim())
+        model = model.transform(SetExecMode("cppsim"))
+        verify_step(model, cfg, "folded_hls_cppsim", need_parent=True)
     return model
 
 
@@ -323,6 +376,20 @@ def step_create_stitched_ip(model: ModelWrapper, cfg: DataflowBuildConfig):
         # TODO copy all ip sources into output dir? as zip?
         copytree(model.get_metadata_prop("vivado_stitch_proj"), stitched_ip_dir)
         print("Vivado stitched IP written into " + stitched_ip_dir)
+    if VerificationStepType.STITCHED_IP_RTLSIM in cfg._resolve_verification_steps():
+        # prepare ip-stitched rtlsim
+        verify_model = copy.deepcopy(model)
+        # rtlsim only supports impl_style=rtl for StreamingFIFO, ensure that
+        for fifo_layer in verify_model.get_nodes_by_op_type("StreamingFIFO"):
+            getCustomOp(fifo_layer).set_nodeattr("impl_style", "rtl")
+        # similarly for StreamingDataWidthConverter with impl_style=hls
+        for dwc_layer in verify_model.get_nodes_by_op_type(
+            "StreamingDataWidthConverter_Batch"
+        ):
+            getCustomOp(dwc_layer).set_nodeattr("impl_style", "hls")
+        verify_model = verify_model.transform(PrepareRTLSim())
+        verify_model.set_metadata_prop("exec_mode", "rtlsim")
+        verify_step(verify_model, cfg, "stitched_ip_rtlsim", need_parent=True)
     return model