[Test] add hw throughput test to end2end test

tests/end2end/test_end2end_bnn_pynq.py

@@ -86,7 +86,8 @@ from finn.transformation.fpgadataflow.insert_fifo import InsertFIFO
 from finn.transformation.fpgadataflow.annotate_cycles import AnnotateCycles
 from finn.analysis.fpgadataflow.dataflow_performance import dataflow_performance
 from finn.core.modelwrapper import ModelWrapper
-
+from scipy.stats import linregress
+from finn.core.throughput_test import throughput_test_remote
 
 build_dir = "/tmp/" + os.environ["FINN_INST_NAME"]
 target_clk_ns = 10
@@ -405,3 +406,50 @@ class TestEnd2End:
         ret = execute_onnx(parent_model, {iname: input_tensor_npy}, True)
         y = ret[oname]
         assert np.isclose(y, output_tensor_npy).all()
+
+    @pytest.mark.parametrize("kind", ["zynq", "alveo"])
+    def test_throughput_hw(self, topology, wbits, abits, kind):
+        prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "deploy_" + kind)
+        end2end_example = "%s_w%da%d_%s" % (topology, wbits, abits, kind)
+        model = load_test_checkpoint_or_skip(prev_chkpt_name)  # NOQA
+        cfg = get_build_env(kind, target_clk_ns)
+        if cfg["ip"] == "":
+            pytest.skip("PYNQ board IP address not specified")
+        ret = dict()
+        # try a range of batch sizes, some may fail due to insufficient DMA
+        # buffers
+        bsize_range_in = [8 ** i for i in range(5)]
+        bsize_range = []
+        for bsize in bsize_range_in:
+            res = throughput_test_remote(model, bsize)
+            if res is not None:
+                ret[bsize] = res
+                bsize_range.append(bsize)
+            else:
+                # assume we reached largest possible N
+                break
+        y = [ret[key]["runtime[ms]"] for key in bsize_range]
+        lrret = linregress(bsize_range, y)
+        ret_str = ""
+        ret_str += "\n" + "%s Throughput Test Results" % end2end_example
+        ret_str += "\n" + "-----------------------------"
+        ret_str += "\n" + "From linear regression:"
+        ret_str += "\n" + "Invocation overhead: %f ms" % lrret.intercept
+        ret_str += "\n" + "Time per sample: %f ms" % lrret.slope
+        ret_str += "\n" + "Raw data:"
+
+        ret_str += "\n" + "{:<8} {:<16} {:<16} {:<16} {:<16} {:<16}".format(
+            "N", "runtime[ms]", "fclk[mhz]", "fps", "DRAM rd[Mb/s]", "DRAM wr[Mb/s]"
+        )
+        for k in bsize_range:
+            v = ret[k]
+            ret_str += "\n" + "{:<8} {:<16} {:<16} {:<16} {:<16} {:<16}".format(
+                k,
+                np.round(v["runtime[ms]"], 4),
+                v["fclk[mhz]"],
+                np.round(v["throughput[images/s]"], 2),
+                np.round(v["DRAM_in_bandwidth[Mb/s]"], 2),
+                np.round(v["DRAM_out_bandwidth[Mb/s]"], 2),
+            )
+        ret_str += "\n" + "-----------------------------"
+        warnings.warn(ret_str)