From 01b508e3325f153bd45ec9d0e76014d7f9e65120 Mon Sep 17 00:00:00 2001
From: Yaman Umuroglu <maltanar@gmail.com>
Date: Wed, 9 Sep 2020 12:15:50 +0200
Subject: [PATCH] [Test] add hw throughput test to end2end test
---
tests/end2end/test_end2end_bnn_pynq.py | 50 +++++++++++++++++++++++++-
1 file changed, 49 insertions(+), 1 deletion(-)
diff --git a/tests/end2end/test_end2end_bnn_pynq.py b/tests/end2end/test_end2end_bnn_pynq.py
index cb3ad3e47..ac73831ab 100644
--- a/tests/end2end/test_end2end_bnn_pynq.py
+++ b/tests/end2end/test_end2end_bnn_pynq.py
@@ -86,7 +86,8 @@ from finn.transformation.fpgadataflow.insert_fifo import InsertFIFO
from finn.transformation.fpgadataflow.annotate_cycles import AnnotateCycles
from finn.analysis.fpgadataflow.dataflow_performance import dataflow_performance
from finn.core.modelwrapper import ModelWrapper
-
+from scipy.stats import linregress
+from finn.core.throughput_test import throughput_test_remote
build_dir = "/tmp/" + os.environ["FINN_INST_NAME"]
target_clk_ns = 10
@@ -405,3 +406,50 @@ class TestEnd2End:
ret = execute_onnx(parent_model, {iname: input_tensor_npy}, True)
y = ret[oname]
assert np.isclose(y, output_tensor_npy).all()
+
+ @pytest.mark.parametrize("kind", ["zynq", "alveo"])
+ def test_throughput_hw(self, topology, wbits, abits, kind):
+ prev_chkpt_name = get_checkpoint_name(topology, wbits, abits, "deploy_" + kind)
+ end2end_example = "%s_w%da%d_%s" % (topology, wbits, abits, kind)
+ model = load_test_checkpoint_or_skip(prev_chkpt_name) # NOQA
+ cfg = get_build_env(kind, target_clk_ns)
+ if cfg["ip"] == "":
+ pytest.skip("PYNQ board IP address not specified")
+ ret = dict()
+ # try a range of batch sizes, some may fail due to insufficient DMA
+ # buffers
+ bsize_range_in = [8 ** i for i in range(5)]
+ bsize_range = []
+ for bsize in bsize_range_in:
+ res = throughput_test_remote(model, bsize)
+ if res is not None:
+ ret[bsize] = res
+ bsize_range.append(bsize)
+ else:
+ # assume we reached largest possible N
+ break
+ y = [ret[key]["runtime[ms]"] for key in bsize_range]
+ lrret = linregress(bsize_range, y)
+ ret_str = ""
+ ret_str += "\n" + "%s Throughput Test Results" % end2end_example
+ ret_str += "\n" + "-----------------------------"
+ ret_str += "\n" + "From linear regression:"
+ ret_str += "\n" + "Invocation overhead: %f ms" % lrret.intercept
+ ret_str += "\n" + "Time per sample: %f ms" % lrret.slope
+ ret_str += "\n" + "Raw data:"
+
+ ret_str += "\n" + "{:<8} {:<16} {:<16} {:<16} {:<16} {:<16}".format(
+ "N", "runtime[ms]", "fclk[mhz]", "fps", "DRAM rd[Mb/s]", "DRAM wr[Mb/s]"
+ )
+ for k in bsize_range:
+ v = ret[k]
+ ret_str += "\n" + "{:<8} {:<16} {:<16} {:<16} {:<16} {:<16}".format(
+ k,
+ np.round(v["runtime[ms]"], 4),
+ v["fclk[mhz]"],
+ np.round(v["throughput[images/s]"], 2),
+ np.round(v["DRAM_in_bandwidth[Mb/s]"], 2),
+ np.round(v["DRAM_out_bandwidth[Mb/s]"], 2),
+ )
+ ret_str += "\n" + "-----------------------------"
+ warnings.warn(ret_str)
--
GitLab