Merge pull request #306 from Xilinx/feature/end2end_test_cybsec_mlp

Adds an end-to-end test for the UNSW-NB15 (cybersecurity/network intrusion detection) MLP. Also includes a minor fix to the PYNQ driver (using correctly-generated tensors with respect to datatype for throughput testing)

src/finn/qnn-data/cybsec-mlp/validate-unsw-nb15.py

+# Copyright (c) 2020 Xilinx, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of Xilinx nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+from driver import io_shape_dict
+from driver_base import FINNExampleOverlay
+import numpy as np
+
+
+def make_unsw_nb15_test_batches(bsize, dataset_root, limit_batches):
+    unsw_nb15_data = np.load(dataset_root + "/unsw_nb15_binarized.npz")["test"][:82000]
+    test_imgs = unsw_nb15_data[:, :-1]
+    test_labels = unsw_nb15_data[:, -1]
+    n_batches = int(test_imgs.shape[0] / bsize)
+    if limit_batches == -1:
+        limit_batches = n_batches
+    test_imgs = test_imgs.reshape(n_batches, bsize, -1)[:limit_batches]
+    test_labels = test_labels.reshape(n_batches, bsize)[:limit_batches]
+    return (test_imgs, test_labels)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Validate top-1 accuracy for FINN-generated accelerator"
+    )
+    parser.add_argument("--batchsize", help="samples per batch", type=int, default=1000)
+    parser.add_argument(
+        "--platform", help="Target platform: zynq-iodma alveo", default="zynq-iodma"
+    )
+    parser.add_argument(
+        "--bitfile",
+        help='name of bitfile (i.e. "resizer.bit")',
+        default="../bitfile/finn-accel.bit",
+    )
+    parser.add_argument(
+        "--dataset_root", help="dataset root dir for download/reuse", default="."
+    )
+    parser.add_argument(
+        "--limit_batches", help="number of batches, -1 for max", type=int, default=-1
+    )
+    # parse arguments
+    args = parser.parse_args()
+    bsize = args.batchsize
+    bitfile = args.bitfile
+    platform = args.platform
+    dataset_root = args.dataset_root
+    limit_batches = args.limit_batches
+
+    print("Loading dataset...")
+    (test_imgs, test_labels) = make_unsw_nb15_test_batches(
+        bsize, dataset_root, limit_batches
+    )
+
+    ok = 0
+    nok = 0
+    n_batches = test_imgs.shape[0]
+    total = n_batches * bsize
+
+    print("Initializing driver, flashing bitfile...")
+
+    driver = FINNExampleOverlay(
+        bitfile_name=bitfile,
+        platform=platform,
+        io_shape_dict=io_shape_dict,
+        batch_size=bsize,
+    )
+
+    n_batches = int(total / bsize)
+
+    print("Starting...")
+
+    for i in range(n_batches):
+        inp = np.pad(test_imgs[i].astype(np.float32), [(0, 0), (0, 7)], mode="constant")
+        exp = test_labels[i].astype(np.float32)
+        inp = 2 * inp - 1
+        exp = 2 * exp - 1
+        out = driver.execute(inp)
+        matches = np.count_nonzero(out.flatten() == exp.flatten())
+        nok += bsize - matches
+        ok += matches
+        print("batch %d / %d : total OK %d NOK %d" % (i + 1, n_batches, ok, nok))
+
+    acc = 100.0 * ok / (total)
+    print("Final accuracy: %f" % acc)

src/finn/qnn-data/templates/driver/driver_base.py

@@ -37,6 +37,8 @@ from finn.util.data_packing import (
     packed_bytearray_to_finnpy,
 )
 
+from finn.util.basic import gen_finn_dt_tensor
+
 # Driver base class for FINN-generated dataflow accelerators.
 # The particulars of the generated accelerator are specified via the
 # io_shape_dict (generated by the MakePYNQDriver transformation).
@@ -344,7 +346,7 @@ class FINNExampleOverlay(Overlay):
             res["fclk[mhz]"] = self.clock_dict["clock0"]["frequency"]
         res["batch_size"] = self.batch_size
         # also benchmark driver-related overheads
-        input_npy = np.zeros(self.ishape_normal, dtype=self.idt.to_numpy_dt())
+        input_npy = gen_finn_dt_tensor(self.idt, self.ishape_normal)
         start = time.time()
         ibuf_folded = self.fold_input(input_npy)
         end = time.time()

tests/end2end/test_end2end_cybsec_mlp.py

+import torch
+from brevitas.nn import QuantLinear, QuantReLU
+import torch.nn as nn
+import numpy as np
+from brevitas.core.quant import QuantType
+from brevitas.nn import QuantIdentity
+import brevitas.onnx as bo
+from finn.core.modelwrapper import ModelWrapper
+from finn.core.datatype import DataType
+import finn.builder.build_dataflow as build
+import finn.builder.build_dataflow_config as build_cfg
+import os
+import shutil
+from finn.util.test import get_build_env, load_test_checkpoint_or_skip
+import pytest
+from finn.util.basic import make_build_dir
+import pkg_resources as pk
+import json
+import wget
+import subprocess
+
+target_clk_ns = 10
+build_kind = "zynq"
+build_dir = os.environ["FINN_BUILD_DIR"]
+
+
+def get_checkpoint_name(step):
+    if step == "build":
+        # checkpoint for build step is an entire dir
+        return build_dir + "/end2end_cybsecmlp_build"
+    else:
+        # other checkpoints are onnx files
+        return build_dir + "/end2end_cybsecmlp_%s.onnx" % (step)
+
+
+class CybSecMLPForExport(nn.Module):
+    def __init__(self, my_pretrained_model):
+        super(CybSecMLPForExport, self).__init__()
+        self.pretrained = my_pretrained_model
+        self.qnt_output = QuantIdentity(
+            quant_type=QuantType.BINARY, bit_width=1, min_val=-1.0, max_val=1.0
+        )
+
+    def forward(self, x):
+        # assume x contains bipolar {-1,1} elems
+        # shift from {-1,1} -> {0,1} since that is the
+        # input range for the trained network
+        x = (x + torch.tensor([1.0])) / 2.0
+        out_original = self.pretrained(x)
+        out_final = self.qnt_output(out_original)  # output as {-1,1}
+        return out_final
+
+
+def test_end2end_cybsec_mlp_export():
+    assets_dir = pk.resource_filename("finn.qnn-data", "cybsec-mlp/")
+    # load up trained net in Brevitas
+    input_size = 593
+    hidden1 = 64
+    hidden2 = 64
+    hidden3 = 64
+    weight_bit_width = 2
+    act_bit_width = 2
+    num_classes = 1
+    model = nn.Sequential(
+        QuantLinear(input_size, hidden1, bias=True, weight_bit_width=weight_bit_width),
+        nn.BatchNorm1d(hidden1),
+        nn.Dropout(0.5),
+        QuantReLU(bit_width=act_bit_width),
+        QuantLinear(hidden1, hidden2, bias=True, weight_bit_width=weight_bit_width),
+        nn.BatchNorm1d(hidden2),
+        nn.Dropout(0.5),
+        QuantReLU(bit_width=act_bit_width),
+        QuantLinear(hidden2, hidden3, bias=True, weight_bit_width=weight_bit_width),
+        nn.BatchNorm1d(hidden3),
+        nn.Dropout(0.5),
+        QuantReLU(bit_width=act_bit_width),
+        QuantLinear(hidden3, num_classes, bias=True, weight_bit_width=weight_bit_width),
+    )
+    trained_state_dict = torch.load(assets_dir + "/state_dict.pth")[
+        "models_state_dict"
+    ][0]
+    model.load_state_dict(trained_state_dict, strict=False)
+    W_orig = model[0].weight.data.detach().numpy()
+    # pad the second (593-sized) dimensions with 7 zeroes at the end
+    W_new = np.pad(W_orig, [(0, 0), (0, 7)])
+    model[0].weight.data = torch.from_numpy(W_new)
+    model_for_export = CybSecMLPForExport(model)
+    export_onnx_path = get_checkpoint_name("export")
+    input_shape = (1, 600)
+    bo.export_finn_onnx(model_for_export, input_shape, export_onnx_path)
+    assert os.path.isfile(export_onnx_path)
+    # fix input datatype
+    finn_model = ModelWrapper(export_onnx_path)
+    finnonnx_in_tensor_name = finn_model.graph.input[0].name
+    finn_model.set_tensor_datatype(finnonnx_in_tensor_name, DataType.BIPOLAR)
+    finn_model.save(export_onnx_path)
+    assert tuple(finn_model.get_tensor_shape(finnonnx_in_tensor_name)) == (1, 600)
+    assert len(finn_model.graph.node) == 30
+    assert finn_model.graph.node[0].op_type == "Add"
+    assert finn_model.graph.node[1].op_type == "Div"
+    assert finn_model.graph.node[2].op_type == "MatMul"
+    assert finn_model.graph.node[-1].op_type == "MultiThreshold"
+
+
+@pytest.mark.slow
+@pytest.mark.vivado
+def test_end2end_cybsec_mlp_build():
+    model_file = get_checkpoint_name("export")
+    load_test_checkpoint_or_skip(model_file)
+    build_env = get_build_env(build_kind, target_clk_ns)
+    output_dir = make_build_dir("test_end2end_cybsec_mlp_build")
+
+    cfg = build.DataflowBuildConfig(
+        output_dir=output_dir,
+        target_fps=1000000,
+        synth_clk_period_ns=target_clk_ns,
+        board=build_env["board"],
+        shell_flow_type=build_cfg.ShellFlowType.VIVADO_ZYNQ,
+        generate_outputs=[
+            build_cfg.DataflowOutputType.ESTIMATE_REPORTS,
+            build_cfg.DataflowOutputType.BITFILE,
+            build_cfg.DataflowOutputType.PYNQ_DRIVER,
+            build_cfg.DataflowOutputType.DEPLOYMENT_PACKAGE,
+        ],
+    )
+    build.build_dataflow_cfg(model_file, cfg)
+    # check the generated files
+    assert os.path.isfile(output_dir + "/time_per_step.json")
+    assert os.path.isfile(output_dir + "/final_hw_config.json")
+    assert os.path.isfile(output_dir + "/driver/driver.py")
+    est_cycles_report = output_dir + "/report/estimate_layer_cycles.json"
+    assert os.path.isfile(est_cycles_report)
+    est_res_report = output_dir + "/report/estimate_layer_resources.json"
+    assert os.path.isfile(est_res_report)
+    assert os.path.isfile(output_dir + "/report/estimate_network_performance.json")
+    assert os.path.isfile(output_dir + "/bitfile/finn-accel.bit")
+    assert os.path.isfile(output_dir + "/bitfile/finn-accel.hwh")
+    assert os.path.isfile(output_dir + "/report/post_synth_resources.xml")
+    assert os.path.isfile(output_dir + "/report/post_route_timing.rpt")
+    # examine the report contents
+    with open(est_cycles_report, "r") as f:
+        est_cycles_dict = json.load(f)
+        assert est_cycles_dict["StreamingFCLayer_Batch_0"] == 80
+        assert est_cycles_dict["StreamingFCLayer_Batch_1"] == 64
+    with open(est_res_report, "r") as f:
+        est_res_dict = json.load(f)
+        assert est_res_dict["total"]["LUT"] == 11360.0
+        assert est_res_dict["total"]["BRAM_18K"] == 36.0
+    shutil.copytree(output_dir + "/deploy", get_checkpoint_name("build"))
+
+
+def test_end2end_cybsec_mlp_run_on_hw():
+    build_env = get_build_env(build_kind, target_clk_ns)
+    assets_dir = pk.resource_filename("finn.qnn-data", "cybsec-mlp/")
+    deploy_dir = get_checkpoint_name("build")
+    if not os.path.isdir(deploy_dir):
+        pytest.skip(deploy_dir + " not found from previous test step, skipping")
+    driver_dir = deploy_dir + "/driver"
+    assert os.path.isdir(driver_dir)
+    # put all assets into driver dir
+    shutil.copy(assets_dir + "/validate-unsw-nb15.py", driver_dir)
+    # put a copy of binarized dataset into driver dir
+    dataset_url = (
+        "https://zenodo.org/record/4519767/files/unsw_nb15_binarized.npz?download=1"
+    )
+    dataset_local = driver_dir + "/unsw_nb15_binarized.npz"
+    if not os.path.isfile(dataset_local):
+        wget.download(dataset_url, out=dataset_local)
+    assert os.path.isfile(dataset_local)
+    # create a shell script for running validation: 10 batches x 10 imgs
+    with open(driver_dir + "/validate.sh", "w") as f:
+        f.write(
+            """#!/bin/bash
+cd %s/driver
+echo %s | sudo -S python3.6 validate-unsw-nb15.py --batchsize=10 --limit_batches=10
+        """
+            % (
+                build_env["target_dir"] + "/end2end_cybsecmlp_build",
+                build_env["password"],
+            )
+        )
+    # set up rsync command
+    remote_target = "%s@%s:%s" % (
+        build_env["username"],
+        build_env["ip"],
+        build_env["target_dir"],
+    )
+    rsync_res = subprocess.run(
+        [
+            "sshpass",
+            "-p",
+            build_env["password"],
+            "rsync",
+            "-avz",
+            deploy_dir,
+            remote_target,
+        ]
+    )
+    assert rsync_res.returncode == 0
+    remote_verif_cmd = [
+        "sshpass",
+        "-p",
+        build_env["password"],
+        "ssh",
+        "%s@%s" % (build_env["username"], build_env["ip"]),
+        "sh",
+        build_env["target_dir"] + "/end2end_cybsecmlp_build/driver/validate.sh",
+    ]
+    verif_res = subprocess.run(
+        remote_verif_cmd,
+        stdout=subprocess.PIPE,
+        universal_newlines=True,
+        input=build_env["password"],
+    )
+    assert verif_res.returncode == 0
+    log_output = verif_res.stdout.split("\n")
+    assert log_output[-3] == "batch 10 / 10 : total OK 93 NOK 7"
+    assert log_output[-2] == "Final accuracy: 93.000000"