diff --git a/src/finn/util/data_packing.py b/src/finn/util/data_packing.py
index 3602b1bdd5d013ee8ce2f6cf156490478f0cc74e..a41fe882e543db3f6809f0bc269b81c1e8a22ab5 100644
--- a/src/finn/util/data_packing.py
+++ b/src/finn/util/data_packing.py
@@ -220,7 +220,7 @@ def unpack_innermost_dim_from_hex_string(
         if conv_dtype == DataType["BIPOLAR"]:
             ar_list = [2 * x - 1 for x in ar_list]
         # interpret values as signed values
-        elif dtype.signed():
+        elif conv_dtype.signed() and conv_dtype.is_integer():
             mask = 2 ** (conv_dtype.bitwidth() - 1)
             ar_list = [-(x & mask) + (x & ~mask) for x in ar_list]