Use scales representable in fp16 for fp16 casting ops in RGG.

The scale in NNAPI is always a fp32 data type. As there are accelerators
that only support fp16 data type, there is precision loss when
executing a casting operation that converts a fp16 buffer to/from a
quantized buffer. In the worst case, the precision loss in scale could
be up to 0.5 ULP, and results in 0.5 * 255 = 127.5 ULP difference in the
final result. This is much larger than the tolerance of these operations
in RGG. This CL sets the scale of such operations to be always representable
in fp16 to avoid potential precission loss.

Additionally, relax the MSE requirement on DEQUANTIZE.

Bug: 155842363
Test: NNT_static_fuzzing
Change-Id: I511ddb6753be1dabac36e1423820a03a4f28641e

2 files changed, 40 insertions, 30 deletions

diff --git a/nn/runtime/test/fuzzing/TestRandomGraph.cpp b/nn/runtime/test/fuzzing/TestRandomGraph.cpp
index 55c6542f1..2c8024a22 100644
--- a/nn/runtime/test/fuzzing/TestRandomGraph.cpp
+++ b/nn/runtime/test/fuzzing/TestRandomGraph.cpp
@@ -485,7 +485,7 @@ const AccuracyCriteria kStrictCriteria = {
 // broadcast or elementwise, e.g ADD, FLOOR.
 const AccuracyCriteria kMediumCriteria = {
         .float32 = {.bias = 1e-6f, .mse = 1e-8f, .atol = 1e-5f, .rtol = 1e-5f},
-        .float16 = {.bias = 1e-3f, .mse = 1e-6f, .atol = 1e-2f, .rtol = 1e-2f},
+        .float16 = {.bias = 1e-3f, .mse = 1e-5f, .atol = 1e-2f, .rtol = 1e-2f},
         .int32 = {.atol = 1},
         .quant8Asymm = {.bias = 1.2, .mse = 1.2, .atol = 2},
         .quant8AsymmSigned = {.bias = 1.2, .mse = 1.2, .atol = 2},
diff --git a/nn/runtime/test/fuzzing/operation_signatures/Elementwise.cpp b/nn/runtime/test/fuzzing/operation_signatures/Elementwise.cpp
index 567ff0581..d84727d95 100644
--- a/nn/runtime/test/fuzzing/operation_signatures/Elementwise.cpp
+++ b/nn/runtime/test/fuzzing/operation_signatures/Elementwise.cpp
@@ -131,41 +131,51 @@ DEFINE_ELEMENTWISE_WITH_QUANT_OUTPUT_SIGNATURE(LOGISTIC, V1_3, /*scale=*/1.f / 2
 DEFINE_ELEMENTWISE_WITH_QUANT_OUTPUT_SIGNATURE(TANH, V1_3, /*scale=*/1.f / 128, /*zeroPoint=*/0,
                                                TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED);
 
+static void castingOpConstructor(TestOperandType dataType, uint32_t rank, RandomOperation* op) {
+    sameDimensionOpConstructor(dataType, rank, op);
+
+    // If it is casting to/from a FP16 data type, the source/destination should have a scale
+    // representable in FP16 to avoid precision loss.
+    if (op->inputs[0]->dataType == TestOperandType::TENSOR_FLOAT16) {
+        op->outputs[0]->scale = static_cast<_Float16>(op->outputs[0]->scale);
+    } else if (op->outputs[0]->dataType == TestOperandType::TENSOR_FLOAT16) {
+        op->inputs[0]->scale = static_cast<_Float16>(op->inputs[0]->scale);
+    }
+}
+
 // Operations with output data type different from input.
-#define DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(op, ver, outType, ...) \
-    DEFINE_OPERATION_SIGNATURE(op##_##outType##_##ver){                       \
-            .opType = TestOperationType::op,                                  \
-            .supportedDataTypes = {__VA_ARGS__},                              \
-            .supportedRanks = {1, 2, 3, 4},                                   \
-            .version = TestHalVersion::ver,                                   \
-            .inputs = {INPUT_DEFAULT},                                        \
-            .outputs = {OUTPUT_TYPED(TestOperandType::outType)},              \
-            .constructor = sameDimensionOpConstructor};
-
-DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(DEQUANTIZE, V1_0, /*outType=*/TENSOR_FLOAT32,
-                                               TestOperandType::TENSOR_QUANT8_ASYMM);
+#define DEFINE_QUANTIZATION_OP_SIGNATURE(op, ver, outType, ...)  \
+    DEFINE_OPERATION_SIGNATURE(op##_##outType##_##ver){          \
+            .opType = TestOperationType::op,                     \
+            .supportedDataTypes = {__VA_ARGS__},                 \
+            .supportedRanks = {1, 2, 3, 4},                      \
+            .version = TestHalVersion::ver,                      \
+            .inputs = {INPUT_DEFAULT},                           \
+            .outputs = {OUTPUT_TYPED(TestOperandType::outType)}, \
+            .constructor = castingOpConstructor};
 
-DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(DEQUANTIZE, V1_2, /*outType=*/TENSOR_FLOAT32,
-                                               TestOperandType::TENSOR_QUANT8_SYMM);
+DEFINE_QUANTIZATION_OP_SIGNATURE(DEQUANTIZE, V1_0, /*outType=*/TENSOR_FLOAT32,
+                                 TestOperandType::TENSOR_QUANT8_ASYMM);
 
-DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(DEQUANTIZE, V1_2, /*outType=*/TENSOR_FLOAT16,
-                                               TestOperandType::TENSOR_QUANT8_ASYMM,
-                                               TestOperandType::TENSOR_QUANT8_SYMM);
+DEFINE_QUANTIZATION_OP_SIGNATURE(DEQUANTIZE, V1_2, /*outType=*/TENSOR_FLOAT32,
+                                 TestOperandType::TENSOR_QUANT8_SYMM);
 
-DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(DEQUANTIZE, V1_3, /*outType=*/TENSOR_FLOAT32,
-                                               TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED);
+DEFINE_QUANTIZATION_OP_SIGNATURE(DEQUANTIZE, V1_2, /*outType=*/TENSOR_FLOAT16,
+                                 TestOperandType::TENSOR_QUANT8_ASYMM,
+                                 TestOperandType::TENSOR_QUANT8_SYMM);
 
-DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(DEQUANTIZE, V1_3, /*outType=*/TENSOR_FLOAT16,
-                                               TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED);
+DEFINE_QUANTIZATION_OP_SIGNATURE(DEQUANTIZE, V1_3, /*outType=*/TENSOR_FLOAT32,
+                                 TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED);
 
-DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(QUANTIZE, V1_2, /*outType=*/TENSOR_QUANT8_ASYMM,
-                                               TestOperandType::TENSOR_FLOAT32,
-                                               TestOperandType::TENSOR_FLOAT16);
+DEFINE_QUANTIZATION_OP_SIGNATURE(DEQUANTIZE, V1_3, /*outType=*/TENSOR_FLOAT16,
+                                 TestOperandType::TENSOR_QUANT8_ASYMM_SIGNED);
 
-DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(QUANTIZE, V1_3,
-                                               /*outType=*/TENSOR_QUANT8_ASYMM_SIGNED,
-                                               TestOperandType::TENSOR_FLOAT32,
-                                               TestOperandType::TENSOR_FLOAT16);
+DEFINE_QUANTIZATION_OP_SIGNATURE(QUANTIZE, V1_2, /*outType=*/TENSOR_QUANT8_ASYMM,
+                                 TestOperandType::TENSOR_FLOAT32, TestOperandType::TENSOR_FLOAT16);
+
+DEFINE_QUANTIZATION_OP_SIGNATURE(QUANTIZE, V1_3,
+                                 /*outType=*/TENSOR_QUANT8_ASYMM_SIGNED,
+                                 TestOperandType::TENSOR_FLOAT32, TestOperandType::TENSOR_FLOAT16);
 
 #define DEFINE_CAST_SIGNATURE(ver, outType, ...)                 \
     DEFINE_OPERATION_SIGNATURE(CAST_##outType##_##ver){          \
@@ -175,7 +185,7 @@ DEFINE_ELEMENTWISE_WITH_TYPED_OUTPUT_SIGNATURE(QUANTIZE, V1_3,
             .version = TestHalVersion::ver,                      \
             .inputs = {INPUT_DEFAULT},                           \
             .outputs = {OUTPUT_TYPED(TestOperandType::outType)}, \
-            .constructor = sameDimensionOpConstructor};
+            .constructor = castingOpConstructor};
 
 DEFINE_CAST_SIGNATURE(V1_2, /*outType=*/TENSOR_FLOAT32, TestOperandType::TENSOR_FLOAT32,
                       TestOperandType::TENSOR_FLOAT16, TestOperandType::TENSOR_QUANT8_ASYMM,