Add some error/overflow checks in codebook handlingandroid-security-8.1.0_r93android-security-8.1.0_r92android-security-8.1.0_r91android-security-8.1.0_r90android-security-8.1.0_r89android-security-8.1.0_r88android-security-8.1.0_r87android-security-8.1.0_r86android-security-8.1.0_r85android-security-8.1.0_r84android-security-8.1.0_r83android-security-8.1.0_r82android-8.1.0_r81android-8.1.0_r80android-8.1.0_r79android-8.1.0_r78android-8.1.0_r77android-8.1.0_r76android-8.1.0_r75android-8.1.0_r74android-8.1.0_r73android-8.1.0_r72android-8.1.0_r71android-8.1.0_r70android-8.1.0_r69android-8.1.0_r68android-8.1.0_r66security-oc-mr1-releaseoreo-mr1-security-release

Bug: 119120561
Test: manual
Change-Id: I2476ef29d81a8335fdcbdbbaa175a575138d4039
(cherry picked from commit b5e041da6bfe79b5ee5fb48374cd753df6d18015)
(cherry picked from commit 6eb63e119294ba92f93f03bbda6c4b203f0f87bb)

1 files changed, 12 insertions, 5 deletions

diff --git a/Tremolo/codebook.c b/Tremolo/codebook.c
index 467640d..a06302d 100644
--- a/Tremolo/codebook.c
+++ b/Tremolo/codebook.c
@@ -376,14 +376,21 @@ long _book_maptype1_quantvals(codebook *b){
     long acc=1;
     long acc1=1;
     int i;
-    for(i=0;i<b->dim;i++){
-      acc*=vals;
-      acc1*=vals+1;
+    for (i = 0; i < b->dim; i++) {
+      if (acc > b->entries / vals) {
+          break;
+      }
+      acc *= vals;
+      if (acc1 > LONG_MAX / (vals + 1)) {
+        acc1 = LONG_MAX;
+      } else {
+        acc1 *= (vals + 1);
+      }
     }
-    if(acc<=b->entries && acc1>b->entries){
+    if (i >= b->dim && acc <= b->entries && acc1 > b->entries) {
       return(vals);
     }else{
-      if(acc>b->entries){
+      if (i < b->dim || acc > b->entries) {
         vals--;
       }else{
         vals++;