svcenc: RC parameters verified before RC init

RC uses int32_t to store bitrates and other
parameters internally. For specific magnitudes of
bitrate, framerate, and GOP period, this can
result in signed integer overflow. This is now
detected before calls to RC init.

Note that calls to the 'ISVCE_CMD_CTL_SET_BITRATE' API can also
result in this behaviour but it will be appropriately handled
by 'isvce_svc_frame_params_validate'.

Bug = ossfuzz:63053
Test: svc_enc_fuzzer

4 files changed, 47 insertions, 5 deletions

diff --git a/encoder/svc/isvce_encode.c b/encoder/svc/isvce_encode.c
index d3f6198..8c6aa11 100644
--- a/encoder/svc/isvce_encode.c
+++ b/encoder/svc/isvce_encode.c
@@ -243,14 +243,17 @@ WORD32 isvce_encode(iv_obj_t *ps_codec_obj, void *pv_api_ip, void *pv_api_op)
     /* initialize codec ctxt with default params for the first encode api call */
     if(ps_codec->i4_encode_api_call_cnt == 0)
     {
-        isvce_codec_init(ps_codec);
+        error_status = isvce_codec_init(ps_codec);
+
+        SET_ERROR_ON_RETURN(error_status, IVE_FATALERROR,
+                            ps_video_encode_op->s_ive_op.u4_error_code, IV_FAIL);
     }
 
     error_status =
         isvce_svc_frame_params_validate(ps_codec->s_rate_control.apps_rate_control_api,
                                         ps_codec->s_cfg.s_svc_params.u1_num_spatial_layers);
-    SET_ERROR_ON_RETURN(error_status, IVE_UNSUPPORTEDPARAM,
-                        ps_video_encode_op->s_ive_op.u4_error_code, IV_FAIL);
+    SET_ERROR_ON_RETURN(error_status, IVE_FATALERROR, ps_video_encode_op->s_ive_op.u4_error_code,
+                        IV_FAIL);
 
     /* parse configuration params */
     for(i = 0; i < MAX_ACTIVE_CONFIG_PARAMS; i++)
diff --git a/encoder/svc/isvce_rate_control.c b/encoder/svc/isvce_rate_control.c
index cc3c41f..e4a85bb 100644
--- a/encoder/svc/isvce_rate_control.c
+++ b/encoder/svc/isvce_rate_control.c
@@ -159,11 +159,9 @@ void isvce_rc_init(void *pv_rc_api, void *pv_frame_time, void *pv_time_stamp, vo
                    UWORD8 *pu1_init_qp, WORD32 i4_max_inter_frm_int, UWORD8 *pu1_min_max_qp,
                    UWORD8 u1_profile_level)
 {
-    //    UWORD8  u1_is_mb_level_rc_on = 0;
     UWORD32 au4_peak_bit_rate[2] = {0, 0};
     UWORD32 u4_min_bit_rate = 0;
     WORD32 i4_is_gop_closed = 1;
-    //    WORD32  i4_use_est_intra_sad = 1;
     UWORD32 u4_src_ticks = 0;
     UWORD32 u4_tgt_ticks = 0;
     UWORD8 u1_level_idx = ih264e_get_lvl_idx(u1_profile_level);
diff --git a/encoder/svc/isvce_utils.c b/encoder/svc/isvce_utils.c
index 23fe569..485a52e 100644
--- a/encoder/svc/isvce_utils.c
+++ b/encoder/svc/isvce_utils.c
@@ -519,6 +519,36 @@ WORD32 isvce_svc_inp_params_validate(isvce_init_ip_t *ps_ip, isvce_cfg_params_t
 *******************************************************************************
 *
 * @brief
+*  Validates SVC RC params
+*
+* @param[in] ps_cfg
+*  Cfg parameters
+*
+* @returns  error code in conformance with 'IH264E_ERROR_T'
+*
+*******************************************************************************
+*/
+WORD32 isvce_svc_rc_params_validate(isvce_cfg_params_t *ps_cfg)
+{
+    WORD32 i;
+
+    /* RC requires total bits in a second to fit int32_t */
+    for(i = 0; i < ps_cfg->s_svc_params.u1_num_spatial_layers; i++)
+    {
+        if((((((UWORD64) ps_cfg->au4_target_bitrate[i]) * 1000llu) / ps_cfg->u4_tgt_frame_rate) *
+            ps_cfg->u4_idr_frm_interval) > ((UWORD64) INT32_MAX))
+        {
+            return IH264E_BITRATE_NOT_SUPPORTED;
+        }
+    }
+
+    return IH264E_SUCCESS;
+}
+
+/**
+*******************************************************************************
+*
+* @brief
 *  Validates SVC frame-level input params
 *
 * @param[in] ps_cfg
@@ -3545,6 +3575,15 @@ IH264E_ERROR_T isvce_codec_init(isvce_codec_t *ps_codec)
         ps_codec->i4_air_pic_cnt = -1;
     }
 
+    {
+        WORD32 i4_err_code = isvce_svc_rc_params_validate(&ps_codec->s_cfg);
+
+        if(IH264E_SUCCESS != i4_err_code)
+        {
+            return i4_err_code;
+        }
+    }
+
     /****************************************************/
     /*           INITIALIZE RATE CONTROL                */
     /****************************************************/
diff --git a/encoder/svc/isvce_utils.h b/encoder/svc/isvce_utils.h
index ad14446..497809e 100644
--- a/encoder/svc/isvce_utils.h
+++ b/encoder/svc/isvce_utils.h
@@ -180,6 +180,8 @@ extern WORD32 isvce_svc_au_props_validate(svc_inp_params_t *ps_svc_inp_params, U
 
 extern WORD32 isvce_svc_inp_params_validate(isvce_init_ip_t *ps_ip, isvce_cfg_params_t *ps_cfg);
 
+extern WORD32 isvce_svc_rc_params_validate(isvce_cfg_params_t *ps_cfg);
+
 extern WORD32 isvce_svc_frame_params_validate(
     rate_control_api_t *aps_rate_control_api[MAX_NUM_SPATIAL_LAYERS], UWORD8 u1_num_spatial_layers);