GitHub/libvpx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vp9 encoder: fix integer overflows

Browse Source 
fixing integer overflow with 16K content and enabling the test

Bug: webm:1750
Fixed: webm:1750
Change-Id: I76eebd915bcae55bc755613251a98e1716dea4c0
This commit is contained in:
Fyodor Kyslov
2021-12-14 09:59:17 -08:00
parent 2d9e4d3c7a
commit ea042a676e
5 changed files with 31 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/realtime_test.cc
+1 -3
View File
@@ -85,9 +85,7 @@ TEST_P(RealtimeTest, IntegerOverflow) { TestIntegerOverflow(2048, 2048); }
TEST_P(RealtimeTest, IntegerOverflowLarge) {
if (IsVP9()) {
GTEST_SKIP() << "TODO(https://crbug.com/webm/1750): Enable this test after "
"undefined sanitizer warnings are fixed.";
// TestIntegerOverflow(16384, 16384);
TestIntegerOverflow(16384, 16384);
} else {
GTEST_SKIP()
<< "TODO(https://crbug.com/webm/1748,https://crbug.com/webm/1751):"
vp9/encoder/vp9_bitstream.c
+3 -3
View File
@@ -563,7 +563,7 @@ static void update_coef_probs_common(vpx_writer *const bc, VP9_COMP *cpi,
for (t = 0; t < entropy_nodes_update; ++t) {
vpx_prob newp = new_coef_probs[i][j][k][l][t];
const vpx_prob oldp = old_coef_probs[i][j][k][l][t];
int s;
int64_t s;
int u = 0;
if (t == PIVOT_NODE)
s = vp9_prob_diff_update_savings_search_model(
@@ -600,7 +600,7 @@ static void update_coef_probs_common(vpx_writer *const bc, VP9_COMP *cpi,
vpx_prob newp = new_coef_probs[i][j][k][l][t];
vpx_prob *oldp = old_coef_probs[i][j][k][l] + t;
const vpx_prob upd = DIFF_UPDATE_PROB;
int s;
int64_t s;
int u = 0;
if (t == PIVOT_NODE)
s = vp9_prob_diff_update_savings_search_model(
@@ -636,7 +636,7 @@ static void update_coef_probs_common(vpx_writer *const bc, VP9_COMP *cpi,
for (t = 0; t < entropy_nodes_update; ++t) {
vpx_prob newp = new_coef_probs[i][j][k][l][t];
vpx_prob *oldp = old_coef_probs[i][j][k][l] + t;
int s;
int64_t s;
int u = 0;
if (t == PIVOT_NODE) {
vp9/encoder/vp9_cost.h
+2 -3
View File
@@ -29,9 +29,8 @@ extern const uint16_t vp9_prob_cost[256];
#define vp9_cost_bit(prob, bit) vp9_cost_zero((bit) ? 256 - (prob) : (prob))
static INLINE unsigned int cost_branch256(const unsigned int ct[2],
vpx_prob p) {
return ct[0] * vp9_cost_zero(p) + ct[1] * vp9_cost_one(p);
static INLINE uint64_t cost_branch256(const unsigned int ct[2], vpx_prob p) {
return (uint64_t)ct[0] * vp9_cost_zero(p) + (uint64_t)ct[1] * vp9_cost_one(p);
}
static INLINE int treed_cost(vpx_tree tree, const vpx_prob *probs, int bits,
vp9/encoder/vp9_subexp.c
+18 -17
View File
@@ -114,19 +114,20 @@ void vp9_write_prob_diff_update(vpx_writer *w, vpx_prob newp, vpx_prob oldp) {
encode_term_subexp(w, delp);
}
int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp,
vpx_prob *bestp, vpx_prob upd) {
const int old_b = cost_branch256(ct, oldp);
int bestsavings = 0;
int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct,
vpx_prob oldp, vpx_prob *bestp,
vpx_prob upd) {
const int64_t old_b = cost_branch256(ct, oldp);
int64_t bestsavings = 0;
vpx_prob newp, bestnewp = oldp;
const int step = *bestp > oldp ? -1 : 1;
const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd);
if (old_b > upd_cost + (MIN_DELP_BITS << VP9_PROB_COST_SHIFT)) {
for (newp = *bestp; newp != oldp; newp += step) {
const int new_b = cost_branch256(ct, newp);
const int update_b = prob_diff_update_cost(newp, oldp) + upd_cost;
const int savings = old_b - new_b - update_b;
const int64_t new_b = cost_branch256(ct, newp);
const int64_t update_b = prob_diff_update_cost(newp, oldp) + upd_cost;
const int64_t savings = old_b - new_b - update_b;
if (savings > bestsavings) {
bestsavings = savings;
bestnewp = newp;
@@ -137,15 +138,15 @@ int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp,
return bestsavings;
}
int vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
const vpx_prob oldp,
vpx_prob *bestp, vpx_prob upd,
int stepsize) {
int i, old_b, new_b, update_b, savings, bestsavings;
int newp;
const int step_sign = *bestp > oldp ? -1 : 1;
const int step = stepsize * step_sign;
const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd);
int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
const vpx_prob oldp,
vpx_prob *bestp, vpx_prob upd,
int stepsize) {
int64_t i, old_b, new_b, update_b, savings, bestsavings;
int64_t newp;
const int64_t step_sign = *bestp > oldp ? -1 : 1;
const int64_t step = stepsize * step_sign;
const int64_t upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd);
const vpx_prob *newplist, *oldplist;
vpx_prob bestnewp;
oldplist = vp9_pareto8_full[oldp - 1];
@@ -182,7 +183,7 @@ void vp9_cond_prob_diff_update(vpx_writer *w, vpx_prob *oldp,
const unsigned int ct[2]) {
const vpx_prob upd = DIFF_UPDATE_PROB;
vpx_prob newp = get_binary_prob(ct[0], ct[1]);
const int savings =
const int64_t savings =
vp9_prob_diff_update_savings_search(ct, *oldp, &newp, upd);
assert(newp >= 1);
if (savings > 0) {
vp9/encoder/vp9_subexp.h
+7 -6
View File
@@ -25,13 +25,14 @@ void vp9_write_prob_diff_update(struct vpx_writer *w, vpx_prob newp,
void vp9_cond_prob_diff_update(struct vpx_writer *w, vpx_prob *oldp,
const unsigned int ct[2]);
int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp,
vpx_prob *bestp, vpx_prob upd);
int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct,
vpx_prob oldp, vpx_prob *bestp,
vpx_prob upd);
int vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
const vpx_prob oldp,
vpx_prob *bestp, vpx_prob upd,
int stepsize);
int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct,
const vpx_prob oldp,
vpx_prob *bestp, vpx_prob upd,
int stepsize);
#ifdef __cplusplus
} // extern "C"