You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
139 lines
5.1 KiB
139 lines
5.1 KiB
From 6129dbe86484393a2ea520be94ef9b550b0bd113 Mon Sep 17 00:00:00 2001
|
|
From: Jason Garrett-Glaser <jason@x264.com>
|
|
Date: Wed, 27 Feb 2013 13:30:22 -0800
|
|
Subject: [PATCH] Fix array overreads that caused miscompilation in gcc 4.8
|
|
|
|
---
|
|
common/common.h | 1 +
|
|
common/set.c | 78 +++++++++++++++++++++++++++++++-------------------------
|
|
tools/checkasm.c | 2 +-
|
|
3 files changed, 45 insertions(+), 36 deletions(-)
|
|
|
|
diff --git a/common/common.h b/common/common.h
|
|
index 39ad5cb..0da1d43 100644
|
|
--- a/common/common.h
|
|
+++ b/common/common.h
|
|
@@ -40,6 +40,7 @@
|
|
#define IS_DISPOSABLE(type) ( type == X264_TYPE_B )
|
|
#define FIX8(f) ((int)(f*(1<<8)+.5))
|
|
#define ALIGN(x,a) (((x)+((a)-1))&~((a)-1))
|
|
+#define ARRAY_ELEMS(a) ((sizeof(a))/(sizeof(a[0])))
|
|
|
|
#define CHECKED_MALLOC( var, size )\
|
|
do {\
|
|
diff --git a/common/set.c b/common/set.c
|
|
index 4c72125..fa8b158 100644
|
|
--- a/common/set.c
|
|
+++ b/common/set.c
|
|
@@ -85,44 +85,49 @@ int x264_cqm_init( x264_t *h )
|
|
int max_qp_err = -1;
|
|
int max_chroma_qp_err = -1;
|
|
int min_qp_err = QP_MAX+1;
|
|
- int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4 : 2; /* Checkasm may segfault if optimized out by --chroma-format */
|
|
+ int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4
|
|
+ : h->param.analyse.b_transform_8x8 ? 2 : 0; /* Checkasm may segfault if optimized out by --chroma-format */
|
|
|
|
- for( int i = 0; i < 4 + num_8x8_lists; i++ )
|
|
- {
|
|
- int size = i<4 ? 16 : 64;
|
|
- int j;
|
|
- for( j = (i<4 ? 0 : 4); j < i; j++ )
|
|
- if( !memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
|
|
- break;
|
|
- if( j < i )
|
|
- {
|
|
- h-> quant4_mf[i] = h-> quant4_mf[j];
|
|
- h->dequant4_mf[i] = h->dequant4_mf[j];
|
|
- h->unquant4_mf[i] = h->unquant4_mf[j];
|
|
- }
|
|
- else
|
|
- {
|
|
- CHECKED_MALLOC( h-> quant4_mf[i], (QP_MAX+1)*size*sizeof(udctcoef) );
|
|
- CHECKED_MALLOC( h->dequant4_mf[i], 6*size*sizeof(int) );
|
|
- CHECKED_MALLOC( h->unquant4_mf[i], (QP_MAX+1)*size*sizeof(int) );
|
|
- }
|
|
-
|
|
- for( j = (i<4 ? 0 : 4); j < i; j++ )
|
|
- if( deadzone[j&3] == deadzone[i&3] &&
|
|
- !memcmp( h->pps->scaling_list[i], h->pps->scaling_list[j], size*sizeof(uint8_t) ) )
|
|
- break;
|
|
- if( j < i )
|
|
- {
|
|
- h->quant4_bias[i] = h->quant4_bias[j];
|
|
- h->quant4_bias0[i] = h->quant4_bias0[j];
|
|
- }
|
|
- else
|
|
- {
|
|
- CHECKED_MALLOC( h->quant4_bias[i], (QP_MAX+1)*size*sizeof(udctcoef) );
|
|
- CHECKED_MALLOC( h->quant4_bias0[i], (QP_MAX+1)*size*sizeof(udctcoef) );
|
|
- }
|
|
+#define CQM_ALLOC( w, count )\
|
|
+ for( int i = 0; i < count; i++ )\
|
|
+ {\
|
|
+ int size = w*w;\
|
|
+ int start = w == 8 ? 4 : 0;\
|
|
+ int j;\
|
|
+ for( j = 0; j < i; j++ )\
|
|
+ if( !memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
|
|
+ break;\
|
|
+ if( j < i )\
|
|
+ {\
|
|
+ h-> quant##w##_mf[i] = h-> quant##w##_mf[j];\
|
|
+ h->dequant##w##_mf[i] = h->dequant##w##_mf[j];\
|
|
+ h->unquant##w##_mf[i] = h->unquant##w##_mf[j];\
|
|
+ }\
|
|
+ else\
|
|
+ {\
|
|
+ CHECKED_MALLOC( h-> quant##w##_mf[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
|
|
+ CHECKED_MALLOC( h->dequant##w##_mf[i], 6*size*sizeof(int) );\
|
|
+ CHECKED_MALLOC( h->unquant##w##_mf[i], (QP_MAX+1)*size*sizeof(int) );\
|
|
+ }\
|
|
+ for( j = 0; j < i; j++ )\
|
|
+ if( deadzone[j] == deadzone[i] &&\
|
|
+ !memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
|
|
+ break;\
|
|
+ if( j < i )\
|
|
+ {\
|
|
+ h->quant##w##_bias[i] = h->quant##w##_bias[j];\
|
|
+ h->quant##w##_bias0[i] = h->quant##w##_bias0[j];\
|
|
+ }\
|
|
+ else\
|
|
+ {\
|
|
+ CHECKED_MALLOC( h->quant##w##_bias[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
|
|
+ CHECKED_MALLOC( h->quant##w##_bias0[i], (QP_MAX+1)*size*sizeof(udctcoef) );\
|
|
+ }\
|
|
}
|
|
|
|
+ CQM_ALLOC( 4, 4 )
|
|
+ CQM_ALLOC( 8, num_8x8_lists )
|
|
+
|
|
for( int q = 0; q < 6; q++ )
|
|
{
|
|
for( int i = 0; i < 16; i++ )
|
|
@@ -204,6 +209,9 @@ int x264_cqm_init( x264_t *h )
|
|
for( int cat = 0; cat < 3 + CHROMA444; cat++ )
|
|
{
|
|
int dct8x8 = cat&1;
|
|
+ if( !h->param.analyse.b_transform_8x8 && dct8x8 )
|
|
+ continue;
|
|
+
|
|
int size = dct8x8 ? 64 : 16;
|
|
udctcoef *nr_offset = h->nr_offset_emergency[q][cat];
|
|
/* Denoise chroma first (due to h264's chroma QP offset), then luma, then DC. */
|
|
diff --git a/tools/checkasm.c b/tools/checkasm.c
|
|
index 9135b70..441687b 100644
|
|
--- a/tools/checkasm.c
|
|
+++ b/tools/checkasm.c
|
|
@@ -309,7 +309,7 @@ static int check_pixel( int cpu_ref, int cpu_new )
|
|
|
|
#define TEST_PIXEL( name, align ) \
|
|
ok = 1, used_asm = 0; \
|
|
- for( int i = 0; i < 8; i++ ) \
|
|
+ for( int i = 0; i < ARRAY_ELEMS(pixel_c.name); i++ ) \
|
|
{ \
|
|
int res_c, res_asm; \
|
|
if( pixel_asm.name[i] != pixel_ref.name[i] ) \
|
|
--
|
|
1.8.1.5
|
|
|