[FFmpeg-devel] [PATCH 5/6] x86/hevcdsp: add ff_hevc_sao_edge_filter_8_{ssse3, avx2}
James Almer
jamrial at gmail.com
Wed Feb 4 04:55:52 CET 2015
Original x86 intrinsics code and initial yasm port by Pierre-Edouard Lepere.
Refactoring and optimizations by James Almer.
Benchmarks of BQTerrace_1920x1080_60_qp22.bin with an Intel Core i5-4200U
Width 32
158583 decicycles in edge, sao_edge_filter_8 runs, 0 skips
5205 decicycles in ff_hevc_sao_edge_filter_32_8_ssse3, 32767 runs, 1 skips
2942 decicycles in ff_hevc_sao_edge_filter_32_8_avx2, 32767 runs, 1 skips
Width 64
705639 decicycles in sao_edge_filter_8, 262144 runs, 0 skips
19224 decicycles in ff_hevc_sao_edge_filter_64_8_ssse3, 262111 runs, 33 skips
10433 decicycles in ff_hevc_sao_edge_filter_64_8_avx2, 262115 runs, 29 skips
Signed-off-by: James Almer <jamrial at gmail.com>
---
libavcodec/hevc_filter.c | 10 +--
libavcodec/hevcdsp.c | 6 +-
libavcodec/hevcdsp.h | 6 +-
libavcodec/x86/hevc_sao.asm | 172 +++++++++++++++++++++++++++++++++++++++++-
libavcodec/x86/hevcdsp_init.c | 51 ++++++++++---
5 files changed, 224 insertions(+), 21 deletions(-)
diff --git a/libavcodec/hevc_filter.c b/libavcodec/hevc_filter.c
index bf88b47..df7fa12 100644
--- a/libavcodec/hevc_filter.c
+++ b/libavcodec/hevc_filter.c
@@ -246,7 +246,7 @@ static void restore_tqb_pixels(HEVCContext *s,
static void sao_filter_CTB(HEVCContext *s, int x, int y)
{
- static const uint8_t band_tab[8] = { 0, 1, 2, 2, 3, 3, 4, 4 };
+ static const uint8_t sao_tab[8] = { 0, 1, 2, 2, 3, 3, 4, 4 };
HEVCLocalContext *lc = s->HEVClc;
int c_idx;
int edges[4]; // 0 left 1 top 2 right 3 bottom
@@ -312,7 +312,7 @@ static void sao_filter_CTB(HEVCContext *s, int x, int y)
int ctb_size_v = (1 << (s->sps->log2_ctb_size)) >> s->sps->vshift[c_idx];
int width = FFMIN(ctb_size_h, (s->sps->width >> s->sps->hshift[c_idx]) - x0);
int height = FFMIN(ctb_size_v, (s->sps->height >> s->sps->vshift[c_idx]) - y0);
- int tab = band_tab[(FFALIGN(width, 8) >> 3) - 1];
+ int tab = sao_tab[(FFALIGN(width, 8) >> 3) - 1];
uint8_t *src = &s->frame->data[c_idx][y0 * stride_src + (x0 << s->sps->pixel_shift)];
int stride_dst;
uint8_t *dst;
@@ -427,9 +427,9 @@ static void sao_filter_CTB(HEVCContext *s, int x, int y)
copy_CTB_to_hv(s, src, stride_src, x0, y0, width, height, c_idx,
x_ctb, y_ctb);
- s->hevcdsp.sao_edge_filter(src, dst, stride_src, stride_dst,
- sao->offset_val[c_idx], sao->eo_class[c_idx],
- width, height);
+ s->hevcdsp.sao_edge_filter[tab](src, dst, stride_src, stride_dst,
+ sao->offset_val[c_idx], sao->eo_class[c_idx],
+ width, height);
s->hevcdsp.sao_edge_restore[restore](src, dst,
stride_src, stride_dst,
sao,
diff --git a/libavcodec/hevcdsp.c b/libavcodec/hevcdsp.c
index b9ae787..7dcdfff 100644
--- a/libavcodec/hevcdsp.c
+++ b/libavcodec/hevcdsp.c
@@ -217,7 +217,11 @@ void ff_hevc_dsp_init(HEVCDSPContext *hevcdsp, int bit_depth)
hevcdsp->sao_band_filter[2] = \
hevcdsp->sao_band_filter[3] = \
hevcdsp->sao_band_filter[4] = FUNC(sao_band_filter_0, depth); \
- hevcdsp->sao_edge_filter = FUNC(sao_edge_filter, depth); \
+ hevcdsp->sao_edge_filter[0] = \
+ hevcdsp->sao_edge_filter[1] = \
+ hevcdsp->sao_edge_filter[2] = \
+ hevcdsp->sao_edge_filter[3] = \
+ hevcdsp->sao_edge_filter[4] = FUNC(sao_edge_filter, depth); \
hevcdsp->sao_edge_restore[0] = FUNC(sao_edge_restore_0, depth); \
hevcdsp->sao_edge_restore[1] = FUNC(sao_edge_restore_1, depth); \
\
diff --git a/libavcodec/hevcdsp.h b/libavcodec/hevcdsp.h
index 1510f39..d4ded4c 100644
--- a/libavcodec/hevcdsp.h
+++ b/libavcodec/hevcdsp.h
@@ -61,9 +61,9 @@ typedef struct HEVCDSPContext {
void (*sao_band_filter[5])(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src,
int16_t *sao_offset_val, int sao_left_class, int width, int height);
- void (*sao_edge_filter)(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst,
- ptrdiff_t stride_src, int16_t *sao_offset_val, int sao_eo_class,
- int width, int height);
+ void (*sao_edge_filter[5])(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst,
+ ptrdiff_t stride_src, int16_t *sao_offset_val, int sao_eo_class,
+ int width, int height);
void (*sao_edge_restore[2])(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src,
struct SAOParams *sao, int *borders, int _width, int _height, int c_idx,
diff --git a/libavcodec/x86/hevc_sao.asm b/libavcodec/x86/hevc_sao.asm
index b389a9f..40cef3b 100644
--- a/libavcodec/x86/hevc_sao.asm
+++ b/libavcodec/x86/hevc_sao.asm
@@ -23,14 +23,22 @@
%include "libavutil/x86/x86util.asm"
-%if ARCH_X86_64
SECTION_RODATA 32
pw_mask10: times 16 dw 0x03FF
pw_mask12: times 16 dw 0x0FFF
+pb_2: times 32 db 2
+pb_edge_shuffle: times 2 db 1, 2, 0, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
+pb_eo: db -1, 0, 1, 0, 0, -1, 0, 1, -1, -1, 1, 1, 1, -1, -1, 1
+cextern pb_1
SECTION_TEXT
+;******************************************************************************
+;SAO Band Filter
+;******************************************************************************
+
+%if ARCH_X86_64
%macro HEVC_SAO_BAND_FILTER_INIT 1
and leftq, 31
movd xm0, leftd
@@ -239,3 +247,165 @@ HEVC_SAO_BAND_FILTER_16 12, 48, 1
HEVC_SAO_BAND_FILTER_16 12, 64, 2
%endif
%endif
+
+;******************************************************************************
+;SAO Edge Filter
+;******************************************************************************
+
+%macro HEVC_SAO_EDGE_FILTER_COMPUTE_8 1
+ pminub m4, m1, m2
+ pminub m5, m1, m3
+ pcmpeqb m2, m4
+ pcmpeqb m3, m5
+ pcmpeqb m4, m1
+ pcmpeqb m5, m1
+ psubb m4, m2
+ psubb m5, m3
+ paddb m4, m6
+ paddb m4, m5
+
+ pshufb m2, m0, m4
+%if %1 > 8
+ punpckhbw m5, m7, m1
+ punpckhbw m4, m2, m7
+ punpcklbw m3, m7, m1
+ punpcklbw m2, m7
+ pmaddubsw m5, m4
+ pmaddubsw m3, m2
+ packuswb m3, m5
+%else
+ punpcklbw m3, m7, m1
+ punpcklbw m2, m7
+ pmaddubsw m3, m2
+ packuswb m3, m3
+%endif
+%endmacro
+
+;void ff_hevc_sao_edge_filter_<width>_8_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, ptrdiff_t stride_src,
+; int16_t *sao_offset_val, int eo, int width, int height);
+%macro HEVC_SAO_EDGE_FILTER_8 2
+%if WIN64
+cglobal hevc_sao_edge_filter_%1_8, 4, 8, 8, dst, src, dststride, srcstride, eo, a_stride, b_stride, height
+%define tmpq heightq
+%define offsetq r4q
+ movsxd eoq, dword r5m
+ movsx a_strideq, byte [pb_eo+eoq*4+1]
+ movsx b_strideq, byte [pb_eo+eoq*4+3]
+ imul a_strideq, srcstrideq
+ imul b_strideq, srcstrideq
+ movsx tmpq, byte [pb_eo+eoq*4]
+ add a_strideq, tmpq
+ movsx tmpq, byte [pb_eo+eoq*4+2]
+ add b_strideq, tmpq
+
+ mov offsetq, r4m
+ mov heightd, heightm
+
+%elif ARCH_X86_64
+cglobal hevc_sao_edge_filter_%1_8, 6, 10, 8, dst, src, dststride, srcstride, offset, eo, a_stride, b_stride, height
+%define tmpq r9q
+%define tmp2q heightq
+ movsxd eoq, eod
+ lea tmp2q, [pb_eo]
+ movsx a_strideq, byte [tmp2q+eoq*4+1]
+ movsx b_strideq, byte [tmp2q+eoq*4+3]
+ imul a_strideq, srcstrideq
+ imul b_strideq, srcstrideq
+ movsx tmpq, byte [tmp2q+eoq*4]
+ add a_strideq, tmpq
+ movsx tmpq, byte [tmp2q+eoq*4+2]
+ add b_strideq, tmpq
+
+ mov heightd, r7m
+
+%else ; ARCH_X86_32
+cglobal hevc_sao_edge_filter_%1_8, 1, 7, 8, dst, src, dststride, srcstride, a_stride, b_stride, height
+%define eoq srcq
+%define tmpq heightq
+%define tmp2q dststrideq
+%define offsetq heightq
+ mov eoq, r5m
+ lea tmp2q, [pb_eo]
+ mov srcstrideq, srcstridem
+ movsx a_strideq, byte [tmp2q+eoq*4+1]
+ movsx b_strideq, byte [tmp2q+eoq*4+3]
+ imul a_strideq, srcstrideq
+ imul b_strideq, srcstrideq
+ movsx tmpq, byte [tmp2q+eoq*4]
+ add a_strideq, tmpq
+ movsx tmpq, byte [tmp2q+eoq*4+2]
+ add b_strideq, tmpq
+
+ mov srcq, srcm
+ mov offsetq, r4m
+ mov dststrideq, dststridem
+%endif ; ARCH
+
+%if mmsize > 16
+ vbroadcasti128 m0, [offsetq]
+%else
+ movu m0, [offsetq]
+%endif
+ mova m1, [pb_edge_shuffle]
+ packsswb m0, m0
+ mova m7, [pb_1]
+ pshufb m0, m1
+ mova m6, [pb_2]
+%if ARCH_X86_32
+ mov heightd, r7m
+%endif
+
+align 16
+.loop:
+
+%if %1 == 8
+ movq m1, [srcq]
+ movq m2, [srcq + a_strideq]
+ movq m3, [srcq + b_strideq]
+ HEVC_SAO_EDGE_FILTER_COMPUTE_8 %1
+ movq [dstq], m3
+%endif
+
+%assign i 0
+%rep %2
+ mova m1, [srcq + i]
+ movu m2, [srcq + a_strideq + i]
+ movu m3, [srcq + b_strideq + i]
+ HEVC_SAO_EDGE_FILTER_COMPUTE_8 %1
+ mova [dstq + i], m3
+%assign i i+mmsize
+%endrep
+
+%if %1 == 48
+INIT_XMM cpuname
+
+ mova m1, [srcq + i]
+ movu m2, [srcq + a_strideq + i]
+ movu m3, [srcq + b_strideq + i]
+ HEVC_SAO_EDGE_FILTER_COMPUTE_8 %1
+ mova [dstq + i], m3
+%if cpuflag(avx2)
+INIT_YMM cpuname
+%endif
+%endif
+
+ add dstq, dststrideq
+ add srcq, srcstrideq
+ dec heightd
+ jg .loop
+ RET
+%endmacro
+
+INIT_XMM ssse3
+HEVC_SAO_EDGE_FILTER_8 8, 0
+HEVC_SAO_EDGE_FILTER_8 16, 1
+HEVC_SAO_EDGE_FILTER_8 32, 2
+HEVC_SAO_EDGE_FILTER_8 48, 2
+HEVC_SAO_EDGE_FILTER_8 64, 4
+
+%if HAVE_AVX2_EXTERNAL
+INIT_YMM avx2
+HEVC_SAO_EDGE_FILTER_8 32, 1
+HEVC_SAO_EDGE_FILTER_8 48, 1
+HEVC_SAO_EDGE_FILTER_8 64, 2
+%endif
diff --git a/libavcodec/x86/hevcdsp_init.c b/libavcodec/x86/hevcdsp_init.c
index 5a01ff6..2970adf 100644
--- a/libavcodec/x86/hevcdsp_init.c
+++ b/libavcodec/x86/hevcdsp_init.c
@@ -500,6 +500,37 @@ SAO_BAND_FILTER_FUNCS(8, avx2);
SAO_BAND_FILTER_FUNCS(10, avx2);
SAO_BAND_FILTER_FUNCS(12, avx2);
+#define SAO_BAND_INIT(bitd, opt) do { \
+ c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_##bitd##_##opt; \
+ c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_##bitd##_##opt; \
+ c->sao_band_filter[2] = ff_hevc_sao_band_filter_32_##bitd##_##opt; \
+ c->sao_band_filter[3] = ff_hevc_sao_band_filter_48_##bitd##_##opt; \
+ c->sao_band_filter[4] = ff_hevc_sao_band_filter_64_##bitd##_##opt; \
+} while (0)
+
+#define SAO_EDGE_FILTER_FUNCS(bitd, opt) \
+void ff_hevc_sao_edge_filter_8_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, ptrdiff_t stride_src, \
+ int16_t *sao_offset_val, int eo, int width, int height); \
+void ff_hevc_sao_edge_filter_16_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, ptrdiff_t stride_src, \
+ int16_t *sao_offset_val, int eo, int width, int height); \
+void ff_hevc_sao_edge_filter_32_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, ptrdiff_t stride_src, \
+ int16_t *sao_offset_val, int eo, int width, int height); \
+void ff_hevc_sao_edge_filter_48_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, ptrdiff_t stride_src, \
+ int16_t *sao_offset_val, int eo, int width, int height); \
+void ff_hevc_sao_edge_filter_64_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t stride_dst, ptrdiff_t stride_src, \
+ int16_t *sao_offset_val, int eo, int width, int height)
+
+SAO_EDGE_FILTER_FUNCS(8, ssse3);
+SAO_EDGE_FILTER_FUNCS(8, avx2);
+
+#define SAO_EDGE_INIT(bitd, opt) do { \
+ c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8_##bitd##_##opt; \
+ c->sao_edge_filter[1] = ff_hevc_sao_edge_filter_16_##bitd##_##opt; \
+ c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_##bitd##_##opt; \
+ c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_##bitd##_##opt; \
+ c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_##bitd##_##opt; \
+} while (0)
+
#define EPEL_LINKS(pointer, my, mx, fname, bitd, opt ) \
PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
PEL_LINK(pointer, 2, my , mx , fname##6 , bitd, opt ); \
@@ -520,14 +551,6 @@ SAO_BAND_FILTER_FUNCS(12, avx2);
PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
-#define SAO_BAND_INIT(bitd, opt) do { \
- c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_##bitd##_##opt; \
- c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_##bitd##_##opt; \
- c->sao_band_filter[2] = ff_hevc_sao_band_filter_32_##bitd##_##opt; \
- c->sao_band_filter[3] = ff_hevc_sao_band_filter_48_##bitd##_##opt; \
- c->sao_band_filter[4] = ff_hevc_sao_band_filter_64_##bitd##_##opt; \
-} while (0)
-
void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
{
int cpu_flags = av_get_cpu_flags();
@@ -555,9 +578,12 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
c->transform_add[2] = ff_hevc_transform_add16_8_sse2;
c->transform_add[3] = ff_hevc_transform_add32_8_sse2;
}
- if (EXTERNAL_SSSE3(cpu_flags) && ARCH_X86_64) {
- c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3;
- c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3;
+ if (EXTERNAL_SSSE3(cpu_flags)) {
+ if(ARCH_X86_64) {
+ c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3;
+ c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3;
+ }
+ SAO_EDGE_INIT(8, ssse3);
}
if (EXTERNAL_SSE4(cpu_flags) && ARCH_X86_64) {
@@ -590,6 +616,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
if (ARCH_X86_64) {
SAO_BAND_INIT(8, avx2);
}
+ c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_8_avx2;
+ c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_8_avx2;
+ c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_8_avx2;
c->transform_add[3] = ff_hevc_transform_add32_8_avx2;
}
--
2.2.2
More information about the ffmpeg-devel
mailing list