[FFmpeg-devel] [PATCH 5/6] lavc/apv: AVX2 transquant for x86-64
Mark Thompson
sw at jkqxz.net
Sat Apr 19 22:07:03 EEST 2025
Typical checkasm result on Alder Lake:
decode_transquant_8_c: 408.7 ( 1.00x)
decode_transquant_8_avx2: 94.2 ( 4.34x)
decode_transquant_10_c: 413.1 ( 1.00x)
decode_transquant_10_avx2: 87.5 ( 4.72x)
---
libavcodec/apv_dsp.c | 4 +
libavcodec/apv_dsp.h | 2 +
libavcodec/x86/Makefile | 2 +
libavcodec/x86/apv_dsp.asm | 243 ++++++++++++++++++++++++++++++++++
libavcodec/x86/apv_dsp_init.c | 41 ++++++
tests/checkasm/Makefile | 1 +
tests/checkasm/apv_dsp.c | 113 ++++++++++++++++
tests/checkasm/checkasm.c | 3 +
tests/checkasm/checkasm.h | 1 +
9 files changed, 410 insertions(+)
create mode 100644 libavcodec/x86/apv_dsp.asm
create mode 100644 libavcodec/x86/apv_dsp_init.c
create mode 100644 tests/checkasm/apv_dsp.c
diff --git a/libavcodec/apv_dsp.c b/libavcodec/apv_dsp.c
index 6b8b6ce590..108a3cd370 100644
--- a/libavcodec/apv_dsp.c
+++ b/libavcodec/apv_dsp.c
@@ -133,4 +133,8 @@ static void apv_decode_transquant_c(void *output,
av_cold void ff_apv_dsp_init(APVDSPContext *dsp)
{
dsp->decode_transquant = apv_decode_transquant_c;
+
+#if ARCH_X86_64
+ ff_apv_dsp_init_x86_64(dsp);
+#endif
}
diff --git a/libavcodec/apv_dsp.h b/libavcodec/apv_dsp.h
index 13f401c4cc..a7b5377357 100644
--- a/libavcodec/apv_dsp.h
+++ b/libavcodec/apv_dsp.h
@@ -34,4 +34,6 @@ typedef struct APVDSPContext {
void ff_apv_dsp_init(APVDSPContext *dsp);
+void ff_apv_dsp_init_x86_64(APVDSPContext *dsp);
+
#endif /* AVCODEC_APV_DSP_H */
diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile
index 5d53515381..821c410a0f 100644
--- a/libavcodec/x86/Makefile
+++ b/libavcodec/x86/Makefile
@@ -44,6 +44,7 @@ OBJS-$(CONFIG_ADPCM_G722_DECODER) += x86/g722dsp_init.o
OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp_init.o
OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp_init.o
OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp_init.o
+OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp_init.o
OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsdsp.o
OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp_init.o
OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp_init.o
@@ -149,6 +150,7 @@ X86ASM-OBJS-$(CONFIG_ADPCM_G722_DECODER) += x86/g722dsp.o
X86ASM-OBJS-$(CONFIG_ADPCM_G722_ENCODER) += x86/g722dsp.o
X86ASM-OBJS-$(CONFIG_ALAC_DECODER) += x86/alacdsp.o
X86ASM-OBJS-$(CONFIG_APNG_DECODER) += x86/pngdsp.o
+X86ASM-OBJS-$(CONFIG_APV_DECODER) += x86/apv_dsp.o
X86ASM-OBJS-$(CONFIG_CAVS_DECODER) += x86/cavsidct.o
X86ASM-OBJS-$(CONFIG_CFHD_ENCODER) += x86/cfhdencdsp.o
X86ASM-OBJS-$(CONFIG_CFHD_DECODER) += x86/cfhddsp.o
diff --git a/libavcodec/x86/apv_dsp.asm b/libavcodec/x86/apv_dsp.asm
new file mode 100644
index 0000000000..0329089f45
--- /dev/null
+++ b/libavcodec/x86/apv_dsp.asm
@@ -0,0 +1,243 @@
+;************************************************************************
+;* This file is part of FFmpeg.
+;*
+;* FFmpeg is free software; you can redistribute it and/or
+;* modify it under the terms of the GNU Lesser General Public
+;* License as published by the Free Software Foundation; either
+;* version 2.1 of the License, or (at your option) any later version.
+;*
+;* FFmpeg is distributed in the hope that it will be useful,
+;* but WITHOUT ANY WARRANTY; without even the implied warranty of
+;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+;* Lesser General Public License for more details.
+;*
+;* You should have received a copy of the GNU Lesser General Public
+;* License along with FFmpeg; if not, write to the Free Software
+;* 51, Inc., Foundation Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+;******************************************************************************
+
+%include "libavutil/x86/x86util.asm"
+
+SECTION .text
+
+align 32
+const tmatrixh
+ dw 64, 89, 84, 75, 64, 50, 35, 18
+ dw 64, 75, 35, -18, -64, -89, -84, -50
+ dw 64, 50, -35, -89, -64, 18, 84, 75
+ dw 64, 18, -84, -50, 64, 75, -35, -89
+ dw 64, -18, -84, 50, 64, -75, -35, 89
+ dw 64, -50, -35, 89, -64, -18, 84, -75
+ dw 64, -75, 35, 18, -64, 89, -84, 50
+ dw 64, -89, 84, -75, 64, -50, 35, -18
+const tmatrixv
+ dw 64, 89, 84, 75, 64, 50, 35, 18
+ dw 64, -18, -84, 50, 64, -75, -35, 89
+ dw 64, 75, 35, -18, -64, -89, -84, -50
+ dw 64, -50, -35, 89, -64, -18, 84, -75
+ dw 64, 50, -35, -89, -64, 18, 84, 75
+ dw 64, -75, 35, 18, -64, 89, -84, 50
+ dw 64, 18, -84, -50, 64, 75, -35, -89
+ dw 64, -89, 84, -75, 64, -50, 35, -18
+
+; Memory targets for vpbroadcastd (register version requires AVX512).
+const one
+ dd 1
+const sixtyfour
+ dd 64
+
+; void ff_apv_decode_transquant_avx2(void *output,
+; ptrdiff_t pitch,
+; const int16_t *input,
+; const int16_t *qmatrix,
+; int64_t bit_depth,
+; int64_t qp_shift);
+
+INIT_YMM avx2
+
+cglobal apv_decode_transquant, 6, 6, 16, output, pitch, input, qmatrix, bit_depth, qp_shift
+
+ ; Load input and dequantise
+
+ lea rax, [bit_depthq - 2]
+ movq xm8, qp_shiftq
+ movq xm9, rax
+ vpbroadcastd m10, [one]
+ vpslld m10, m10, xm9
+ vpsrld m10, m10, 1
+
+ ; m8 = scalar qp_shift
+ ; m9 = scalar bd_shift
+ ; m10 = vector 1 << (bd_shift - 1)
+ ; m11 = qmatrix load
+%macro LOAD_AND_DEQUANT 2 ; (xmm input, constant offset)
+ vpmovsxwd m%1, [inputq + %2]
+ vpmovsxwd m11, [qmatrixq + %2]
+ vpmulld m%1, m%1, m11
+ vpslld m%1, m%1, xm8
+ vpaddd m%1, m%1, m10
+ vpsrad m%1, m%1, xm9
+ vpackssdw m%1, m%1, m%1
+%endmacro
+
+ LOAD_AND_DEQUANT 0, 0x00
+ LOAD_AND_DEQUANT 1, 0x10
+ LOAD_AND_DEQUANT 2, 0x20
+ LOAD_AND_DEQUANT 3, 0x30
+ LOAD_AND_DEQUANT 4, 0x40
+ LOAD_AND_DEQUANT 5, 0x50
+ LOAD_AND_DEQUANT 6, 0x60
+ LOAD_AND_DEQUANT 7, 0x70
+
+ ; Transform columns
+ ; This produces the output in rows to avoid needing any
+ ; transpose operations
+
+ vpunpcklwd m12, m0, m1
+ vpunpcklwd m13, m2, m3
+ vpunpcklwd m14, m4, m5
+ vpunpcklwd m15, m6, m7
+
+%macro TRANS_COL_STEP 2
+ vpbroadcastd m8, [tmatrixh + %2 + 0x00]
+ vpbroadcastd m9, [tmatrixh + %2 + 0x04]
+ vpbroadcastd m10, [tmatrixh + %2 + 0x08]
+ vpbroadcastd m11, [tmatrixh + %2 + 0x0c]
+ vpmaddwd m8, m8, m12
+ vpmaddwd m9, m9, m13
+ vpmaddwd m10, m10, m14
+ vpmaddwd m11, m11, m15
+ vpaddd m8, m8, m9
+ vpaddd m10, m10, m11
+ vpaddd m%1, m8, m10
+%endmacro
+
+ TRANS_COL_STEP 0, 0x00
+ TRANS_COL_STEP 1, 0x10
+ TRANS_COL_STEP 2, 0x20
+ TRANS_COL_STEP 3, 0x30
+ TRANS_COL_STEP 4, 0x40
+ TRANS_COL_STEP 5, 0x50
+ TRANS_COL_STEP 6, 0x60
+ TRANS_COL_STEP 7, 0x70
+
+ ; Mid-transform normalisation
+ ; Note that outputs here are fitted to 16 bits
+
+ vpbroadcastd m8, [sixtyfour]
+
+%macro NORMALISE 1
+ vpaddd m%1, m%1, m8
+ vpsrad m%1, m%1, 7
+ vpackssdw m%1, m%1, m%1
+ vpermq m%1, m%1, q3120
+%endmacro
+
+ NORMALISE 0
+ NORMALISE 1
+ NORMALISE 2
+ NORMALISE 3
+ NORMALISE 4
+ NORMALISE 5
+ NORMALISE 6
+ NORMALISE 7
+
+ ; Vertical transform
+
+ mova m12, [tmatrixv + 0x00]
+ mova m13, [tmatrixv + 0x20]
+ mova m14, [tmatrixv + 0x40]
+ mova m15, [tmatrixv + 0x60]
+
+%macro TRANS_ROW_STEP 1
+ vperm2i128 m%1, m%1, m%1, 1
+ vpmaddwd m8, m%1, m12
+ vpmaddwd m9, m%1, m13
+ vpmaddwd m10, m%1, m14
+ vpmaddwd m11, m%1, m15
+ vphaddd m8, m8, m9
+ vphaddd m10, m10, m11
+ vphaddd m%1, m8, m10
+%endmacro
+
+ TRANS_ROW_STEP 0
+ TRANS_ROW_STEP 1
+ TRANS_ROW_STEP 2
+ TRANS_ROW_STEP 3
+ TRANS_ROW_STEP 4
+ TRANS_ROW_STEP 5
+ TRANS_ROW_STEP 6
+ TRANS_ROW_STEP 7
+
+ ; Renormalise, clip and store output
+
+ vpbroadcastd m14, [one]
+ mov rax, 20
+ sub rax, bit_depthq ; bd_shift
+ movq xm9, rax
+ dec rax
+ movq xm13, rax
+ vpslld m8, m14, xm13
+ movq xm13, bit_depthq
+ vpslld m12, m14, xm13
+ vpsrld m10, m12, 1
+ vpsubd m12, m12, m14
+ vpxor m11, m11, m11
+
+ cmp bit_depthq, 8
+ jne store_10
+
+ ; m8 = vector 1 << (bd_shift - 1)
+ ; m9 = scalar bd_shift
+ ; m10 = vector 1 << (bit_depth - 1)
+ ; m11 = zero
+ ; m12 = vector (1 << bit_depth) - 1
+
+%macro NORMALISE_AND_STORE_8 1
+ vpaddd m%1, m%1, m8
+ vpsrad m%1, m%1, xm9
+ vpaddd m%1, m%1, m10
+ vextracti128 xm13, m%1, 0
+ vextracti128 xm14, m%1, 1
+ vpackusdw xm%1, xm13, xm14
+ vpackuswb xm%1, xm%1, xm%1
+ movq [outputq], xm%1
+ add outputq, pitchq
+%endmacro
+
+ NORMALISE_AND_STORE_8 0
+ NORMALISE_AND_STORE_8 1
+ NORMALISE_AND_STORE_8 2
+ NORMALISE_AND_STORE_8 3
+ NORMALISE_AND_STORE_8 4
+ NORMALISE_AND_STORE_8 5
+ NORMALISE_AND_STORE_8 6
+ NORMALISE_AND_STORE_8 7
+
+ RET
+
+store_10:
+
+%macro NORMALISE_AND_STORE_10 1
+ vpaddd m%1, m%1, m8
+ vpsrad m%1, m%1, xm9
+ vpaddd m%1, m%1, m10
+ vpmaxsd m%1, m%1, m11
+ vpminsd m%1, m%1, m12
+ vextracti128 xm13, m%1, 0
+ vextracti128 xm14, m%1, 1
+ vpackusdw xm%1, xm13, xm14
+ mova [outputq], xm%1
+ add outputq, pitchq
+%endmacro
+
+ NORMALISE_AND_STORE_10 0
+ NORMALISE_AND_STORE_10 1
+ NORMALISE_AND_STORE_10 2
+ NORMALISE_AND_STORE_10 3
+ NORMALISE_AND_STORE_10 4
+ NORMALISE_AND_STORE_10 5
+ NORMALISE_AND_STORE_10 6
+ NORMALISE_AND_STORE_10 7
+
+ RET
diff --git a/libavcodec/x86/apv_dsp_init.c b/libavcodec/x86/apv_dsp_init.c
new file mode 100644
index 0000000000..f7f209cea8
--- /dev/null
+++ b/libavcodec/x86/apv_dsp_init.c
@@ -0,0 +1,41 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+#include "libavutil/attributes.h"
+#include "libavutil/cpu.h"
+#include "libavutil/x86/asm.h"
+#include "libavutil/x86/cpu.h"
+#include "libavcodec/apv_dsp.h"
+
+void ff_apv_decode_transquant_avx2(void *output,
+ ptrdiff_t pitch,
+ const int16_t *input,
+ const int16_t *qmatrix,
+ int64_t bit_depth,
+ int64_t qp_shift);
+
+
+av_cold void ff_apv_dsp_init_x86_64(APVDSPContext *dsp)
+{
+ int cpu_flags = av_get_cpu_flags();
+
+ if (EXTERNAL_AVX2_FAST(cpu_flags)) {
+ dsp->decode_transquant = ff_apv_decode_transquant_avx2;
+ }
+}
diff --git a/tests/checkasm/Makefile b/tests/checkasm/Makefile
index d5c50e5599..193c1e4633 100644
--- a/tests/checkasm/Makefile
+++ b/tests/checkasm/Makefile
@@ -28,6 +28,7 @@ AVCODECOBJS-$(CONFIG_AAC_DECODER) += aacpsdsp.o \
sbrdsp.o
AVCODECOBJS-$(CONFIG_AAC_ENCODER) += aacencdsp.o
AVCODECOBJS-$(CONFIG_ALAC_DECODER) += alacdsp.o
+AVCODECOBJS-$(CONFIG_APV_DECODER) += apv_dsp.o
AVCODECOBJS-$(CONFIG_DCA_DECODER) += synth_filter.o
AVCODECOBJS-$(CONFIG_DIRAC_DECODER) += diracdsp.o
AVCODECOBJS-$(CONFIG_EXR_DECODER) += exrdsp.o
diff --git a/tests/checkasm/apv_dsp.c b/tests/checkasm/apv_dsp.c
new file mode 100644
index 0000000000..a0272d8edc
--- /dev/null
+++ b/tests/checkasm/apv_dsp.c
@@ -0,0 +1,113 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+
+#include "checkasm.h"
+
+#include "libavutil/attributes.h"
+#include "libavutil/mem_internal.h"
+#include "libavcodec/apv_dsp.h"
+
+
+static void check_decode_transquant_8(void)
+{
+ LOCAL_ALIGNED_16(int16_t, input, [64]);
+ LOCAL_ALIGNED_16(int16_t, qmatrix, [64]);
+ LOCAL_ALIGNED_16(uint8_t, new_output, [64]);
+ LOCAL_ALIGNED_16(uint8_t, ref_output, [64]);
+
+ declare_func(void,
+ uint8_t *output,
+ ptrdiff_t pitch,
+ const int16_t *input,
+ const int16_t *qmatrix,
+ int64_t bit_depth,
+ int64_t qp_shift);
+
+ for (int i = 0; i < 64; i++) {
+ // Any signed 12-bit integer.
+ input[i] = rnd() % 2048 - 1024;
+
+ // qmatrix input is premultiplied by level_scale, so
+ // range is 1 to 255 * 71. Interesting values are all
+ // at the low end of that, though.
+ qmatrix[i] = rnd() % 16 + 16;
+ }
+
+ call_ref(ref_output, 8, input, qmatrix, 8, 4);
+ call_new(new_output, 8, input, qmatrix, 8, 4);
+
+ for (int i = 0; i < 64; i++) {
+ if (ref_output[i] != new_output[i])
+ fail();
+ }
+
+ bench_new(new_output, 8, input, qmatrix, 8, 4);
+}
+
+static void check_decode_transquant_10(void)
+{
+ LOCAL_ALIGNED_16( int16_t, input, [64]);
+ LOCAL_ALIGNED_16( int16_t, qmatrix, [64]);
+ LOCAL_ALIGNED_16(uint16_t, new_output, [64]);
+ LOCAL_ALIGNED_16(uint16_t, ref_output, [64]);
+
+ declare_func(void,
+ uint16_t *output,
+ ptrdiff_t pitch,
+ const int16_t *input,
+ const int16_t *qmatrix,
+ int64_t bit_depth,
+ int64_t qp_shift);
+
+ for (int i = 0; i < 64; i++) {
+ // Any signed 14-bit integer.
+ input[i] = rnd() % 16384 - 8192;
+
+ // qmatrix input is premultiplied by level_scale, so
+ // range is 1 to 255 * 71. Interesting values are all
+ // at the low end of that, though.
+ qmatrix[i] = 16; //rnd() % 16 + 16;
+ }
+
+ call_ref(ref_output, 16, input, qmatrix, 10, 4);
+ call_new(new_output, 16, input, qmatrix, 10, 4);
+
+ for (int i = 0; i < 64; i++) {
+ if (ref_output[i] != new_output[i])
+ fail();
+ }
+
+ bench_new(new_output, 16, input, qmatrix, 10, 4);
+}
+
+void checkasm_check_apv_dsp(void)
+{
+ APVDSPContext dsp;
+
+ ff_apv_dsp_init(&dsp);
+
+ if (check_func(dsp.decode_transquant, "decode_transquant_8"))
+ check_decode_transquant_8();
+
+ if (check_func(dsp.decode_transquant, "decode_transquant_10"))
+ check_decode_transquant_10();
+
+ report("apv_dsp");
+}
diff --git a/tests/checkasm/checkasm.c b/tests/checkasm/checkasm.c
index 412b8b2cd1..3bb82ed0e5 100644
--- a/tests/checkasm/checkasm.c
+++ b/tests/checkasm/checkasm.c
@@ -129,6 +129,9 @@ static const struct {
#if CONFIG_ALAC_DECODER
{ "alacdsp", checkasm_check_alacdsp },
#endif
+ #if CONFIG_APV_DECODER
+ { "apv_dsp", checkasm_check_apv_dsp },
+ #endif
#if CONFIG_AUDIODSP
{ "audiodsp", checkasm_check_audiodsp },
#endif
diff --git a/tests/checkasm/checkasm.h b/tests/checkasm/checkasm.h
index ad239fb2a4..a6b5965e02 100644
--- a/tests/checkasm/checkasm.h
+++ b/tests/checkasm/checkasm.h
@@ -83,6 +83,7 @@ void checkasm_check_ac3dsp(void);
void checkasm_check_aes(void);
void checkasm_check_afir(void);
void checkasm_check_alacdsp(void);
+void checkasm_check_apv_dsp(void);
void checkasm_check_audiodsp(void);
void checkasm_check_av_tx(void);
void checkasm_check_blend(void);
--
2.47.2
More information about the ffmpeg-devel
mailing list