[FFmpeg-devel] [PATCH] lavc/aarch64: new optimization for 8-bit hevc_pel_uni_w_pixels, qpel_uni_w_h, qpel_uni_w_v, qpel_uni_w_hv and qpel_h
myais
Logan.Lyu at myais.com.cn
Sun Apr 30 11:57:42 EEST 2023
Hi,
This is a patch for the aarch64, which completes the neon versions of
the hevc_pel_uni_w_pixels, qpel_uni_w_h, qpel_uni_w_v, qpel_uni_w_hv
interfaces.
put_hevc_pel_uni_w_pixels4_8_c: 54.3
put_hevc_pel_uni_w_pixels4_8_neon: 24.1
put_hevc_pel_uni_w_pixels6_8_c: 105.3
put_hevc_pel_uni_w_pixels6_8_neon: 53.1
put_hevc_pel_uni_w_pixels8_8_c: 176.6
put_hevc_pel_uni_w_pixels8_8_neon: 63.8
put_hevc_pel_uni_w_pixels12_8_c: 391.1
put_hevc_pel_uni_w_pixels12_8_neon: 193.3
put_hevc_pel_uni_w_pixels16_8_c: 688.1
put_hevc_pel_uni_w_pixels16_8_neon: 226.1
put_hevc_pel_uni_w_pixels24_8_c: 1542.3
put_hevc_pel_uni_w_pixels24_8_neon: 536.8
put_hevc_pel_uni_w_pixels32_8_c: 2753.1
put_hevc_pel_uni_w_pixels32_8_neon: 875.8
put_hevc_pel_uni_w_pixels48_8_c: 6251.1
put_hevc_pel_uni_w_pixels48_8_neon: 1966.1
put_hevc_pel_uni_w_pixels64_8_c: 11047.1
put_hevc_pel_uni_w_pixels64_8_neon: 3449.8
put_hevc_qpel_uni_w_h4_8_c: 156.6
put_hevc_qpel_uni_w_h4_8_neon: 44.6
put_hevc_qpel_uni_w_h6_8_c: 324.6
put_hevc_qpel_uni_w_h6_8_neon: 103.1
put_hevc_qpel_uni_w_h8_8_c: 549.3
put_hevc_qpel_uni_w_h8_8_neon: 138.6
put_hevc_qpel_uni_w_h12_8_c: 1240.3
put_hevc_qpel_uni_w_h12_8_neon: 277.3
put_hevc_qpel_uni_w_h16_8_c: 2161.8
put_hevc_qpel_uni_w_h16_8_neon: 394.1
put_hevc_qpel_uni_w_h24_8_c: 4874.8
put_hevc_qpel_uni_w_h24_8_neon: 972.6
put_hevc_qpel_uni_w_h32_8_c: 8517.8
put_hevc_qpel_uni_w_h32_8_neon: 1517.3
put_hevc_qpel_uni_w_h48_8_c: 19856.1
put_hevc_qpel_uni_w_h48_8_neon: 3429.8
put_hevc_qpel_uni_w_h64_8_c: 35159.3
put_hevc_qpel_uni_w_h64_8_neon: 6018.1
put_hevc_qpel_uni_w_v4_8_c: 180.6
put_hevc_qpel_uni_w_v4_8_neon: 63.8
put_hevc_qpel_uni_w_v6_8_c: 318.6
put_hevc_qpel_uni_w_v6_8_neon: 117.8
put_hevc_qpel_uni_w_v8_8_c: 547.6
put_hevc_qpel_uni_w_v8_8_neon: 132.1
put_hevc_qpel_uni_w_v12_8_c: 1202.8
put_hevc_qpel_uni_w_v12_8_neon: 350.1
put_hevc_qpel_uni_w_v16_8_c: 2109.6
put_hevc_qpel_uni_w_v16_8_neon: 442.1
put_hevc_qpel_uni_w_v24_8_c: 4748.8
put_hevc_qpel_uni_w_v24_8_neon: 1287.1
put_hevc_qpel_uni_w_v32_8_c: 8487.3
put_hevc_qpel_uni_w_v32_8_neon: 1704.3
put_hevc_qpel_uni_w_v48_8_c: 18798.8
put_hevc_qpel_uni_w_v48_8_neon: 3790.8
put_hevc_qpel_uni_w_v64_8_c: 35614.6
put_hevc_qpel_uni_w_v64_8_neon: 6725.6
put_hevc_qpel_uni_w_hv4_8_c: 498.8
put_hevc_qpel_uni_w_hv4_8_neon: 139.3
put_hevc_qpel_uni_w_hv6_8_c: 874.6
put_hevc_qpel_uni_w_hv6_8_neon: 295.3
put_hevc_qpel_uni_w_hv8_8_c: 1372.1
put_hevc_qpel_uni_w_hv8_8_neon: 387.1
put_hevc_qpel_uni_w_hv12_8_c: 2721.8
put_hevc_qpel_uni_w_hv12_8_neon: 804.8
put_hevc_qpel_uni_w_hv16_8_c: 4503.1
put_hevc_qpel_uni_w_hv16_8_neon: 1038.1
put_hevc_qpel_uni_w_hv24_8_c: 9321.8
put_hevc_qpel_uni_w_hv24_8_neon: 2962.1
put_hevc_qpel_uni_w_hv32_8_c: 15926.8
put_hevc_qpel_uni_w_hv32_8_neon: 3858.6
put_hevc_qpel_uni_w_hv48_8_c: 35051.1
put_hevc_qpel_uni_w_hv48_8_neon: 9301.1
put_hevc_qpel_uni_w_hv64_8_c: 61215.3
put_hevc_qpel_uni_w_hv64_8_neon: 14920.1
put_hevc_qpel_uni_h4_8_c: 143.3
put_hevc_qpel_uni_h4_8_neon: 55.3
put_hevc_qpel_uni_h6_8_c: 304.6
put_hevc_qpel_uni_h6_8_neon: 82.3
put_hevc_qpel_uni_h8_8_c: 557.8
put_hevc_qpel_uni_h8_8_neon: 99.3
put_hevc_qpel_uni_h12_8_c: 1228.3
put_hevc_qpel_uni_h12_8_neon: 251.6
put_hevc_qpel_uni_h16_8_c: 2210.3
put_hevc_qpel_uni_h16_8_neon: 324.6
put_hevc_qpel_uni_h24_8_c: 4859.1
put_hevc_qpel_uni_h24_8_neon: 962.3
put_hevc_qpel_uni_h32_8_c: 8728.6
put_hevc_qpel_uni_h32_8_neon: 1249.6
put_hevc_qpel_uni_h48_8_c: 20346.3
put_hevc_qpel_uni_h48_8_neon: 2824.1
put_hevc_qpel_uni_h64_8_c: 36702.6
put_hevc_qpel_uni_h64_8_neon: 5012.1
Signed-off-by: myais <Logan.Lyu at myais.com.cn>
---
libavcodec/aarch64/hevcdsp_init_aarch64.c | 96 +
libavcodec/aarch64/hevcdsp_qpel_neon.S | 2223 +++++++++++++++++++++
2 files changed, 2319 insertions(+)
diff --git a/libavcodec/aarch64/hevcdsp_init_aarch64.c
b/libavcodec/aarch64/hevcdsp_init_aarch64.c
index be1049a2ec..42b8e9169d 100644
--- a/libavcodec/aarch64/hevcdsp_init_aarch64.c
+++ b/libavcodec/aarch64/hevcdsp_init_aarch64.c
@@ -128,6 +128,91 @@ void ff_hevc_put_hevc_qpel_bi_h16_8_neon(uint8_t
*_dst, ptrdiff_t _dststride, co
ptrdiff_t _srcstride, const
int16_t *src2, int height, intptr_t
mx, intptr_t my, int width);
+#define NEON8_FNPROTO(fn, args, ext) \
+ void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##6_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##12_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##24_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##32_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##48_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##64_8_neon##ext args; \
+
+#define NEON8_FNPROTO_PARTIAL_4(fn, args, ext) \
+ void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##64_8_neon##ext args; \
+
+#define NEON8_FNPROTO_PARTIAL_5(fn, args, ext) \
+ void ff_hevc_put_hevc_##fn##4_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##8_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##16_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##32_8_neon##ext args; \
+ void ff_hevc_put_hevc_##fn##64_8_neon##ext args; \
+
+
+NEON8_FNPROTO(pel_uni_w_pixels, (uint8_t *_dst, ptrdiff_t _dststride,
+ const uint8_t *_src, ptrdiff_t _srcstride,
+ int height, int denom, int wx, int ox, + intptr_t mx,
intptr_t my, int width),);
+
+NEON8_FNPROTO_PARTIAL_4(qpel_uni_w_v, (uint8_t *_dst, ptrdiff_t
_dststride,
+ const uint8_t *_src, ptrdiff_t _srcstride,
+ int height, int denom, int wx, int ox,
+ intptr_t mx, intptr_t my, int width),);
+
+#if defined(__ARM_FEATURE_DOTPROD)
+NEON8_FNPROTO(qpel_h, (int16_t *dst,
+ const uint8_t *_src, ptrdiff_t _srcstride,
+ int height, intptr_t mx, intptr_t my, int width), _dotprod);
+
+NEON8_FNPROTO(qpel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
+ const uint8_t *_src, ptrdiff_t _srcstride,
+ int height, int denom, int wx, int ox,
+ intptr_t mx, intptr_t my, int width), _dotprod);
+
+NEON8_FNPROTO_PARTIAL_5(qpel_uni_w_hv, (uint8_t *_dst, ptrdiff_t
_dststride,
+ const uint8_t *_src, ptrdiff_t _srcstride,
+ int height, int denom, int wx, int ox,
+ intptr_t mx, intptr_t my, int width), _dotprod);
+
+#endif
+
+#define NEON8_FNASSIGN(member, v, h, fn, ext) \
+ member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
+ member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext; \
+ member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
+ member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon##ext; \
+ member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
+ member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon##ext; \
+ member[7][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; \
+ member[8][v][h] = ff_hevc_put_hevc_##fn##48_8_neon##ext; \
+ member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
+
+#define NEON8_FNASSIGN_PARTIAL_4(member, v, h, fn, ext) \
+ member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
+ member[2][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
+ member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
+ member[4][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
+ member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
+ member[6][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
+ member[7][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
+ member[8][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
+ member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
+
+#define NEON8_FNASSIGN_PARTIAL_5(member, v, h, fn, ext) \
+ member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext; \
+ member[2][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
+ member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext; \
+ member[4][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
+ member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
+ member[6][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; \
+ member[7][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext; \
+ member[8][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext; \
+ member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
+
av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int
bit_depth)
{
if (!have_neon(av_get_cpu_flags())) return;
@@ -185,6 +270,17 @@ av_cold void
ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth)
c->put_hevc_qpel_bi[7][0][1] =
c->put_hevc_qpel_bi[8][0][1] =
c->put_hevc_qpel_bi[9][0][1] =
ff_hevc_put_hevc_qpel_bi_h16_8_neon;
+
+ NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 0, pel_uni_w_pixels,);
+ NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 0,
pel_uni_w_pixels,); +
NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,);
+
+ #if defined(__ARM_FEATURE_DOTPROD)
+ NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _dotprod);
+ NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h,
_dotprod);
+ NEON8_FNASSIGN_PARTIAL_5(c->put_hevc_qpel_uni_w, 1, 1,
qpel_uni_w_hv, _dotprod);
+
+ #endif
}
if (bit_depth == 10) {
c->hevc_h_loop_filter_chroma =
ff_hevc_h_loop_filter_chroma_10_neon;
diff --git a/libavcodec/aarch64/hevcdsp_qpel_neon.S
b/libavcodec/aarch64/hevcdsp_qpel_neon.S
index 0e7b912678..e30ac1b465 100644
--- a/libavcodec/aarch64/hevcdsp_qpel_neon.S
+++ b/libavcodec/aarch64/hevcdsp_qpel_neon.S
@@ -30,6 +30,13 @@ const qpel_filters, align=4
.byte 0, 1, -5, 17, 58,-10, 4, -1
endconst
+const qpel_filters_abs, align=4
+ .byte 0, 0, 0, 0, 0, 0, 0, 0
+ .byte 1, 4, 10, 58, 17, 5, 1, 0
+ .byte 1, 4, 11, 40, 40, 11, 4, 1
+ .byte 0, 1, 5, 17, 58, 10, 4, 1
+endconst
+
.macro load_filter m
movrel x15, qpel_filters
add x15, x15, \m, lsl #3
@@ -482,3 +489,2219 @@ endfunc
put_hevc qpel
put_hevc qpel_uni
put_hevc qpel_bi
+
+
+function ff_hevc_put_hevc_pel_uni_w_pixels4_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.8h, w7
+1:
+ ldr s0, [x2]
+ ldr s1, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ushll v0.8h, v0.8b, #6
+ ushll v1.8h, v1.8b, #6
+ smull v0.4s, v0.4h, v30.4h
+ smull v1.4s, v1.4h, v30.4h
+ sqrshl v0.4s, v0.4s, v31.4s
+ sqrshl v1.4s, v1.4s, v31.4s
+ sqadd v0.4s, v0.4s, v29.4s
+ sqadd v1.4s, v1.4s, v29.4s
+ sqxtn v0.4h, v0.4s
+ sqxtn v1.4h, v1.4s
+ sqxtun v0.8b, v0.8h
+ sqxtun v1.8b, v1.8h
+ str s0, [x0]
+ str s1, [x0, x1]
+ add x0, x0, x1, lsl 1
+ subs w4, w4, #2
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_pel_uni_w_pixels6_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+ sub x1, x1, #4
+1:
+ ldr d0, [x2]
+ ldr d1, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ushll v0.8h, v0.8b, #6
+ ushll v1.8h, v1.8b, #6
+ smull v4.4s, v0.4h, v30.4h
+ smull2 v5.4s, v0.8h, v30.8h
+ smull v6.4s, v1.4h, v30.4h
+ smull2 v7.4s, v1.8h, v30.8h
+ sqrshl v4.4s, v4.4s, v31.4s
+ sqrshl v5.4s, v5.4s, v31.4s
+ sqrshl v6.4s, v6.4s, v31.4s
+ sqrshl v7.4s, v7.4s, v31.4s
+ sqadd v4.4s, v4.4s, v29.4s
+ sqadd v5.4s, v5.4s, v29.4s
+ sqadd v6.4s, v6.4s, v29.4s
+ sqadd v7.4s, v7.4s, v29.4s
+ sqxtn v0.4h, v4.4s
+ sqxtn2 v0.8h, v5.4s
+ sqxtn v1.4h, v6.4s
+ sqxtn2 v1.8h, v7.4s
+ sqxtun v0.8b, v0.8h
+ sqxtun v1.8b, v1.8h
+ str s0, [x0], #4
+ st1 {v0.h}[2], [x0], x1
+ str s1, [x0], #4
+ st1 {v1.h}[2], [x0], x1
+ subs w4, w4, #2
+ b.ne 1b
+ ret +endfunc
+
+function ff_hevc_put_hevc_pel_uni_w_pixels8_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+1:
+ ldr d0, [x2]
+ ldr d1, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ushll v0.8h, v0.8b, #6
+ ushll v1.8h, v1.8b, #6
+ smull v4.4s, v0.4h, v30.4h
+ smull2 v5.4s, v0.8h, v30.8h
+ smull v6.4s, v1.4h, v30.4h
+ smull2 v7.4s, v1.8h, v30.8h
+ sqrshl v4.4s, v4.4s, v31.4s
+ sqrshl v5.4s, v5.4s, v31.4s
+ sqrshl v6.4s, v6.4s, v31.4s
+ sqrshl v7.4s, v7.4s, v31.4s
+ sqadd v4.4s, v4.4s, v29.4s
+ sqadd v5.4s, v5.4s, v29.4s
+ sqadd v6.4s, v6.4s, v29.4s
+ sqadd v7.4s, v7.4s, v29.4s
+ sqxtn v0.4h, v4.4s
+ sqxtn2 v0.8h, v5.4s
+ sqxtn v1.4h, v6.4s
+ sqxtn2 v1.8h, v7.4s
+ sqxtun v0.8b, v0.8h
+ sqxtun v1.8b, v1.8h
+ str d0, [x0]
+ str d1, [x0, x1]
+ add x0, x0, x1, lsl 1
+ subs w4, w4, #2
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_pel_uni_w_pixels12_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+ sub x1, x1, #8
+1:
+ ldr q0, [x2]
+ ldr q1, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ushll v4.8h, v0.8b, #6
+ ushll2 v5.8h, v0.16b, #6
+ ushll v6.8h, v1.8b, #6
+ ushll2 v7.8h, v1.16b, #6
+ smull v16.4s, v4.4h, v30.4h
+ smull2 v17.4s, v4.8h, v30.8h
+ smull v18.4s, v5.4h, v30.4h
+ smull2 v19.4s, v5.8h, v30.8h
+ smull v20.4s, v6.4h, v30.4h
+ smull2 v21.4s, v6.8h, v30.8h
+ smull v22.4s, v7.4h, v30.4h
+ smull2 v23.4s, v7.8h, v30.8h
+ + sqrshl v16.4s, v16.4s, v31.4s
+ sqrshl v17.4s, v17.4s, v31.4s
+ sqrshl v18.4s, v18.4s, v31.4s
+ sqrshl v19.4s, v19.4s, v31.4s
+ sqrshl v20.4s, v20.4s, v31.4s
+ sqrshl v21.4s, v21.4s, v31.4s
+ sqrshl v22.4s, v22.4s, v31.4s
+ sqrshl v23.4s, v23.4s, v31.4s
+ sqadd v16.4s, v16.4s, v29.4s
+ sqadd v17.4s, v17.4s, v29.4s
+ sqadd v18.4s, v18.4s, v29.4s
+ sqadd v19.4s, v19.4s, v29.4s
+ sqadd v20.4s, v20.4s, v29.4s
+ sqadd v21.4s, v21.4s, v29.4s
+ sqadd v22.4s, v22.4s, v29.4s
+ sqadd v23.4s, v23.4s, v29.4s
+ sqxtn v0.4h, v16.4s
+ sqxtn2 v0.8h, v17.4s
+ sqxtn v1.4h, v18.4s
+ sqxtn2 v1.8h, v19.4s
+ sqxtn v2.4h, v20.4s
+ sqxtn2 v2.8h, v21.4s
+ sqxtn v3.4h, v22.4s
+ sqxtn2 v3.8h, v23.4s
+ sqxtun v0.8b, v0.8h
+ sqxtun2 v0.16b, v1.8h
+ sqxtun v2.8b, v2.8h
+ sqxtun2 v2.16b, v3.8h
+ str d0, [x0], #8
+ st1 {v0.s}[2], [x0], x1
+ str d2, [x0], #8
+ st1 {v2.s}[2], [x0], x1
+ subs w4, w4, #2
+ b.ne 1b
+ ret
+endfunc
+
+.macro PEL_UNI_W_PIXEL_CALC s0, t0, t1, d0, d1, d2, d3
+ ushll \t0\().8h, \s0\().8b, #6
+ ushll2 \t1\().8h, \s0\().16b, #6
+ smull \d0\().4s, \t0\().4h, v30.4h
+ smull2 \d1\().4s, \t0\().8h, v30.8h
+ smull \d2\().4s, \t1\().4h, v30.4h
+ smull2 \d3\().4s, \t1\().8h, v30.8h
+ sqrshl \d0\().4s, \d0\().4s, v31.4s
+ sqrshl \d1\().4s, \d1\().4s, v31.4s
+ sqrshl \d2\().4s, \d2\().4s, v31.4s
+ sqrshl \d3\().4s, \d3\().4s, v31.4s
+ sqadd \d0\().4s, \d0\().4s, v29.4s
+ sqadd \d1\().4s, \d1\().4s, v29.4s
+ sqadd \d2\().4s, \d2\().4s, v29.4s
+ sqadd \d3\().4s, \d3\().4s, v29.4s
+ sqxtn \t0\().4h, \d0\().4s
+ sqxtn2 \t0\().8h, \d1\().4s
+ sqxtn \t1\().4h, \d2\().4s
+ sqxtn2 \t1\().8h, \d3\().4s
+ sqxtun \s0\().8b, \t0\().8h
+ sqxtun2 \s0\().16b, \t1\().8h
+.endm
+
+
+function ff_hevc_put_hevc_pel_uni_w_pixels16_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+1:
+ ldr q0, [x2]
+ ldr q1, [x2, x3]
+ add x2, x2, x3, lsl 1
+ PEL_UNI_W_PIXEL_CALC v0, v4, v5, v16, v17, v18, v19
+ PEL_UNI_W_PIXEL_CALC v1, v6, v7, v20, v21, v22, v23
+ str q0, [x0]
+ str q1, [x0, x1]
+ add x0, x0, x1, lsl 1
+ subs w4, w4, #2
+ b.ne 1b
+ ret
+endfunc
+
+
+
+function ff_hevc_put_hevc_pel_uni_w_pixels24_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+1:
+ ld1 {v0.16b, v1.16b}, [x2], x3
+ ushll v4.8h, v0.8b, #6
+ ushll2 v5.8h, v0.16b, #6
+ ushll v6.8h, v1.8b, #6
+ smull v16.4s, v4.4h, v30.4h
+ smull2 v17.4s, v4.8h, v30.8h
+ smull v18.4s, v5.4h, v30.4h
+ smull2 v19.4s, v5.8h, v30.8h
+ smull v20.4s, v6.4h, v30.4h
+ smull2 v21.4s, v6.8h, v30.8h
+ sqrshl v16.4s, v16.4s, v31.4s
+ sqrshl v17.4s, v17.4s, v31.4s
+ sqrshl v18.4s, v18.4s, v31.4s
+ sqrshl v19.4s, v19.4s, v31.4s
+ sqrshl v20.4s, v20.4s, v31.4s
+ sqrshl v21.4s, v21.4s, v31.4s
+ sqadd v16.4s, v16.4s, v29.4s
+ sqadd v17.4s, v17.4s, v29.4s
+ sqadd v18.4s, v18.4s, v29.4s
+ sqadd v19.4s, v19.4s, v29.4s
+ sqadd v20.4s, v20.4s, v29.4s
+ sqadd v21.4s, v21.4s, v29.4s
+ sqxtn v0.4h, v16.4s
+ sqxtn2 v0.8h, v17.4s
+ sqxtn v1.4h, v18.4s
+ sqxtn2 v1.8h, v19.4s
+ sqxtn v2.4h, v20.4s
+ sqxtn2 v2.8h, v21.4s
+ sqxtun v0.8b, v0.8h
+ sqxtun v1.8b, v1.8h
+ sqxtun v2.8b, v2.8h
+ st1 {v0.8b, v1.8b, v2.8b}, [x0], x1
+ subs w4, w4, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_pel_uni_w_pixels32_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+1:
+ ld1 {v0.16b, v1.16b}, [x2], x3
+ PEL_UNI_W_PIXEL_CALC v0, v4, v5, v16, v17, v18, v19
+ PEL_UNI_W_PIXEL_CALC v1, v6, v7, v20, v21, v22, v23
+ st1 {v0.16b, v1.16b}, [x0], x1
+ subs w4, w4, #1
+ b.ne 1b
+ ret
+endfunc
+
+
+function ff_hevc_put_hevc_pel_uni_w_pixels48_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+1:
+ ld1 {v0.16b, v1.16b, v2.16b}, [x2], x3
+ PEL_UNI_W_PIXEL_CALC v0, v4, v5, v16, v17, v18, v19
+ PEL_UNI_W_PIXEL_CALC v1, v6, v7, v20, v21, v22, v23
+ PEL_UNI_W_PIXEL_CALC v2, v4, v5, v16, v17, v18, v19
+ st1 {v0.16b, v1.16b, v2.16b}, [x0], x1
+ subs w4, w4, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_pel_uni_w_pixels64_8_neon, export=1
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6
+ dup v31.4s, w10
+ dup v29.4s, w7
+1:
+ ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], x3
+ PEL_UNI_W_PIXEL_CALC v0, v4, v5, v16, v17, v18, v19
+ PEL_UNI_W_PIXEL_CALC v1, v6, v7, v20, v21, v22, v23
+ PEL_UNI_W_PIXEL_CALC v2, v4, v5, v16, v17, v18, v19
+ PEL_UNI_W_PIXEL_CALC v3, v6, v7, v20, v21, v22, v23
+ st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
+ subs w4, w4, #1
+ b.ne 1b
+ ret
+endfunc
+
+.macro QPEL_UNI_W_V_HEADER
+ ldur x12, [sp, #8] // my
+ sub x2, x2, x3, lsl #1
+ sub x2, x2, x3
+ movrel x9, qpel_filters_abs
+ add x9, x9, x12, lsl 3
+ ldr d28, [x9]
+ dup v0.16b, v28.b[0]
+ dup v1.16b, v28.b[1]
+ dup v2.16b, v28.b[2]
+ dup v3.16b, v28.b[3]
+ dup v4.16b, v28.b[4]
+ dup v5.16b, v28.b[5]
+ dup v6.16b, v28.b[6]
+ dup v7.16b, v28.b[7]
+
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.8h, w6 // wx
+ dup v31.4s, w10 // shift
+ dup v29.4s, w7 // ox
+.endm
+
+.macro QPEL_FILTER_B dst, src0, src1, src2, src3, src4, src5, src6, src7
+ umull \dst\().8h, \src1\().8b, v1.8b
+ umlsl \dst\().8h, \src0\().8b, v0.8b
+ umlsl \dst\().8h, \src2\().8b, v2.8b
+ umlal \dst\().8h, \src3\().8b, v3.8b
+ umlal \dst\().8h, \src4\().8b, v4.8b
+ umlsl \dst\().8h, \src5\().8b, v5.8b
+ umlal \dst\().8h, \src6\().8b, v6.8b
+ umlsl \dst\().8h, \src7\().8b, v7.8b
+.endm
+
+.macro QPEL_FILTER_B2 dst, src0, src1, src2, src3, src4, src5, src6, src7
+ umull2 \dst\().8h, \src1\().16b, v1.16b
+ umlsl2 \dst\().8h, \src0\().16b, v0.16b
+ umlsl2 \dst\().8h, \src2\().16b, v2.16b
+ umlal2 \dst\().8h, \src3\().16b, v3.16b
+ umlal2 \dst\().8h, \src4\().16b, v4.16b
+ umlsl2 \dst\().8h, \src5\().16b, v5.16b
+ umlal2 \dst\().8h, \src6\().16b, v6.16b
+ umlsl2 \dst\().8h, \src7\().16b, v7.16b
+.endm
+
+.macro QPEL_UNI_W_V_4
+ smull v24.4s, v24.4h, v30.4h
+ sqrshl v24.4s, v24.4s, v31.4s
+ sqadd v24.4s, v24.4s, v29.4s
+ sqxtn v24.4h, v24.4s
+ sqxtun v24.8b, v24.8h
+ st1 {v24.s}[0], [x0], x1
+.endm
+
+function ff_hevc_put_hevc_qpel_uni_w_v4_8_neon, export=1
+ QPEL_UNI_W_V_HEADER
+ ldr s16, [x2]
+ ldr s17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr s18, [x2]
+ ldr s19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr s20, [x2]
+ ldr s21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr s22, [x2]
+
+1: ldr s23, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v24, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr s16, [x2]
+ QPEL_FILTER_B v24, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr s17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v24, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr s18, [x2]
+ QPEL_FILTER_B v24, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr s19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v24, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr s20, [x2]
+ QPEL_FILTER_B v24, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr s21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v24, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr s22, [x2]
+ QPEL_FILTER_B v24, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_UNI_W_V_4
+ subs w4, w4, #1
+ b.ne 1b
+2:
+ ret
+endfunc
+
+.macro QPEL_UNI_W_V_8
+ smull v24.4s, v26.4h, v30.4h
+ smull2 v25.4s, v26.8h, v30.8h
+ sqrshl v24.4s, v24.4s, v31.4s
+ sqrshl v25.4s, v25.4s, v31.4s
+ sqadd v24.4s, v24.4s, v29.4s
+ sqadd v25.4s, v25.4s, v29.4s
+ sqxtn v24.4h, v24.4s
+ sqxtn2 v24.8h, v25.4s
+ sqxtun v24.8b, v24.8h
+ st1 {v24.d}[0], [x0], x1
+.endm
+
+function ff_hevc_put_hevc_qpel_uni_w_v8_8_neon, export=1
+ QPEL_UNI_W_V_HEADER
+ ldr d16, [x2]
+ ldr d17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr d18, [x2]
+ ldr d19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr d20, [x2]
+ ldr d21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr d22, [x2]
+
+1: ldr d23, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr d16, [x2]
+ QPEL_FILTER_B v26, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr d17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr d18, [x2]
+ QPEL_FILTER_B v26, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr d19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr d20, [x2]
+ QPEL_FILTER_B v26, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr d21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr d22, [x2]
+ QPEL_FILTER_B v26, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_UNI_W_V_8
+ subs w4, w4, #1
+ b.ne 1b
+2:
+ ret
+endfunc
+
+.macro QPEL_UNI_W_V_16
+ smull v24.4s, v26.4h, v30.4h
+ smull2 v25.4s, v26.8h, v30.8h
+ smull v26.4s, v27.4h, v30.4h
+ smull2 v27.4s, v27.8h, v30.8h
+ sqrshl v24.4s, v24.4s, v31.4s
+ sqrshl v25.4s, v25.4s, v31.4s
+ sqrshl v26.4s, v26.4s, v31.4s
+ sqrshl v27.4s, v27.4s, v31.4s
+ sqadd v24.4s, v24.4s, v29.4s
+ sqadd v25.4s, v25.4s, v29.4s
+ sqadd v26.4s, v26.4s, v29.4s
+ sqadd v27.4s, v27.4s, v29.4s
+ sqxtn v24.4h, v24.4s
+ sqxtn2 v24.8h, v25.4s
+ sqxtn v26.4h, v26.4s
+ sqxtn2 v26.8h, v27.4s
+ sqxtun v24.8b, v24.8h
+ sqxtun2 v24.16b, v26.8h
+ st1 {v24.16b}, [x0], x1
+.endm
+
+function ff_hevc_put_hevc_qpel_uni_w_v16_8_neon, export=1
+ QPEL_UNI_W_V_HEADER
+ ldr q16, [x2]
+ ldr q17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr q18, [x2]
+ ldr q19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr q20, [x2]
+ ldr q21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr q22, [x2]
+
+1: ldr q23, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_B2 v27, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q16, [x2]
+ QPEL_FILTER_B v26, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_B2 v27, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_B2 v27, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q18, [x2]
+ QPEL_FILTER_B v26, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_B2 v27, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_B2 v27, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q20, [x2]
+ QPEL_FILTER_B v26, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_B2 v27, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_B2 v27, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q22, [x2]
+ QPEL_FILTER_B v26, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_B2 v27, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.ne 1b
+2:
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_uni_w_v64_8_neon, export=1
+ QPEL_UNI_W_V_HEADER
+ ldur w13, [sp, #16]
+ mov x14, x0
+ mov x15, x2
+ mov w11, w4
+
+3:
+ ldr q16, [x2]
+ ldr q17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr q18, [x2]
+ ldr q19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr q20, [x2]
+ ldr q21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ ldr q22, [x2]
+
+
+1: ldr q23, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_B2 v27, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q16, [x2]
+ QPEL_FILTER_B v26, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_B2 v27, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q17, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_B2 v27, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q18, [x2]
+ QPEL_FILTER_B v26, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_B2 v27, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q19, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_B2 v27, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q20, [x2]
+ QPEL_FILTER_B v26, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_B2 v27, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q21, [x2, x3]
+ add x2, x2, x3, lsl 1
+ QPEL_FILTER_B v26, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_B2 v27, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.eq 2f
+
+ ldr q22, [x2]
+ QPEL_FILTER_B v26, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_B2 v27, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_UNI_W_V_16
+ subs w4, w4, #1
+ b.ne 1b
+2:
+ subs w13, w13, #16
+ add x14, x14, #16
+ add x15, x15, #16
+ mov x0, x14
+ mov x2, x15
+ mov w4, w11
+ b.hi 3b
+ ret
+endfunc
+
+#if __ARM_FEATURE_DOTPROD
+.macro QPEL_UNI_W_H_HEADER
+ ldr x12, [sp]
+ sub x2, x2, #3
+ movrel x9, qpel_filters
+ add x9, x9, x12, lsl 3
+ ldr x11, [x9]
+ dup v28.2d, x11
+ mov w10, #-6
+ sub w10, w10, w5
+ dup v30.4s, w6 // wx
+ dup v31.4s, w10 // shift
+ dup v29.4s, w7 // ox
+.endm
+
+function ff_hevc_put_hevc_qpel_uni_w_h4_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+1:
+ ld1 {v0.16b}, [x2], x3
+ ext v1.16b, v0.16b, v0.16b, #1
+ ext v2.16b, v0.16b, v0.16b, #2
+ ext v3.16b, v0.16b, v0.16b, #3
+ zip1 v0.2d, v0.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ movi v16.2d, #0
+ movi v17.2d, #0
+ usdot v16.4s, v0.16b, v28.16b
+ usdot v17.4s, v2.16b, v28.16b
+ addp v16.4s, v16.4s, v17.4s
+ mul v16.4s, v16.4s, v30.4s
+ sqrshl v16.4s, v16.4s, v31.4s
+ sqadd v16.4s, v16.4s, v29.4s
+ sqxtn v16.4h, v16.4s
+ sqxtun v16.8b, v16.8h
+ str s16, [x0]
+ add x0, x0, x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_uni_w_h6_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+ sub x1, x1, #4
+1:
+ ld1 {v0.16b}, [x2], x3
+ ext v1.16b, v0.16b, v0.16b, #1
+ ext v2.16b, v0.16b, v0.16b, #2
+ ext v3.16b, v0.16b, v0.16b, #3
+ ext v4.16b, v0.16b, v0.16b, #4
+ ext v5.16b, v0.16b, v0.16b, #5
+ zip1 v0.2d, v0.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ zip1 v4.2d, v4.2d, v5.2d
+ movi v16.2d, #0
+ movi v17.2d, #0
+ movi v18.2d, #0
+ usdot v16.4s, v0.16b, v28.16b
+ usdot v17.4s, v2.16b, v28.16b
+ usdot v18.4s, v4.16b, v28.16b
+ addp v16.4s, v16.4s, v17.4s
+ addp v18.4s, v18.4s, v18.4s
+ mul v16.4s, v16.4s, v30.4s
+ mul v18.2s, v18.2s, v30.2s
+ sqrshl v16.4s, v16.4s, v31.4s
+ sqrshl v18.2s, v18.2s, v31.2s
+ sqadd v16.4s, v16.4s, v29.4s
+ sqadd v18.2s, v18.2s, v29.2s
+ sqxtn v16.4h, v16.4s
+ sqxtn2 v16.8h, v18.4s
+ sqxtun v16.8b, v16.8h
+ str s16, [x0], #4
+ st1 {v16.h}[2], [x0], x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+
+.macro QPEL_UNI_W_H_CALC s0, s1, s2, s3, d0, d1, d2, d3
+ movi \d0\().2d, #0
+ movi \d1\().2d, #0
+ movi \d2\().2d, #0
+ movi \d3\().2d, #0
+ usdot \d0\().4s, \s0\().16b, v28.16b
+ usdot \d1\().4s, \s1\().16b, v28.16b
+ usdot \d2\().4s, \s2\().16b, v28.16b
+ usdot \d3\().4s, \s3\().16b, v28.16b
+ addp \d0\().4s, \d0\().4s, \d1\().4s
+ addp \d2\().4s, \d2\().4s, \d3\().4s
+ mul \d0\().4s, \d0\().4s, v30.4s
+ mul \d2\().4s, \d2\().4s, v30.4s
+ sqrshl \d0\().4s, \d0\().4s, v31.4s
+ sqrshl \d2\().4s, \d2\().4s, v31.4s
+ sqadd \d0\().4s, \d0\().4s, v29.4s
+ sqadd \d2\().4s, \d2\().4s, v29.4s
+.endm
+
+.macro QPEL_UNI_W_H_CALC_HALF s0, s1, d0, d1
+ movi \d0\().2d, #0
+ movi \d1\().2d, #0
+ usdot \d0\().4s, \s0\().16b, v28.16b
+ usdot \d1\().4s, \s1\().16b, v28.16b
+ addp \d0\().4s, \d0\().4s, \d1\().4s
+ mul \d0\().4s, \d0\().4s, v30.4s
+ sqrshl \d0\().4s, \d0\().4s, v31.4s
+ sqadd \d0\().4s, \d0\().4s, v29.4s
+.endm
+
+
+function ff_hevc_put_hevc_qpel_uni_w_h8_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+1:
+ ld1 {v16.16b, v17.16b}, [x2], x3
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ zip1 v0.2d, v16.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ zip1 v4.2d, v4.2d, v5.2d
+ zip1 v6.2d, v6.2d, v7.2d
+ QPEL_UNI_W_H_CALC v0, v2, v4, v6, v18, v19, v20, v21
+ sqxtn v18.4h, v18.4s
+ sqxtn2 v18.8h, v20.4s
+ sqxtun v18.8b, v18.8h
+ str d18, [x0]
+ add x0, x0, x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_uni_w_h12_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+ add x13, x0, #8
+1:
+ ld1 {v16.16b, v17.16b}, [x2], x3
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ zip1 v18.2d, v16.2d, v1.2d
+ zip1 v19.2d, v2.2d, v3.2d
+ zip1 v20.2d, v4.2d, v5.2d
+ zip1 v21.2d, v6.2d, v7.2d
+ zip2 v22.2d, v16.2d, v1.2d
+ zip2 v23.2d, v2.2d, v3.2d
+ QPEL_UNI_W_H_CALC v18, v19, v20, v21, v0, v2, v4, v6
+ QPEL_UNI_W_H_CALC_HALF v22, v23, v24, v25
+ sqxtn v0.4h, v0.4s
+ sqxtn2 v0.8h, v4.4s
+ sqxtn v1.4h, v24.4s
+ sqxtun v0.8b, v0.8h
+ sqxtun v1.8b, v1.8h
+
+ str d0, [x0]
+ str s1, [x13]
+ add x0, x0, x1
+ add x13, x13, x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_uni_w_h16_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+1:
+ ld1 {v16.16b, v17.16b}, [x2], x3
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_UNI_W_H_CALC v16, v2, v1, v3, v18, v19, v20, v21 //
v18: 0, 8, 2, 10 v20: 1, 9, 3, 11
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v23, v24, v25 //
v22: 4, 12, 6, 14 v24: 5, 13, 7, 15
+ sqxtn v0.4h, v18.4s
+ sqxtn2 v0.8h, v22.4s
+ sqxtn v1.4h, v20.4s
+ sqxtn2 v1.8h, v24.4s
+ trn1 v2.8h, v0.8h, v1.8h
+ trn2 v3.8h, v0.8h, v1.8h
+ sqxtun v0.8b, v2.8h
+ sqxtun2 v0.16b, v3.8h
+ st1 {v0.16b}, [x0], x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_uni_w_h24_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+ sub x1, x1, #16
+1:
+ ld1 {v16.16b, v17.16b}, [x2], x3
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_UNI_W_H_CALC v16, v2, v1, v3, v18, v19, v20, v21
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v23, v24, v25
+ sqxtn v18.4h, v18.4s
+ sqxtn2 v18.8h, v22.4s
+ sqxtn v19.4h, v20.4s
+ sqxtn2 v19.8h, v24.4s
+ trn1 v20.8h, v18.8h, v19.8h
+ trn2 v21.8h, v18.8h, v19.8h
+ sqxtun v26.8b, v20.8h
+ sqxtun2 v26.16b, v21.8h // 0-15
+ ext v1.16b, v17.16b, v17.16b, #1
+ ext v2.16b, v17.16b, v17.16b, #2
+ ext v3.16b, v17.16b, v17.16b, #3
+ ext v4.16b, v17.16b, v17.16b, #4
+ ext v5.16b, v17.16b, v17.16b, #5
+ ext v6.16b, v17.16b, v17.16b, #6
+ ext v7.16b, v17.16b, v17.16b, #7
+ zip1 v0.2d, v17.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ zip1 v4.2d, v4.2d, v5.2d
+ zip1 v6.2d, v6.2d, v7.2d
+ QPEL_UNI_W_H_CALC v0, v2, v4, v6, v18, v19, v20, v21
+ sqxtn v18.4h, v18.4s
+ sqxtn2 v18.8h, v20.4s
+ sqxtun v27.8b, v18.8h
+
+ st1 {v26.16b}, [x0], #16
+ st1 {v27.8b}, [x0], x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+
+function ff_hevc_put_hevc_qpel_uni_w_h32_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+1:
+ ld1 {v16.16b, v17.16b, v18.16b}, [x2], x3
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_UNI_W_H_CALC v16, v2, v1, v3, v0, v19, v20, v21
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v23, v24, v25
+ sqxtn v0.4h, v0.4s
+ sqxtn2 v0.8h, v22.4s
+ sqxtn v19.4h, v20.4s
+ sqxtn2 v19.8h, v24.4s
+ trn1 v20.8h, v0.8h, v19.8h
+ trn2 v21.8h, v0.8h, v19.8h
+ sqxtun v26.8b, v20.8h
+ sqxtun2 v26.16b, v21.8h // 0-15
+ ext v1.16b, v17.16b, v18.16b, #1
+ ext v2.16b, v17.16b, v18.16b, #2
+ ext v3.16b, v17.16b, v18.16b, #3
+ ext v4.16b, v17.16b, v18.16b, #4
+ ext v5.16b, v17.16b, v18.16b, #5
+ ext v6.16b, v17.16b, v18.16b, #6
+ ext v7.16b, v17.16b, v18.16b, #7
+ QPEL_UNI_W_H_CALC v17, v2, v1, v3, v0, v19, v20, v21
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v23, v24, v25
+ sqxtn v0.4h, v0.4s
+ sqxtn2 v0.8h, v22.4s
+ sqxtn v19.4h, v20.4s
+ sqxtn2 v19.8h, v24.4s
+ trn1 v20.8h, v0.8h, v19.8h
+ trn2 v21.8h, v0.8h, v19.8h
+ sqxtun v27.8b, v20.8h
+ sqxtun2 v27.16b, v21.8h // 16-31
+ st1 {v26.16b, v27.16b}, [x0], x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_uni_w_h48_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+1:
+ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x2], x3
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_UNI_W_H_CALC v16, v2, v1, v3, v20, v24, v21, v0
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v24, v23, v0
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ sqxtn v21.4h, v21.4s
+ sqxtn2 v21.8h, v23.4s
+ trn1 v22.8h, v20.8h, v21.8h
+ trn2 v23.8h, v20.8h, v21.8h
+ sqxtun v25.8b, v22.8h
+ sqxtun2 v25.16b, v23.8h // 0-15
+ ext v1.16b, v17.16b, v18.16b, #1
+ ext v2.16b, v17.16b, v18.16b, #2
+ ext v3.16b, v17.16b, v18.16b, #3
+ ext v4.16b, v17.16b, v18.16b, #4
+ ext v5.16b, v17.16b, v18.16b, #5
+ ext v6.16b, v17.16b, v18.16b, #6
+ ext v7.16b, v17.16b, v18.16b, #7
+ QPEL_UNI_W_H_CALC v17, v2, v1, v3, v20, v24, v21, v0
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v24, v23, v0
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ sqxtn v21.4h, v21.4s
+ sqxtn2 v21.8h, v23.4s
+ trn1 v22.8h, v20.8h, v21.8h
+ trn2 v23.8h, v20.8h, v21.8h
+ sqxtun v26.8b, v22.8h
+ sqxtun2 v26.16b, v23.8h // 16-31
+ ext v1.16b, v18.16b, v19.16b, #1
+ ext v2.16b, v18.16b, v19.16b, #2
+ ext v3.16b, v18.16b, v19.16b, #3
+ ext v4.16b, v18.16b, v19.16b, #4
+ ext v5.16b, v18.16b, v19.16b, #5
+ ext v6.16b, v18.16b, v19.16b, #6
+ ext v7.16b, v18.16b, v19.16b, #7
+ QPEL_UNI_W_H_CALC v18, v2, v1, v3, v20, v24, v21, v0
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v24, v23, v0
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ sqxtn v21.4h, v21.4s
+ sqxtn2 v21.8h, v23.4s
+ trn1 v22.8h, v20.8h, v21.8h
+ trn2 v23.8h, v20.8h, v21.8h
+ sqxtun v27.8b, v22.8h
+ sqxtun2 v27.16b, v23.8h // 32-47
+ st1 {v25.16b, v26.16b, v27.16b}, [x0], x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+
+
+function ff_hevc_put_hevc_qpel_uni_w_h64_8_neon_dotprod, export=1
+ QPEL_UNI_W_H_HEADER
+ sub x3, x3, #64
+1:
+ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x2], #64
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_UNI_W_H_CALC v16, v2, v1, v3, v20, v24, v21, v0
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v24, v23, v0
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ sqxtn v21.4h, v21.4s
+ sqxtn2 v21.8h, v23.4s
+ trn1 v22.8h, v20.8h, v21.8h
+ trn2 v23.8h, v20.8h, v21.8h
+ sqxtun v16.8b, v22.8h
+ sqxtun2 v16.16b, v23.8h // 0-15
+ ext v1.16b, v17.16b, v18.16b, #1
+ ext v2.16b, v17.16b, v18.16b, #2
+ ext v3.16b, v17.16b, v18.16b, #3
+ ext v4.16b, v17.16b, v18.16b, #4
+ ext v5.16b, v17.16b, v18.16b, #5
+ ext v6.16b, v17.16b, v18.16b, #6
+ ext v7.16b, v17.16b, v18.16b, #7
+ QPEL_UNI_W_H_CALC v17, v2, v1, v3, v20, v24, v21, v0
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v24, v23, v0
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ sqxtn v21.4h, v21.4s
+ sqxtn2 v21.8h, v23.4s
+ trn1 v22.8h, v20.8h, v21.8h
+ trn2 v23.8h, v20.8h, v21.8h
+ sqxtun v17.8b, v22.8h
+ sqxtun2 v17.16b, v23.8h // 16-31
+ ext v1.16b, v18.16b, v19.16b, #1
+ ext v2.16b, v18.16b, v19.16b, #2
+ ext v3.16b, v18.16b, v19.16b, #3
+ ext v4.16b, v18.16b, v19.16b, #4
+ ext v5.16b, v18.16b, v19.16b, #5
+ ext v6.16b, v18.16b, v19.16b, #6
+ ext v7.16b, v18.16b, v19.16b, #7
+ QPEL_UNI_W_H_CALC v18, v2, v1, v3, v20, v24, v21, v0
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v24, v23, v0
+ ld1 {v0.16b}, [x2], x3
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ sqxtn v21.4h, v21.4s
+ sqxtn2 v21.8h, v23.4s
+ trn1 v22.8h, v20.8h, v21.8h
+ trn2 v23.8h, v20.8h, v21.8h
+ sqxtun v18.8b, v22.8h
+ sqxtun2 v18.16b, v23.8h // 32-47
+ ext v1.16b, v19.16b, v0.16b, #1
+ ext v2.16b, v19.16b, v0.16b, #2
+ ext v3.16b, v19.16b, v0.16b, #3
+ ext v4.16b, v19.16b, v0.16b, #4
+ ext v5.16b, v19.16b, v0.16b, #5
+ ext v6.16b, v19.16b, v0.16b, #6
+ ext v7.16b, v19.16b, v0.16b, #7
+ QPEL_UNI_W_H_CALC v19, v2, v1, v3, v20, v24, v21, v0
+ QPEL_UNI_W_H_CALC v4, v6, v5, v7, v22, v24, v23, v0
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ sqxtn v21.4h, v21.4s
+ sqxtn2 v21.8h, v23.4s
+ trn1 v22.8h, v20.8h, v21.8h
+ trn2 v23.8h, v20.8h, v21.8h
+ sqxtun v19.8b, v22.8h
+ sqxtun2 v19.16b, v23.8h // 48-63
+
+ st1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x0], x1
+ subs w4, w4, #1
+ b.hi 1b
+ ret
+endfunc
+
+
+.macro QPEL_H_HEADER
+ movrel x9, qpel_filters
+ add x9, x9, x4, lsl 3
+ ldr x11, [x9]
+ dup v31.2d, x11
+ sub x1, x1, #3
+.endm
+
+function ff_hevc_put_hevc_qpel_h4_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2
+1:
+ ld1 {v0.16b}, [x1], x2
+ ext v1.16b, v0.16b, v0.16b, #1
+ ext v2.16b, v0.16b, v0.16b, #2
+ ext v3.16b, v0.16b, v0.16b, #3
+ zip1 v0.2d, v0.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ movi v16.2d, #0
+ movi v17.2d, #0
+ usdot v16.4s, v0.16b, v31.16b
+ usdot v17.4s, v2.16b, v31.16b
+ addp v16.4s, v16.4s, v17.4s
+ sqxtn v16.4h, v16.4s
+ str d16, [x0]
+ add x0, x0, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_h6_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2
+ add x15, x0, #8
+1:
+ ld1 {v0.16b}, [x1], x2
+ ext v1.16b, v0.16b, v0.16b, #1
+ ext v2.16b, v0.16b, v0.16b, #2
+ ext v3.16b, v0.16b, v0.16b, #3
+ ext v4.16b, v0.16b, v0.16b, #4
+ ext v5.16b, v0.16b, v0.16b, #5
+ zip1 v0.2d, v0.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ zip1 v4.2d, v4.2d, v5.2d
+ movi v16.2d, #0
+ movi v17.2d, #0
+ movi v18.2d, #0
+ usdot v16.4s, v0.16b, v31.16b
+ usdot v17.4s, v2.16b, v31.16b
+ usdot v18.4s, v4.16b, v31.16b
+ addp v16.4s, v16.4s, v17.4s
+ addp v18.4s, v18.4s, v18.4s
+ sqxtn v16.4h, v16.4s
+ sqxtn v18.4h, v18.4s
+ str d16, [x0]
+ str s18, [x15]
+ add x0, x0, x10
+ add x15, x15, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_h8_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2
+1:
+ ld1 {v0.16b}, [x1], x2
+ ext v1.16b, v0.16b, v0.16b, #1
+ ext v2.16b, v0.16b, v0.16b, #2
+ ext v3.16b, v0.16b, v0.16b, #3
+ ext v4.16b, v0.16b, v0.16b, #4
+ ext v5.16b, v0.16b, v0.16b, #5
+ ext v6.16b, v0.16b, v0.16b, #6
+ ext v7.16b, v0.16b, v0.16b, #7
+ zip1 v0.2d, v0.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ zip1 v4.2d, v4.2d, v5.2d
+ zip1 v6.2d, v6.2d, v7.2d
+ movi v16.2d, #0
+ movi v17.2d, #0
+ movi v18.2d, #0
+ movi v19.2d, #0
+ usdot v16.4s, v0.16b, v31.16b
+ usdot v17.4s, v2.16b, v31.16b
+ usdot v18.4s, v4.16b, v31.16b
+ usdot v19.4s, v6.16b, v31.16b
+ addp v16.4s, v16.4s, v17.4s
+ addp v18.4s, v18.4s, v19.4s
+ sqxtn v16.4h, v16.4s
+ sqxtn2 v16.8h, v18.4s
+ str q16, [x0]
+ add x0, x0, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+.macro QPEL_H_CALC s0, s1, s2, s3, d0, d1, d2, d3 + movi
\d0\().2d, #0
+ movi \d1\().2d, #0
+ movi \d2\().2d, #0
+ movi \d3\().2d, #0
+ usdot \d0\().4s, \s0\().16b, v31.16b
+ usdot \d1\().4s, \s1\().16b, v31.16b
+ usdot \d2\().4s, \s2\().16b, v31.16b
+ usdot \d3\().4s, \s3\().16b, v31.16b
+.endm
+
+function ff_hevc_put_hevc_qpel_h12_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2
+ add x15, x0, #16
+1:
+ ld1 {v16.16b, v17.16b}, [x1], x2
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ zip1 v18.2d, v4.2d, v5.2d
+ zip1 v19.2d, v6.2d, v7.2d
+ QPEL_H_CALC v16, v1, v2, v3, v20, v21, v22, v23
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ movi v24.2d, #0
+ movi v25.2d, #0
+ usdot v24.4s, v18.16b, v31.16b
+ usdot v25.4s, v19.16b, v31.16b
+ addp v24.4s, v24.4s, v25.4s
+ trn1 v26.4s, v20.4s, v21.4s
+ trn2 v27.4s, v20.4s, v21.4s
+ sqxtn v26.4h, v26.4s
+ sqxtn v27.4h, v27.4s
+ sqxtn2 v26.8h, v24.4s
+
+ str q26, [x0]
+ str d27, [x15]
+ add x0, x0, x10
+ add x15, x15, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_h16_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2
+1:
+ ld1 {v16.16b, v17.16b}, [x1], x2
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+
+ QPEL_H_CALC v16, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+
+ sqxtn v18.4h, v22.4s
+ sqxtn2 v18.8h, v26.4s
+ sqxtn v19.4h, v23.4s
+ sqxtn2 v19.8h, v27.4s
+ + stp q18, q19, [x0]
+ add x0, x0, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_h24_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2
+ add x15, x0, #32
+1:
+ ld1 {v16.16b, v17.16b}, [x1], x2
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_H_CALC v16, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v18.4h, v22.4s
+ sqxtn2 v18.8h, v26.4s
+ sqxtn v19.4h, v23.4s
+ sqxtn2 v19.8h, v27.4s
+ stp q18, q19, [x0]
+ add x0, x0, x10
+ ext v1.16b, v17.16b, v17.16b, #1
+ ext v2.16b, v17.16b, v17.16b, #2
+ ext v3.16b, v17.16b, v17.16b, #3
+ ext v4.16b, v17.16b, v17.16b, #4
+ ext v5.16b, v17.16b, v17.16b, #5
+ ext v6.16b, v17.16b, v17.16b, #6
+ ext v7.16b, v17.16b, v17.16b, #7
+ zip1 v0.2d, v17.2d, v1.2d
+ zip1 v2.2d, v2.2d, v3.2d
+ zip1 v4.2d, v4.2d, v5.2d
+ zip1 v6.2d, v6.2d, v7.2d
+ QPEL_H_CALC v0, v2, v4, v5, v20, v21, v22, v23
+ addp v20.4s, v20.4s, v21.4s
+ addp v22.4s, v22.4s, v23.4s
+ sqxtn v20.4h, v20.4s
+ sqxtn2 v20.8h, v22.4s
+ str q20, [x15]
+ add x15, x15, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_h32_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2
+ add x15, x0, #32
+1:
+ ld1 {v16.16b, v17.16b, v18.16b}, [x1], x2
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_H_CALC v16, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0]
+ add x0, x0, x10
+ ext v1.16b, v17.16b, v18.16b, #1
+ ext v2.16b, v17.16b, v18.16b, #2
+ ext v3.16b, v17.16b, v18.16b, #3
+ ext v4.16b, v17.16b, v18.16b, #4
+ ext v5.16b, v17.16b, v18.16b, #5
+ ext v6.16b, v17.16b, v18.16b, #6
+ ext v7.16b, v17.16b, v18.16b, #7
+ QPEL_H_CALC v17, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x15]
+ add x15, x15, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_h48_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ mov x10, #MAX_PB_SIZE * 2 - 64
+1:
+ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x1], x2
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_H_CALC v16, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0], #32
+
+ ext v1.16b, v17.16b, v18.16b, #1
+ ext v2.16b, v17.16b, v18.16b, #2
+ ext v3.16b, v17.16b, v18.16b, #3
+ ext v4.16b, v17.16b, v18.16b, #4
+ ext v5.16b, v17.16b, v18.16b, #5
+ ext v6.16b, v17.16b, v18.16b, #6
+ ext v7.16b, v17.16b, v18.16b, #7
+ QPEL_H_CALC v17, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0], #32
+ ext v1.16b, v18.16b, v19.16b, #1
+ ext v2.16b, v18.16b, v19.16b, #2
+ ext v3.16b, v18.16b, v19.16b, #3
+ ext v4.16b, v18.16b, v19.16b, #4
+ ext v5.16b, v18.16b, v19.16b, #5
+ ext v6.16b, v18.16b, v19.16b, #6
+ ext v7.16b, v18.16b, v19.16b, #7
+ QPEL_H_CALC v18, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0]
+ add x0, x0, x10
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_h64_8_neon_dotprod, export=1
+ QPEL_H_HEADER
+ sub x2, x2, #64
+1:
+ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x1], #64
+ ext v1.16b, v16.16b, v17.16b, #1
+ ext v2.16b, v16.16b, v17.16b, #2
+ ext v3.16b, v16.16b, v17.16b, #3
+ ext v4.16b, v16.16b, v17.16b, #4
+ ext v5.16b, v16.16b, v17.16b, #5
+ ext v6.16b, v16.16b, v17.16b, #6
+ ext v7.16b, v16.16b, v17.16b, #7
+ QPEL_H_CALC v16, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0], #32
+
+ ext v1.16b, v17.16b, v18.16b, #1
+ ext v2.16b, v17.16b, v18.16b, #2
+ ext v3.16b, v17.16b, v18.16b, #3
+ ext v4.16b, v17.16b, v18.16b, #4
+ ext v5.16b, v17.16b, v18.16b, #5
+ ext v6.16b, v17.16b, v18.16b, #6
+ ext v7.16b, v17.16b, v18.16b, #7
+ QPEL_H_CALC v17, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0], #32
+ ext v1.16b, v18.16b, v19.16b, #1
+ ext v2.16b, v18.16b, v19.16b, #2
+ ext v3.16b, v18.16b, v19.16b, #3
+ ext v4.16b, v18.16b, v19.16b, #4
+ ext v5.16b, v18.16b, v19.16b, #5
+ ext v6.16b, v18.16b, v19.16b, #6
+ ext v7.16b, v18.16b, v19.16b, #7
+ QPEL_H_CALC v18, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0], #32
+ ld1 {v28.8b}, [x1], x2
+ ext v1.16b, v19.16b, v28.16b, #1
+ ext v2.16b, v19.16b, v28.16b, #2
+ ext v3.16b, v19.16b, v28.16b, #3
+ ext v4.16b, v19.16b, v28.16b, #4
+ ext v5.16b, v19.16b, v28.16b, #5
+ ext v6.16b, v19.16b, v28.16b, #6
+ ext v7.16b, v19.16b, v28.16b, #7
+ QPEL_H_CALC v19, v1, v2, v3, v20, v21, v22, v23
+ QPEL_H_CALC v4, v5, v6, v7, v24, v25, v26, v27
+ addp v20.4s, v20.4s, v22.4s
+ addp v21.4s, v21.4s, v23.4s
+ addp v24.4s, v24.4s, v26.4s
+ addp v25.4s, v25.4s, v27.4s
+ trn1 v22.4s, v20.4s, v21.4s
+ trn2 v23.4s, v20.4s, v21.4s
+ trn1 v26.4s, v24.4s, v25.4s
+ trn2 v27.4s, v24.4s, v25.4s
+ sqxtn v20.4h, v22.4s
+ sqxtn2 v20.8h, v26.4s
+ sqxtn v21.4h, v23.4s
+ sqxtn2 v21.8h, v27.4s
+ stp q20, q21, [x0], #32
+ subs w3, w3, #1
+ b.ne 1b
+ ret
+endfunc
+
+.macro QPEL_UNI_W_HV_HEADER width
+ ldp x14, x15, [sp] // mx, my
+ ldr w13, [sp, #16] // width
+ stp x20, x21, [sp, #-16]!
+ stp x22, x23, [sp, #-16]!
+ stp x24, x25, [sp, #-16]!
+ stp x26, x27, [sp, #-16]!
+ stp x28, x30, [sp, #-16]!
+ mov x28, sp
+ mov x11, #9088
+ sub sp, sp, x11
+ mov x20, x0
+ mov x21, x1
+ mov x0, sp
+ sub x1, x2, x3, lsl 1
+ sub x1, x1, x3
+ mov x2, x3
+ add w3, w4, #7
+ mov w22, w4 // height
+ mov x4, x14 // mx
+ mov x23, x15 // my
+ mov w24, w6 // wx
+ mov w25, w7 // ox
+ mov w26, #-6
+ sub w26, w26, w5 // -shift
+ mov w27, w13 // width
+ bl X(ff_hevc_put_hevc_qpel_h\width\()_8_neon_dotprod)
+ movrel x9, qpel_filters
+ add x9, x9, x23, lsl 3
+ ld1 {v0.8b}, [x9]
+ sxtl v0.8h, v0.8b
+ mov x10, #(MAX_PB_SIZE * 2)
+ dup v28.4s, w24
+ dup v29.4s, w25
+ dup v30.4s, w26
+.endm
+
+.macro QPEL_UNI_W_HV_END
+ mov sp, x28
+ ldp x28, x30, [sp], #16
+ ldp x26, x27, [sp], #16
+ ldp x24, x25, [sp], #16
+ ldp x22, x23, [sp], #16
+ ldp x20, x21, [sp], #16
+.endm
+
+.macro QPEL_UNI_W_HV_4
+ sshr v26.4s, v26.4s, #6
+ mul v24.4s, v26.4s, v28.4s
+ sqrshl v24.4s, v24.4s, v30.4s
+ sqadd v24.4s, v24.4s, v29.4s
+ sqxtn v24.4h, v24.4s
+ sqxtun v24.8b, v24.8h
+ st1 {v24.s}[0], [x20], x21
+.endm
+
+.macro QPEL_FILTER_H dst, src0, src1, src2, src3, src4, src5, src6, src7
+ smull \dst\().4s, \src0\().4h, v0.h[0]
+ smlal \dst\().4s, \src1\().4h, v0.h[1]
+ smlal \dst\().4s, \src2\().4h, v0.h[2]
+ smlal \dst\().4s, \src3\().4h, v0.h[3]
+ smlal \dst\().4s, \src4\().4h, v0.h[4]
+ smlal \dst\().4s, \src5\().4h, v0.h[5]
+ smlal \dst\().4s, \src6\().4h, v0.h[6]
+ smlal \dst\().4s, \src7\().4h, v0.h[7]
+.endm
+
+.macro QPEL_FILTER_H2 dst, src0, src1, src2, src3, src4, src5, src6,
src7
+ smull2 \dst\().4s, \src0\().8h, v0.h[0]
+ smlal2 \dst\().4s, \src1\().8h, v0.h[1]
+ smlal2 \dst\().4s, \src2\().8h, v0.h[2]
+ smlal2 \dst\().4s, \src3\().8h, v0.h[3]
+ smlal2 \dst\().4s, \src4\().8h, v0.h[4]
+ smlal2 \dst\().4s, \src5\().8h, v0.h[5]
+ smlal2 \dst\().4s, \src6\().8h, v0.h[6]
+ smlal2 \dst\().4s, \src7\().8h, v0.h[7]
+.endm
+
+function ff_hevc_put_hevc_qpel_uni_w_hv4_8_neon_dotprod, export=1
+ QPEL_UNI_W_HV_HEADER 4
+ ldr d16, [sp]
+ ldr d17, [sp, x10]
+ add sp, sp, x10, lsl 1
+ ldr d18, [sp]
+ ldr d19, [sp, x10]
+ add sp, sp, x10, lsl 1
+ ldr d20, [sp]
+ ldr d21, [sp, x10]
+ add sp, sp, x10, lsl 1
+ ldr d22, [sp]
+ add sp, sp, x10
+1:
+ ldr d23, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr d16, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr d17, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr d18, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr d19, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr d20, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr d21, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr d22, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_UNI_W_HV_4
+ subs w22, w22, #1
+ b.hi 1b
+
+2:
+ QPEL_UNI_W_HV_END
+ ret
+endfunc
+
+.macro QPEL_UNI_W_HV_8
+ sshr v26.4s, v26.4s, #6
+ sshr v27.4s, v27.4s, #6
+ mul v24.4s, v26.4s, v28.4s
+ mul v25.4s, v27.4s, v28.4s
+ sqrshl v24.4s, v24.4s, v30.4s
+ sqrshl v25.4s, v25.4s, v30.4s
+ sqadd v24.4s, v24.4s, v29.4s
+ sqadd v25.4s, v25.4s, v29.4s
+ sqxtn v24.4h, v24.4s
+ sqxtn2 v24.8h, v25.4s
+ sqxtun v24.8b, v24.8h
+ st1 {v24.d}[0], [x20], x21
+.endm
+
+function ff_hevc_put_hevc_qpel_uni_w_hv8_8_neon_dotprod, export=1
+ QPEL_UNI_W_HV_HEADER 8
+ ldr q16, [sp]
+ ldr q17, [sp, x10]
+ add sp, sp, x10, lsl 1
+ ldr q18, [sp]
+ ldr q19, [sp, x10]
+ add sp, sp, x10, lsl 1
+ ldr q20, [sp]
+ ldr q21, [sp, x10]
+ add sp, sp, x10, lsl 1
+ ldr q22, [sp]
+ add sp, sp, x10
+1:
+ ldr q23, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_H2 v27, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr q16, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_H2 v27, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr q17, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_H2 v27, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr q18, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_H2 v27, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr q19, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_H2 v27, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr q20, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_H2 v27, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr q21, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_H2 v27, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldr q22, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v26, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_H2 v27, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_UNI_W_HV_8
+ subs w22, w22, #1
+ b.hi 1b
+
+2:
+ QPEL_UNI_W_HV_END
+ ret
+endfunc
+
+.macro QPEL_UNI_W_HV_16
+ sshr v24.4s, v24.4s, #6
+ sshr v25.4s, v25.4s, #6
+ sshr v26.4s, v26.4s, #6
+ sshr v27.4s, v27.4s, #6
+ mul v24.4s, v24.4s, v28.4s
+ mul v25.4s, v25.4s, v28.4s
+ mul v26.4s, v26.4s, v28.4s
+ mul v27.4s, v27.4s, v28.4s
+ sqrshl v24.4s, v24.4s, v30.4s
+ sqrshl v25.4s, v25.4s, v30.4s
+ sqrshl v26.4s, v26.4s, v30.4s
+ sqrshl v27.4s, v27.4s, v30.4s
+ sqadd v24.4s, v24.4s, v29.4s
+ sqadd v25.4s, v25.4s, v29.4s
+ sqadd v26.4s, v26.4s, v29.4s
+ sqadd v27.4s, v27.4s, v29.4s
+ sqxtn v24.4h, v24.4s
+ sqxtn2 v24.8h, v25.4s
+ sqxtn v26.4h, v26.4s
+ sqxtn2 v26.8h, v27.4s
+ sqxtun v24.8b, v24.8h
+ sqxtun2 v24.16b, v26.8h
+
+ st1 {v24.16b}, [x20], x21
+.endm
+
+function ff_hevc_put_hevc_qpel_uni_w_hv16_8_neon_dotprod, export=1
+ QPEL_UNI_W_HV_HEADER 16
+ ldp q16, q1, [sp]
+ add sp, sp, x10
+ ldp q17, q2, [sp]
+ add sp, sp, x10
+ ldp q18, q3, [sp]
+ add sp, sp, x10
+ ldp q19, q4, [sp]
+ add sp, sp, x10
+ ldp q20, q5, [sp]
+ add sp, sp, x10
+ ldp q21, q6, [sp]
+ add sp, sp, x10
+ ldp q22, q7, [sp]
+ add sp, sp, x10
+1:
+ ldp q23, q31, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_H2 v25, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_H v26, v1, v2, v3, v4, v5, v6, v7, v31
+ QPEL_FILTER_H2 v27, v1, v2, v3, v4, v5, v6, v7, v31
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q16, q1, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_H2 v25, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_H v26, v2, v3, v4, v5, v6, v7, v31, v1
+ QPEL_FILTER_H2 v27, v2, v3, v4, v5, v6, v7, v31, v1
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q17, q2, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_H2 v25, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_H v26, v3, v4, v5, v6, v7, v31, v1, v2
+ QPEL_FILTER_H2 v27, v3, v4, v5, v6, v7, v31, v1, v2
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q18, q3, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_H2 v25, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_H v26, v4, v5, v6, v7, v31, v1, v2, v3
+ QPEL_FILTER_H2 v27, v4, v5, v6, v7, v31, v1, v2, v3
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q19, q4, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_H2 v25, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_H v26, v5, v6, v7, v31, v1, v2, v3, v4
+ QPEL_FILTER_H2 v27, v5, v6, v7, v31, v1, v2, v3, v4
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q20, q5, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_H2 v25, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_H v26, v6, v7, v31, v1, v2, v3, v4, v5
+ QPEL_FILTER_H2 v27, v6, v7, v31, v1, v2, v3, v4, v5
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q21, q6, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_H2 v25, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_H v26, v7, v31, v1, v2, v3, v4, v5, v6
+ QPEL_FILTER_H2 v27, v7, v31, v1, v2, v3, v4, v5, v6
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q22, q7, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_H2 v25, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_H v26, v31, v1, v2, v3, v4, v5, v6, v7
+ QPEL_FILTER_H2 v27, v31, v1, v2, v3, v4, v5, v6, v7
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.hi 1b
+
+2:
+ QPEL_UNI_W_HV_END
+ ret
+endfunc
+
+
+function ff_hevc_put_hevc_qpel_uni_w_hv32_8_neon_dotprod, export=1
+ QPEL_UNI_W_HV_HEADER 32
+ mov x11, sp
+ mov w12, w22
+ mov x13, x20
+3:
+ ldp q16, q1, [sp]
+ add sp, sp, x10
+ ldp q17, q2, [sp]
+ add sp, sp, x10
+ ldp q18, q3, [sp]
+ add sp, sp, x10
+ ldp q19, q4, [sp]
+ add sp, sp, x10
+ ldp q20, q5, [sp]
+ add sp, sp, x10
+ ldp q21, q6, [sp]
+ add sp, sp, x10
+ ldp q22, q7, [sp]
+ add sp, sp, x10
+1:
+ ldp q23, q31, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_H2 v25, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_H v26, v1, v2, v3, v4, v5, v6, v7, v31
+ QPEL_FILTER_H2 v27, v1, v2, v3, v4, v5, v6, v7, v31
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q16, q1, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_H2 v25, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_H v26, v2, v3, v4, v5, v6, v7, v31, v1
+ QPEL_FILTER_H2 v27, v2, v3, v4, v5, v6, v7, v31, v1
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q17, q2, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_H2 v25, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_H v26, v3, v4, v5, v6, v7, v31, v1, v2
+ QPEL_FILTER_H2 v27, v3, v4, v5, v6, v7, v31, v1, v2
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q18, q3, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_H2 v25, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_H v26, v4, v5, v6, v7, v31, v1, v2, v3
+ QPEL_FILTER_H2 v27, v4, v5, v6, v7, v31, v1, v2, v3
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q19, q4, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_H2 v25, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_H v26, v5, v6, v7, v31, v1, v2, v3, v4
+ QPEL_FILTER_H2 v27, v5, v6, v7, v31, v1, v2, v3, v4
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q20, q5, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_H2 v25, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_H v26, v6, v7, v31, v1, v2, v3, v4, v5
+ QPEL_FILTER_H2 v27, v6, v7, v31, v1, v2, v3, v4, v5
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q21, q6, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_H2 v25, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_H v26, v7, v31, v1, v2, v3, v4, v5, v6
+ QPEL_FILTER_H2 v27, v7, v31, v1, v2, v3, v4, v5, v6
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q22, q7, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_H2 v25, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_H v26, v31, v1, v2, v3, v4, v5, v6, v7
+ QPEL_FILTER_H2 v27, v31, v1, v2, v3, v4, v5, v6, v7
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.hi 1b
+2:
+ subs w27, w27, #16
+ add sp, x11, #32
+ add x20, x13, #16
+ mov w22, w12
+ mov x11, sp
+ mov x13, x20
+ b.hi 3b
+ QPEL_UNI_W_HV_END
+ ret
+endfunc
+
+function ff_hevc_put_hevc_qpel_uni_w_hv64_8_neon_dotprod, export=1
+ QPEL_UNI_W_HV_HEADER 64
+ mov x11, sp
+ mov w12, w22
+ mov x13, x20
+3:
+ ldp q16, q1, [sp]
+ add sp, sp, x10
+ ldp q17, q2, [sp]
+ add sp, sp, x10
+ ldp q18, q3, [sp]
+ add sp, sp, x10
+ ldp q19, q4, [sp]
+ add sp, sp, x10
+ ldp q20, q5, [sp]
+ add sp, sp, x10
+ ldp q21, q6, [sp]
+ add sp, sp, x10
+ ldp q22, q7, [sp]
+ add sp, sp, x10
+1:
+ ldp q23, q31, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_H2 v25, v16, v17, v18, v19, v20, v21, v22, v23
+ QPEL_FILTER_H v26, v1, v2, v3, v4, v5, v6, v7, v31
+ QPEL_FILTER_H2 v27, v1, v2, v3, v4, v5, v6, v7, v31
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q16, q1, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_H2 v25, v17, v18, v19, v20, v21, v22, v23, v16
+ QPEL_FILTER_H v26, v2, v3, v4, v5, v6, v7, v31, v1
+ QPEL_FILTER_H2 v27, v2, v3, v4, v5, v6, v7, v31, v1
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q17, q2, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_H2 v25, v18, v19, v20, v21, v22, v23, v16, v17
+ QPEL_FILTER_H v26, v3, v4, v5, v6, v7, v31, v1, v2
+ QPEL_FILTER_H2 v27, v3, v4, v5, v6, v7, v31, v1, v2
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q18, q3, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_H2 v25, v19, v20, v21, v22, v23, v16, v17, v18
+ QPEL_FILTER_H v26, v4, v5, v6, v7, v31, v1, v2, v3
+ QPEL_FILTER_H2 v27, v4, v5, v6, v7, v31, v1, v2, v3
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q19, q4, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_H2 v25, v20, v21, v22, v23, v16, v17, v18, v19
+ QPEL_FILTER_H v26, v5, v6, v7, v31, v1, v2, v3, v4
+ QPEL_FILTER_H2 v27, v5, v6, v7, v31, v1, v2, v3, v4
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q20, q5, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_H2 v25, v21, v22, v23, v16, v17, v18, v19, v20
+ QPEL_FILTER_H v26, v6, v7, v31, v1, v2, v3, v4, v5
+ QPEL_FILTER_H2 v27, v6, v7, v31, v1, v2, v3, v4, v5
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q21, q6, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_H2 v25, v22, v23, v16, v17, v18, v19, v20, v21
+ QPEL_FILTER_H v26, v7, v31, v1, v2, v3, v4, v5, v6
+ QPEL_FILTER_H2 v27, v7, v31, v1, v2, v3, v4, v5, v6
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.eq 2f
+
+ ldp q22, q7, [sp]
+ add sp, sp, x10
+ QPEL_FILTER_H v24, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_H2 v25, v23, v16, v17, v18, v19, v20, v21, v22
+ QPEL_FILTER_H v26, v31, v1, v2, v3, v4, v5, v6, v7
+ QPEL_FILTER_H2 v27, v31, v1, v2, v3, v4, v5, v6, v7
+ QPEL_UNI_W_HV_16
+ subs w22, w22, #1
+ b.hi 1b
+2:
+ subs w27, w27, #16
+ add sp, x11, #32
+ add x20, x13, #16
+ mov w22, w12
+ mov x11, sp
+ mov x13, x20
+ b.hi 3b
+ QPEL_UNI_W_HV_END
+ ret
+endfunc
+
+#endif // __ARM_FEATURE_DOTPROD
\ No newline at end of file
--
2.38.0.windows.1
More information about the ffmpeg-devel
mailing list