[FFmpeg-cvslog] Fix visual glitch with XvMC, caused by wrong idct permutation.

[Ivan Kalvachev]

ffmpeg | branch: release/3.4 | Ivan Kalvachev <ikalvachev at gmail.com> | Mon Oct  9 01:25:00 2017 +0300| [a11a18b284afd5ac58fd3b1835f8a3608c4ebc9f] | committer: Michael Niedermayer

Fix visual glitch with XvMC, caused by wrong idct permutation.

In the past XvMC forced simple_idct since
it was using FF_IDCT_PERM_NONE.
However now we have SIMD variants of simple_idct that
are using FF_IDCT_PERM_TRANSPOSE and if they are selected
XvMC would get coefficients in the wrong order.

The patch creates new FF_IDCT_NONE that
is used only for this kind of hardware decoding
and that fallbacks to the old C only simple idct.

Signed-off-by: Ivan Kalvachev <ikalvachev at gmail.com>
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 9054439bad3307dafd9fbadc57e66c276baf22e2)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=a11a18b284afd5ac58fd3b1835f8a3608c4ebc9f
---

 libavcodec/avcodec.h   | 1 +
 libavcodec/idctdsp.c   | 1 +
 libavcodec/mpeg12dec.c | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index 52cc5b0ca0..18c3e3ea1e 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -3146,6 +3146,7 @@ typedef struct AVCodecContext {
 #if FF_API_ARCH_ALPHA
 #define FF_IDCT_SIMPLEALPHA   23
 #endif
+#define FF_IDCT_NONE          24 /* Used by XvMC to extract IDCT coefficients with FF_IDCT_PERM_NONE */
 #define FF_IDCT_SIMPLEAUTO    128
 
     /**
diff --git a/libavcodec/idctdsp.c b/libavcodec/idctdsp.c
index d596aed1a9..0122d29efa 100644
--- a/libavcodec/idctdsp.c
+++ b/libavcodec/idctdsp.c
@@ -279,6 +279,7 @@ av_cold void ff_idctdsp_init(IDCTDSPContext *c, AVCodecContext *avctx)
                 c->perm_type = FF_IDCT_PERM_NONE;
 #endif /* CONFIG_FAANIDCT */
             } else { // accurate/default
+                /* Be sure FF_IDCT_NONE will select this one, since it uses FF_IDCT_PERM_NONE */
                 c->idct_put  = ff_simple_idct_put_8;
                 c->idct_add  = ff_simple_idct_add_8;
                 c->idct      = ff_simple_idct_8;
diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c
index 22c29c1505..4e68be27f1 100644
--- a/libavcodec/mpeg12dec.c
+++ b/libavcodec/mpeg12dec.c
@@ -1217,7 +1217,7 @@ static void setup_hwaccel_for_pixfmt(AVCodecContext *avctx)
 #endif
         )
         if (avctx->idct_algo == FF_IDCT_AUTO)
-            avctx->idct_algo = FF_IDCT_SIMPLE;
+            avctx->idct_algo = FF_IDCT_NONE;
 
     if (avctx->hwaccel && avctx->pix_fmt == AV_PIX_FMT_XVMC) {
         Mpeg1Context *s1 = avctx->priv_data;