[FFmpeg-devel] [RFC]subtitle timestamps (Was Basic XSUB encoder (take 5))

[Björn Axelsson]

On Fri, 6 Feb 2009, Michael Niedermayer wrote:
> On Thu, Feb 05, 2009 at 10:45:25PM +0100, Bj?rn Axelsson wrote:
[...]
> > I still need some help solving the timestamp problem as I don't think the
> > hack used in the patch is acceptable.
>
> i thought about this a little, but i think your hack is not that far from
> a possible solution
> look at AVFrame.pts & AVCodecContext.time_base
> AVSubtitle.pts should be like AVFrame.pts
>
> also start_display_time must be 0 on encoding because not all subtitle
> encoders support it. Actually i think start_display_time should be handled
> somehow different but iam not sure also this is getting off topic i guess

New topic started, so now it is on topic =)

If I understand this correctly some subtitle codecs (e.g. XSUB, SSA)
require absolute timestamps, while other codecs (e.g. DVB, DVD) work
only on timestamps relative to packet timestamps.

I made an initial attempt to implement that in the attached
patch subtitle_pts.diff. I'm not very happy with it as the timestamp
semantics of AVSubtitle got quite complex.

xsubdec_pts.diff implements the new semantics for the XSUB decoder.

(And the ffmpeg.c and ffplay.c patches should probably be broken out and
cleaned up somewhat. But they are mostly included as examples and for
testing.)

-- 
Bj?rn Axelsson
-------------- next part --------------
Index: ffmpeg.c
===================================================================
--- ffmpeg.c.orig	2009-02-07 12:41:56.000000000 +0100
+++ ffmpeg.c	2009-02-07 13:49:08.000000000 +0100
@@ -814,6 +814,18 @@
         nb = 1;
 
     for(i = 0; i < nb; i++) {
+        if(sub->pts == AV_NOPTS_VALUE) {
+            /* start_display_time must be 0 when encoding */
+            if(sub->start_display_time) {
+                pts += av_rescale_q(sub->start_display_time, (AVRational){1, 1000},
+                        ist->st->time_base);
+                sub->end_display_time -= sub->start_display_time;
+                sub->start_display_time = 0;
+            }
+            /* Some encoders require absolute time stamps */
+            sub->pts = av_rescale_q(pts, ist->st->time_base, enc->time_base);
+        }
+
         subtitle_out_size = avcodec_encode_subtitle(enc, subtitle_out,
                                                     subtitle_out_max_size, sub);
 
Index: libavcodec/avcodec.h
===================================================================
--- libavcodec/avcodec.h.orig	2009-02-07 12:41:56.000000000 +0100
+++ libavcodec/avcodec.h	2009-02-07 14:29:03.000000000 +0100
@@ -30,7 +30,7 @@
 #include "libavutil/avutil.h"
 
 #define LIBAVCODEC_VERSION_MAJOR 52
-#define LIBAVCODEC_VERSION_MINOR 13
+#define LIBAVCODEC_VERSION_MINOR 14
 #define LIBAVCODEC_VERSION_MICRO  0
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
@@ -2428,10 +2428,29 @@
 
 typedef struct AVSubtitle {
     uint16_t format; /* 0 = graphics */
-    uint32_t start_display_time; /* relative to packet pts, in ms */
-    uint32_t end_display_time; /* relative to packet pts, in ms */
+    /**
+     * relative to pts, in ms.
+     * - encoding: Must be 0.
+     * - decoding: Set by decoder. Relative to packet pts if
+     *   subtitle pts == AV_NOPTS_VALUE. Otherwise 0.
+     */
+    uint32_t start_display_time;
+    /**
+     * relative to pts, in ms.
+     * - encoding: Set by user.
+     * - decoding: Set by decoder. Relative to packet pts if
+     *   subtitle pts == AV_NOPTS_VALUE. Otherwise relative to subtitle pts.
+     */
+    uint32_t end_display_time;
     unsigned num_rects;
     AVSubtitleRect **rects;
+    /**
+     * Absolute presentation time, in codec time_base.
+     * - encoding: Set by user, based on packet pts and start display time.
+     * - decoding: either AV_NOPTS_VALUE if codec doesn't support absolute
+     *   presentation times, or the presentation time for this subtitle.
+     */
+    int64_t  pts;
 } AVSubtitle;
 
 
Index: ffplay.c
===================================================================
--- ffplay.c.orig	2009-02-07 12:42:06.000000000 +0100
+++ ffplay.c	2009-02-07 14:28:36.000000000 +0100
@@ -1434,6 +1434,9 @@
 //            if (len1 < 0)
 //                break;
         if (got_subtitle && sp->sub.format == 0) {
+            if(sp->sub.pts != AV_NOPTS_VALUE)
+                pts = av_q2d(is->subtitle_st->codec->time_base)*sp->sub.pts;
+
             sp->pts = pts;
 
             for (i = 0; i < sp->sub.num_rects; i++)
Index: libavcodec/utils.c
===================================================================
--- libavcodec/utils.c.orig	2009-02-07 12:53:09.000000000 +0100
+++ libavcodec/utils.c	2009-02-07 12:53:37.000000000 +0100
@@ -563,6 +563,7 @@
     int ret;
 
     *got_sub_ptr = 0;
+    sub->pts = AV_NOPTS_VALUE;
     ret = avctx->codec->decode(avctx, sub, got_sub_ptr,
                                buf, buf_size);
     if (*got_sub_ptr)
-------------- next part --------------
Index: libavcodec/xsubdec.c
===================================================================
--- libavcodec/xsubdec.c.orig	2009-02-07 12:42:43.000000000 +0100
+++ libavcodec/xsubdec.c	2009-02-07 14:29:37.000000000 +0100
@@ -24,6 +24,7 @@
 
 static av_cold int decode_init(AVCodecContext *avctx) {
     avctx->pix_fmt = PIX_FMT_PAL8;
+    avctx->time_base = AV_TIME_BASE_Q;
     return 0;
 }
 
@@ -49,6 +50,7 @@
     const uint8_t *buf_end = buf + buf_size;
     uint8_t *bitmap;
     int w, h, x, y, rlelen, i;
+    uint64_t start_time;
     GetBitContext gb;
 
     // check that at least header fits
@@ -62,8 +64,11 @@
         av_log(avctx, AV_LOG_ERROR, "invalid time code\n");
         return -1;
     }
-    sub->start_display_time = parse_timecode(buf +  1);
-    sub->end_display_time   = parse_timecode(buf + 14);
+    start_time = parse_timecode(buf + 1);
+    sub->pts = av_rescale_q(start_time, (AVRational){1, 1000},
+                            avctx->time_base);
+    sub->start_display_time = 0;
+    sub->end_display_time   = parse_timecode(buf + 14) - start_time;
     buf += 27;
 
     // read header