[FFmpeg-devel] [PATCH 2/2] lavc/ccaption_dec: implement real_time option
Aman Gupta
ffmpeg at tmm1.net
Sat Jan 9 23:55:01 CET 2016
From: Aman Gupta <aman at tmm1.net>
This new mode is useful for realtime decoding of closed captions so they
can be display along with mpeg2 frames.
Closed caption streams contain two major types of captions:
- POPON captions, which are buffered off-screen and displayed
only after EOC (end of caption, aka display buffer)
- PAINTON/ROLLUP captions, which are written to the display as soon as
they arrive.
In a typical real-time eia608 decoder, commands like EOC (end of
caption; display buffer), EDM (erase display memory) and EBM (erase
buffered memory) perform their expected functions as soon as the
commands are processed. This is implemented in the real_time branches
added in this commit.
Before this commit, and in the !real_time branches after this commit,
the decoder cleverly implements its own version of the decoder which is
specifically geared towards buffered decoding. It does so by actively
ignoring commands like EBM (erase buffered memory), and then re-using
the non-display buffer to hold the previous caption while the new one is
received. This is the opposite of the real-time decoder, which uses the
non-display buffer to hold the new caption while the display buffer is
still showing the current caption.
In addition to ignoring EBM, the buffered decoder also has custom
implementations for EDM and EOC. An EDM (erase display memory) command
flushes the existing contents before clearing the screen, and EOC
similarly always flushes the active buffer (the previous subtitle)
before flipping buffers.
---
libavcodec/ccaption_dec.c | 85 +++++++++++++++++++++++++++++++++++++-----
tests/fate/subtitles.mak | 3 ++
tests/ref/fate/sub-cc-realtime | 42 +++++++++++++++++++++
3 files changed, 121 insertions(+), 9 deletions(-)
create mode 100644 tests/ref/fate/sub-cc-realtime
diff --git a/libavcodec/ccaption_dec.c b/libavcodec/ccaption_dec.c
index 8bef771..8c26fcc 100644
--- a/libavcodec/ccaption_dec.c
+++ b/libavcodec/ccaption_dec.c
@@ -116,6 +116,7 @@ struct Screen {
typedef struct CCaptionSubContext {
AVClass *class;
+ int real_time;
struct Screen screen[2];
int active_screen;
uint8_t cursor_row;
@@ -130,6 +131,8 @@ typedef struct CCaptionSubContext {
/* visible screen time */
int64_t startv_time;
int64_t end_time;
+ int screen_touched;
+ int64_t last_real_time;
char prev_cmd[2];
/* buffer to store pkt data */
AVBufferRef *pktbuf;
@@ -180,7 +183,10 @@ static void flush_decoder(AVCodecContext *avctx)
ctx->cursor_color = 0;
ctx->active_screen = 0;
av_bprint_clear(&ctx->buffer);
- ctx->buffer_changed = 0;
+ ctx->last_real_time = 0;
+ ctx->screen_touched = 0;
+ /* emit empty subtitle on seek in realtime mode */
+ ctx->buffer_changed = ctx->real_time ? 1 : 0;
}
/**
@@ -418,15 +424,33 @@ static void handle_edm(CCaptionSubContext *ctx, int64_t pts)
{
struct Screen *screen = ctx->screen + ctx->active_screen;
- reap_screen(ctx, pts);
+ // In buffered mode, keep writing to screen until it is wiped.
+ // Before wiping the display, capture contents to emit subtitle.
+ if (!ctx->real_time)
+ reap_screen(ctx, pts);
+
screen->row_used = 0;
+
+ // In realtime mode, emit an empty caption so the last one doesn't
+ // stay on the screen.
+ if (ctx->real_time)
+ reap_screen(ctx, pts);
}
static void handle_eoc(CCaptionSubContext *ctx, int64_t pts)
{
- handle_edm(ctx,pts);
+ // In buffered mode, we wait til the *next* EOC and
+ // reap what was already on the screen since the last EOC.
+ if (!ctx->real_time)
+ handle_edm(ctx,pts);
+
ctx->active_screen = !ctx->active_screen;
ctx->cursor_column = 0;
+
+ // In realtime mode, we display the buffered contents (after
+ // flipping the buffer to active above) as soon as EOC arrives.
+ if (ctx->real_time)
+ reap_screen(ctx, pts);
}
static void handle_delete_end_of_row(CCaptionSubContext *ctx, char hi, char lo)
@@ -448,6 +472,9 @@ static void handle_char(CCaptionSubContext *ctx, char hi, char lo, int64_t pts)
}
write_char(ctx, screen, 0);
+ if (ctx->mode != CCMODE_POPON)
+ ctx->screen_touched = 1;
+
/* reset prev command since character can repeat */
ctx->prev_cmd[0] = 0;
ctx->prev_cmd[1] = 0;
@@ -497,10 +524,20 @@ static void process_cc608(CCaptionSubContext *ctx, int64_t pts, uint8_t hi, uint
case 0x2d:
/* carriage return */
ff_dlog(ctx, "carriage return\n");
- reap_screen(ctx, pts);
+ if (!ctx->real_time)
+ reap_screen(ctx, pts);
roll_up(ctx);
ctx->cursor_column = 0;
break;
+ case 0x2e:
+ /* erase buffered (non displayed) memory */
+ // Only in realtime mode. In buffered mode, we re-use the inactive screen
+ // for our own buffering.
+ if (ctx->real_time) {
+ struct Screen *screen = ctx->screen + !ctx->active_screen;
+ screen->row_used = 0;
+ }
+ break;
case 0x2f:
/* end of caption */
ff_dlog(ctx, "handle_eoc\n");
@@ -552,24 +589,54 @@ static int decode(AVCodecContext *avctx, void *data, int *got_sub, AVPacket *avp
continue;
else
process_cc608(ctx, avpkt->pts, *(bptr + i + 1) & 0x7f, *(bptr + i + 2) & 0x7f);
- if (ctx->buffer_changed && *ctx->buffer.str)
+
+ if (!ctx->buffer_changed)
+ continue;
+ ctx->buffer_changed = 0;
+
+ if (*ctx->buffer.str || ctx->real_time)
{
- int start_time = av_rescale_q(ctx->start_time, avctx->time_base, ass_tb);
- int end_time = av_rescale_q(ctx->end_time, avctx->time_base, ass_tb);
+ int64_t sub_pts = ctx->real_time ? avpkt->pts : ctx->start_time;
+ int start_time = av_rescale_q(sub_pts, avctx->time_base, ass_tb);
+ int duration = -1;
+ if (!ctx->real_time) {
+ int end_time = av_rescale_q(ctx->end_time, avctx->time_base, ass_tb);
+ duration = end_time - start_time;
+ }
ff_dlog(ctx, "cdp writing data (%s)\n",ctx->buffer.str);
- ret = ff_ass_add_rect_bprint(sub, &ctx->buffer, start_time, end_time - start_time);
+ ret = ff_ass_add_rect_bprint(sub, &ctx->buffer, start_time, duration);
if (ret < 0)
return ret;
- sub->pts = av_rescale_q(ctx->start_time, avctx->time_base, AV_TIME_BASE_Q);
+ sub->pts = av_rescale_q(sub_pts, avctx->time_base, AV_TIME_BASE_Q);
ctx->buffer_changed = 0;
+ ctx->last_real_time = avpkt->pts;
+ ctx->screen_touched = 0;
}
}
+ if (ctx->real_time && ctx->screen_touched &&
+ avpkt->pts > ctx->last_real_time + av_rescale_q(20, ass_tb, avctx->time_base)) {
+ ctx->last_real_time = avpkt->pts;
+ ctx->screen_touched = 0;
+
+ capture_screen(ctx);
+ ctx->buffer_changed = 0;
+
+ int start_time = av_rescale_q(avpkt->pts, avctx->time_base, ass_tb);
+ ret = ff_ass_add_rect_bprint(sub, &ctx->buffer, start_time, -1);
+ if (ret < 0)
+ return ret;
+ sub->pts = av_rescale_q(avpkt->pts, avctx->time_base, AV_TIME_BASE_Q);
+ }
+
*got_sub = sub->num_rects > 0;
return ret;
}
+#define OFFSET(x) offsetof(CCaptionSubContext, x)
+#define SD AV_OPT_FLAG_SUBTITLE_PARAM | AV_OPT_FLAG_DECODING_PARAM
static const AVOption options[] = {
+ { "real_time", "emit subtitle events as they are decoded for real-time display", OFFSET(real_time), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, SD },
{NULL}
};
diff --git a/tests/fate/subtitles.mak b/tests/fate/subtitles.mak
index d273f2e..8aa0279 100644
--- a/tests/fate/subtitles.mak
+++ b/tests/fate/subtitles.mak
@@ -4,6 +4,9 @@ fate-sub-aqtitle: CMD = fmtstdout ass -sub_charenc windows-1250 -i $(TARGET_SAMP
FATE_SUBTITLES_ASS-$(call ALLYES, AVDEVICE LAVFI_INDEV CCAPTION_DECODER MOVIE_FILTER MPEGTS_DEMUXER) += fate-sub-cc
fate-sub-cc: CMD = fmtstdout ass -f lavfi -i "movie=$(TARGET_SAMPLES)/sub/Closedcaption_rollup.m2v[out0+subcc]"
+FATE_SUBTITLES_ASS-$(call ALLYES, AVDEVICE LAVFI_INDEV CCAPTION_DECODER MOVIE_FILTER MPEGTS_DEMUXER) += fate-sub-cc-realtime
+fate-sub-cc-realtime: CMD = fmtstdout ass -real_time 1 -f lavfi -i "movie=$(TARGET_SAMPLES)/sub/Closedcaption_rollup.m2v[out0+subcc]"
+
FATE_SUBTITLES_ASS-$(call DEMDEC, ASS, ASS) += fate-sub-ass-to-ass-transcode
fate-sub-ass-to-ass-transcode: CMD = fmtstdout ass -i $(TARGET_SAMPLES)/sub/1ededcbd7b.ass
diff --git a/tests/ref/fate/sub-cc-realtime b/tests/ref/fate/sub-cc-realtime
new file mode 100644
index 0000000..4d2f110
--- /dev/null
+++ b/tests/ref/fate/sub-cc-realtime
@@ -0,0 +1,42 @@
+[Script Info]
+; Script generated by FFmpeg/Lavc
+ScriptType: v4.00+
+PlayResX: 384
+PlayResY: 288
+
+[V4+ Styles]
+Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+Style: Default,Arial,16,&Hffffff,&Hffffff,&H0,&H0,0,0,0,0,100,100,0,0,1,1,0,2,10,10,10,0
+
+[Events]
+Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+Dialogue: 0,0:00:14.14,9:59:59.99,Default,,0,0,0,,(
+Dialogue: 0,0:00:15.47,9:59:59.99,Default,,0,0,0,,({/i1} in
+Dialogue: 0,0:00:15.92,9:59:59.99,Default,,0,0,0,,({/i1} inau
+Dialogue: 0,0:00:16.36,9:59:59.99,Default,,0,0,0,,({/i1} inaudi
+Dialogue: 0,0:00:16.81,9:59:59.99,Default,,0,0,0,,({/i1} inaudibl
+Dialogue: 0,0:00:17.25,9:59:59.99,Default,,0,0,0,,({/i1} inaudible
+Dialogue: 0,0:00:17.70,9:59:59.99,Default,,0,0,0,,({/i1} inaudible ra
+Dialogue: 0,0:00:18.14,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radi
+Dialogue: 0,0:00:18.59,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
+Dialogue: 0,0:00:19.03,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio ch
+Dialogue: 0,0:00:19.48,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chat
+Dialogue: 0,0:00:19.92,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatte
+Dialogue: 0,0:00:20.36,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter
+Dialogue: 0,0:00:21.70,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )
+Dialogue: 0,0:00:42.61,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>>
+Dialogue: 0,0:00:43.05,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> S
+Dialogue: 0,0:00:43.50,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Saf
+Dialogue: 0,0:00:43.94,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safet
+Dialogue: 0,0:00:44.39,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety
+Dialogue: 0,0:00:44.83,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety re
+Dialogue: 0,0:00:45.28,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety rema
+Dialogue: 0,0:00:45.72,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remain
+Dialogue: 0,0:00:46.17,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains
+Dialogue: 0,0:00:46.61,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains ou
+Dialogue: 0,0:00:47.06,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains our
+Dialogue: 0,0:00:47.50,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains our nu
+Dialogue: 0,0:00:47.95,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains our numb
+Dialogue: 0,0:00:48.39,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains our number
+Dialogue: 0,0:00:48.84,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains our number o
+Dialogue: 0,0:00:49.28,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0} )\N>> Safety remains our number one
--
2.5.3
More information about the ffmpeg-devel
mailing list