[FFmpeg-devel] [PATCH 2/2] lavc/ccaption_dec: implement real_time option
anshul.ffmpeg at gmail.com
anshul.ffmpeg at gmail.com
Mon Jan 11 13:56:58 CET 2016
On Saturday, January 09, 2016 02:55:01 PM Aman Gupta wrote:
> From: Aman Gupta <aman at tmm1.net>
>
> This new mode is useful for realtime decoding of closed captions so they
> can be display along with mpeg2 frames.
>
> Closed caption streams contain two major types of captions:
>
> - POPON captions, which are buffered off-screen and displayed
> only after EOC (end of caption, aka display buffer)
>
> - PAINTON/ROLLUP captions, which are written to the display as soon as
> they arrive.
>
> In a typical real-time eia608 decoder, commands like EOC (end of
> caption; display buffer), EDM (erase display memory) and EBM (erase
> buffered memory) perform their expected functions as soon as the
> commands are processed. This is implemented in the real_time branches
> added in this commit.
>
> Before this commit, and in the !real_time branches after this commit,
> the decoder cleverly implements its own version of the decoder which is
> specifically geared towards buffered decoding. It does so by actively
> ignoring commands like EBM (erase buffered memory), and then re-using
> the non-display buffer to hold the previous caption while the new one is
> received. This is the opposite of the real-time decoder, which uses the
> non-display buffer to hold the new caption while the display buffer is
> still showing the current caption.
>
> In addition to ignoring EBM, the buffered decoder also has custom
> implementations for EDM and EOC. An EDM (erase display memory) command
> flushes the existing contents before clearing the screen, and EOC
> similarly always flushes the active buffer (the previous subtitle)
> before flipping buffers.
> ---
> libavcodec/ccaption_dec.c | 85
> +++++++++++++++++++++++++++++++++++++----- tests/fate/subtitles.mak |
> 3 ++
> tests/ref/fate/sub-cc-realtime | 42 +++++++++++++++++++++
> 3 files changed, 121 insertions(+), 9 deletions(-)
> create mode 100644 tests/ref/fate/sub-cc-realtime
>
> diff --git a/libavcodec/ccaption_dec.c b/libavcodec/ccaption_dec.c
> index 8bef771..8c26fcc 100644
> --- a/libavcodec/ccaption_dec.c
> +++ b/libavcodec/ccaption_dec.c
> @@ -116,6 +116,7 @@ struct Screen {
>
> typedef struct CCaptionSubContext {
> AVClass *class;
> + int real_time;
> struct Screen screen[2];
> int active_screen;
> uint8_t cursor_row;
> @@ -130,6 +131,8 @@ typedef struct CCaptionSubContext {
> /* visible screen time */
> int64_t startv_time;
> int64_t end_time;
> + int screen_touched;
> + int64_t last_real_time;
> char prev_cmd[2];
> /* buffer to store pkt data */
> AVBufferRef *pktbuf;
> @@ -180,7 +183,10 @@ static void flush_decoder(AVCodecContext *avctx)
> ctx->cursor_color = 0;
> ctx->active_screen = 0;
> av_bprint_clear(&ctx->buffer);
> - ctx->buffer_changed = 0;
> + ctx->last_real_time = 0;
> + ctx->screen_touched = 0;
> + /* emit empty subtitle on seek in realtime mode */
> + ctx->buffer_changed = ctx->real_time ? 1 : 0;
> }
>
> /**
> @@ -418,15 +424,33 @@ static void handle_edm(CCaptionSubContext *ctx,
> int64_t pts) {
> struct Screen *screen = ctx->screen + ctx->active_screen;
>
> - reap_screen(ctx, pts);
> + // In buffered mode, keep writing to screen until it is wiped.
> + // Before wiping the display, capture contents to emit subtitle.
> + if (!ctx->real_time)
> + reap_screen(ctx, pts);
> +
> screen->row_used = 0;
> +
> + // In realtime mode, emit an empty caption so the last one doesn't
> + // stay on the screen.
> + if (ctx->real_time)
> + reap_screen(ctx, pts);
> }
>
> static void handle_eoc(CCaptionSubContext *ctx, int64_t pts)
> {
> - handle_edm(ctx,pts);
> + // In buffered mode, we wait til the *next* EOC and
> + // reap what was already on the screen since the last EOC.
> + if (!ctx->real_time)
> + handle_edm(ctx,pts);
> +
> ctx->active_screen = !ctx->active_screen;
> ctx->cursor_column = 0;
> +
> + // In realtime mode, we display the buffered contents (after
> + // flipping the buffer to active above) as soon as EOC arrives.
> + if (ctx->real_time)
> + reap_screen(ctx, pts);
> }
>
> static void handle_delete_end_of_row(CCaptionSubContext *ctx, char hi, char
> lo) @@ -448,6 +472,9 @@ static void handle_char(CCaptionSubContext *ctx,
> char hi, char lo, int64_t pts) }
> write_char(ctx, screen, 0);
>
> + if (ctx->mode != CCMODE_POPON)
> + ctx->screen_touched = 1;
> +
> /* reset prev command since character can repeat */
> ctx->prev_cmd[0] = 0;
> ctx->prev_cmd[1] = 0;
> @@ -497,10 +524,20 @@ static void process_cc608(CCaptionSubContext *ctx,
> int64_t pts, uint8_t hi, uint case 0x2d:
> /* carriage return */
> ff_dlog(ctx, "carriage return\n");
> - reap_screen(ctx, pts);
> + if (!ctx->real_time)
> + reap_screen(ctx, pts);
> roll_up(ctx);
> ctx->cursor_column = 0;
> break;
> + case 0x2e:
> + /* erase buffered (non displayed) memory */
> + // Only in realtime mode. In buffered mode, we re-use the
> inactive screen + // for our own buffering.
> + if (ctx->real_time) {
> + struct Screen *screen = ctx->screen + !ctx->active_screen;
> + screen->row_used = 0;
> + }
> + break;
> case 0x2f:
> /* end of caption */
> ff_dlog(ctx, "handle_eoc\n");
> @@ -552,24 +589,54 @@ static int decode(AVCodecContext *avctx, void *data,
> int *got_sub, AVPacket *avp continue;
> else
> process_cc608(ctx, avpkt->pts, *(bptr + i + 1) & 0x7f, *(bptr +
> i + 2) & 0x7f); - if (ctx->buffer_changed && *ctx->buffer.str)
> +
> + if (!ctx->buffer_changed)
> + continue;
> + ctx->buffer_changed = 0;
> +
> + if (*ctx->buffer.str || ctx->real_time)
> {
> - int start_time = av_rescale_q(ctx->start_time,
> avctx->time_base, ass_tb); - int end_time =
> av_rescale_q(ctx->end_time, avctx->time_base, ass_tb); + int64_t
> sub_pts = ctx->real_time ? avpkt->pts : ctx->start_time; + int
> start_time = av_rescale_q(sub_pts, avctx->time_base, ass_tb); +
> int duration = -1;
> + if (!ctx->real_time) {
> + int end_time = av_rescale_q(ctx->end_time,
> avctx->time_base, ass_tb); + duration = end_time -
> start_time;
> + }
> ff_dlog(ctx, "cdp writing data (%s)\n",ctx->buffer.str);
> - ret = ff_ass_add_rect_bprint(sub, &ctx->buffer, start_time,
> end_time - start_time); + ret = ff_ass_add_rect_bprint(sub,
> &ctx->buffer, start_time, duration); if (ret < 0)
> return ret;
> - sub->pts = av_rescale_q(ctx->start_time, avctx->time_base,
> AV_TIME_BASE_Q); + sub->pts = av_rescale_q(sub_pts,
> avctx->time_base, AV_TIME_BASE_Q); ctx->buffer_changed = 0;
> + ctx->last_real_time = avpkt->pts;
> + ctx->screen_touched = 0;
> }
> }
>
> + if (ctx->real_time && ctx->screen_touched &&
> + avpkt->pts > ctx->last_real_time + av_rescale_q(20, ass_tb,
> avctx->time_base)) { + ctx->last_real_time = avpkt->pts;
> + ctx->screen_touched = 0;
> +
> + capture_screen(ctx);
> + ctx->buffer_changed = 0;
> +
> + int start_time = av_rescale_q(avpkt->pts, avctx->time_base,
> ass_tb); + ret = ff_ass_add_rect_bprint(sub, &ctx->buffer,
> start_time, -1); + if (ret < 0)
> + return ret;
> + sub->pts = av_rescale_q(avpkt->pts, avctx->time_base,
> AV_TIME_BASE_Q); + }
> +
> *got_sub = sub->num_rects > 0;
> return ret;
> }
>
> +#define OFFSET(x) offsetof(CCaptionSubContext, x)
> +#define SD AV_OPT_FLAG_SUBTITLE_PARAM | AV_OPT_FLAG_DECODING_PARAM
> static const AVOption options[] = {
> + { "real_time", "emit subtitle events as they are decoded for real-time
> display", OFFSET(real_time), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, SD },
> {NULL}
> };
>
> diff --git a/tests/fate/subtitles.mak b/tests/fate/subtitles.mak
> index d273f2e..8aa0279 100644
> --- a/tests/fate/subtitles.mak
> +++ b/tests/fate/subtitles.mak
> @@ -4,6 +4,9 @@ fate-sub-aqtitle: CMD = fmtstdout ass -sub_charenc
> windows-1250 -i $(TARGET_SAMP FATE_SUBTITLES_ASS-$(call ALLYES, AVDEVICE
> LAVFI_INDEV CCAPTION_DECODER MOVIE_FILTER MPEGTS_DEMUXER) += fate-sub-cc
> fate-sub-cc: CMD = fmtstdout ass -f lavfi -i
> "movie=$(TARGET_SAMPLES)/sub/Closedcaption_rollup.m2v[out0+subcc]"
>
> +FATE_SUBTITLES_ASS-$(call ALLYES, AVDEVICE LAVFI_INDEV CCAPTION_DECODER
> MOVIE_FILTER MPEGTS_DEMUXER) += fate-sub-cc-realtime +fate-sub-cc-realtime:
> CMD = fmtstdout ass -real_time 1 -f lavfi -i
> "movie=$(TARGET_SAMPLES)/sub/Closedcaption_rollup.m2v[out0+subcc]" +
> FATE_SUBTITLES_ASS-$(call DEMDEC, ASS, ASS) +=
> fate-sub-ass-to-ass-transcode fate-sub-ass-to-ass-transcode: CMD =
> fmtstdout ass -i $(TARGET_SAMPLES)/sub/1ededcbd7b.ass
>
> diff --git a/tests/ref/fate/sub-cc-realtime b/tests/ref/fate/sub-cc-realtime
> new file mode 100644
> index 0000000..4d2f110
> --- /dev/null
> +++ b/tests/ref/fate/sub-cc-realtime
> @@ -0,0 +1,42 @@
> +[Script Info]
> +; Script generated by FFmpeg/Lavc
> +ScriptType: v4.00+
> +PlayResX: 384
> +PlayResY: 288
> +
> +[V4+ Styles]
> +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour,
> OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX,
> ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL,
> MarginR, MarginV, Encoding +Style:
> Default,Arial,16,&Hffffff,&Hffffff,&H0,&H0,0,0,0,0,100,100,0,0,1,1,0,2,10,1
> 0,10,0 +
> +[Events]
> +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect,
> Text +Dialogue: 0,0:00:14.14,9:59:59.99,Default,,0,0,0,,(
> +Dialogue: 0,0:00:15.47,9:59:59.99,Default,,0,0,0,,({/i1} in
> +Dialogue: 0,0:00:15.92,9:59:59.99,Default,,0,0,0,,({/i1} inau
> +Dialogue: 0,0:00:16.36,9:59:59.99,Default,,0,0,0,,({/i1} inaudi
> +Dialogue: 0,0:00:16.81,9:59:59.99,Default,,0,0,0,,({/i1} inaudibl
> +Dialogue: 0,0:00:17.25,9:59:59.99,Default,,0,0,0,,({/i1} inaudible
> +Dialogue: 0,0:00:17.70,9:59:59.99,Default,,0,0,0,,({/i1} inaudible ra
> +Dialogue: 0,0:00:18.14,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radi
> +Dialogue: 0,0:00:18.59,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
> +Dialogue: 0,0:00:19.03,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio ch
> +Dialogue: 0,0:00:19.48,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
> chat +Dialogue: 0,0:00:19.92,9:59:59.99,Default,,0,0,0,,({/i1} inaudible
> radio chatte +Dialogue: 0,0:00:20.36,9:59:59.99,Default,,0,0,0,,({/i1}
> inaudible radio chatter +Dialogue:
> 0,0:00:21.70,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> ) +Dialogue: 0,0:00:42.61,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
> chatter{/i0} )\N>> +Dialogue:
> 0,0:00:43.05,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> S +Dialogue: 0,0:00:43.50,9:59:59.99,Default,,0,0,0,,({/i1} inaudible
> radio chatter{/i0} )\N>> Saf +Dialogue:
> 0,0:00:43.94,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safet +Dialogue: 0,0:00:44.39,9:59:59.99,Default,,0,0,0,,({/i1}
> inaudible radio chatter{/i0} )\N>> Safety +Dialogue:
> 0,0:00:44.83,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety re +Dialogue: 0,0:00:45.28,9:59:59.99,Default,,0,0,0,,({/i1}
> inaudible radio chatter{/i0} )\N>> Safety rema +Dialogue:
> 0,0:00:45.72,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remain +Dialogue:
> 0,0:00:46.17,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains +Dialogue:
> 0,0:00:46.61,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains ou +Dialogue:
> 0,0:00:47.06,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains our +Dialogue:
> 0,0:00:47.50,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains our nu +Dialogue:
> 0,0:00:47.95,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains our numb +Dialogue:
> 0,0:00:48.39,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains our number +Dialogue:
> 0,0:00:48.84,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains our number o +Dialogue:
> 0,0:00:49.28,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio chatter{/i0}
> )\N>> Safety remains our number one
Unable to apply, I get following errors.
anshul at linux-kiran:~/Project/FFmpeg> git am ~/Documents/\[FFmpeg-devel\]\
\[PATCH\ 2_2\]\ lavc_ccaption_dec_implement\ real_time\ option.mbox
Applying: lavc/ccaption_dec: implement real_time option
/home/anshul/Project/FFmpeg/.git/rebase-apply/patch:206: trailing whitespace.
Dialogue: 0,0:00:17.25,9:59:59.99,Default,,0,0,0,,({/i1} inaudible
/home/anshul/Project/FFmpeg/.git/rebase-apply/patch:209: trailing whitespace.
Dialogue: 0,0:00:18.59,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
/home/anshul/Project/FFmpeg/.git/rebase-apply/patch:219: trailing whitespace.
Dialogue: 0,0:00:44.39,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
chatter{/i0} )\N>> Safety
/home/anshul/Project/FFmpeg/.git/rebase-apply/patch:223: trailing whitespace.
Dialogue: 0,0:00:46.17,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
chatter{/i0} )\N>> Safety remains
/home/anshul/Project/FFmpeg/.git/rebase-apply/patch:225: trailing whitespace.
Dialogue: 0,0:00:47.06,9:59:59.99,Default,,0,0,0,,({/i1} inaudible radio
chatter{/i0} )\N>> Safety remains our
warning: 5 lines add whitespace errors.
These errors are all from test file, can you tell me your github fork url
-Anshul
More information about the ffmpeg-devel
mailing list