[FFmpeg-devel] [PATCH 2/2] ffmpeg: add video heartbeat capability to fix_sub_duration
Jan Ekström
jeebjp at gmail.com
Tue Dec 21 09:56:48 EET 2021
From: Jan Ekström <jan.ekstrom at 24i.com>
Splits the currently handled subtitle at random access point
packets that can be configured to follow a specific output stream.
This way the subtitle - which is known to be shown at this time
can be split and passed to muxer before its full duration is
yet known.
Signed-off-by: Jan Ekström <jan.ekstrom at 24i.com>
---
doc/ffmpeg.texi | 11 ++++
fftools/ffmpeg.c | 142 +++++++++++++++++++++++++++++++++++++++++++
fftools/ffmpeg.h | 8 +++
fftools/ffmpeg_opt.c | 9 +++
4 files changed, 170 insertions(+)
diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi
index 164419cad3..7ceb5532ce 100644
--- a/doc/ffmpeg.texi
+++ b/doc/ffmpeg.texi
@@ -1297,6 +1297,17 @@ List all hardware acceleration components enabled in this build of ffmpeg.
Actual runtime availability depends on the hardware and its suitable driver
being installed.
+ at item -fix_sub_duration_heartbeat[:@var{stream_specifier}]
+Set a specific output video stream as the heartbeat stream according to which
+to split and push through currently in-progress subtitle upon receipt of a
+random access packet.
+
+This lowers the latency of subtitles for which the end packet or the following
+subtitle has not yet been received.
+
+Requires @option{-fix_sub_duration} to be set for the relevant input subtitle
+stream for this to have any effect.
+
@end table
@section Audio Options
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index 1697891a99..f8cc11d1f4 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -126,6 +126,7 @@ typedef struct BenchmarkTimeStamps {
int64_t sys_usec;
} BenchmarkTimeStamps;
+static int trigger_fix_sub_duration_heartbeat(OutputStream *ost, const AVPacket *pkt);
static void do_video_stats(OutputStream *ost, int frame_size);
static BenchmarkTimeStamps get_benchmark_time_stamps(void);
static int64_t getmaxrss(void);
@@ -1382,6 +1383,13 @@ static void do_video_out(OutputFile *of,
av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, &ost->mux_timebase));
}
+ if ((ret = trigger_fix_sub_duration_heartbeat(ost, pkt)) < 0) {
+ av_log(NULL, AV_LOG_ERROR,
+ "Subtitle heartbeat logic failed in %s! (%s)\n",
+ __func__, av_err2str(ret));
+ exit_program(1);
+ }
+
frame_size = pkt->size;
output_packet(of, pkt, ost, 0);
@@ -2091,6 +2099,16 @@ static void do_streamcopy(InputStream *ist, OutputStream *ost, const AVPacket *p
ost->sync_opts += opkt->duration;
+ if (ost->st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+ int ret;
+ if ((ret = trigger_fix_sub_duration_heartbeat(ost, pkt)) < 0) {
+ av_log(NULL, AV_LOG_ERROR,
+ "Subtitle heartbeat logic failed in %s! (%s)\n",
+ __func__, av_err2str(ret));
+ exit_program(1);
+ }
+ }
+
output_packet(of, opkt, ost, 0);
}
@@ -2548,6 +2566,130 @@ out:
return ret;
}
+static int copy_av_subtitle(AVSubtitle *dst, AVSubtitle *src)
+{
+ int ret = AVERROR_BUG;
+ AVSubtitle tmp = {
+ .format = src->format,
+ .start_display_time = src->start_display_time,
+ .end_display_time = src->end_display_time,
+ .num_rects = 0,
+ .rects = NULL,
+ .pts = src->pts
+ };
+
+ if (!src->num_rects)
+ goto success;
+
+ if (!(tmp.rects = av_calloc(src->num_rects, sizeof(*tmp.rects))))
+ return AVERROR(ENOMEM);
+
+ for (int i = 0; i < src->num_rects; i++) {
+ AVSubtitleRect *src_rect = src->rects[i];
+ AVSubtitleRect *dst_rect;
+
+ if (!(dst_rect = tmp.rects[i] = av_mallocz(sizeof(*tmp.rects[0])))) {
+ ret = AVERROR(ENOMEM);
+ goto cleanup;
+ }
+
+ tmp.num_rects++;
+
+ dst_rect->type = src_rect->type;
+ dst_rect->flags = src_rect->flags;
+
+ dst_rect->x = src_rect->x;
+ dst_rect->y = src_rect->y;
+ dst_rect->w = src_rect->w;
+ dst_rect->h = src_rect->h;
+ dst_rect->nb_colors = src_rect->nb_colors;
+
+ if (src_rect->text)
+ if (!(dst_rect->text = av_strdup(src_rect->text))) {
+ ret = AVERROR(ENOMEM);
+ goto cleanup;
+ }
+
+ if (src_rect->ass)
+ if (!(dst_rect->ass = av_strdup(src_rect->ass))) {
+ ret = AVERROR(ENOMEM);
+ goto cleanup;
+ }
+
+ for (int j = 0; j < 4; j++) {
+ // SUBTITLE_BITMAP images are special in the sense that they
+ // are like PAL8 images. first pointer to data, second to
+ // palette. This makes the size calculation match this.
+ size_t buf_size = src_rect->type == SUBTITLE_BITMAP && j == 1 ?
+ AVPALETTE_SIZE :
+ src_rect->h * src_rect->linesize[j];
+
+ if (!src_rect->data[j])
+ continue;
+
+ if (!(dst_rect->data[j] = av_memdup(src_rect->data[j], buf_size))) {
+ ret = AVERROR(ENOMEM);
+ goto cleanup;
+ }
+ dst_rect->linesize[j] = src_rect->linesize[j];
+ }
+ }
+
+success:
+ *dst = tmp;
+
+ return 0;
+
+cleanup:
+ avsubtitle_free(&tmp);
+
+ return ret;
+}
+
+static int fix_sub_duration_heartbeat(InputStream *ist, int64_t signal_pts)
+{
+ int ret = AVERROR_BUG;
+ int got_output = 1;
+ AVSubtitle *prev_subtitle = &ist->prev_sub.subtitle;
+ AVSubtitle subtitle;
+
+ if (!ist->fix_sub_duration || !prev_subtitle->num_rects ||
+ signal_pts <= prev_subtitle->pts)
+ return 0;
+
+ if ((ret = copy_av_subtitle(&subtitle, prev_subtitle)) < 0)
+ return ret;
+
+ subtitle.pts = signal_pts;
+
+ return encode_mux_subtitles(ist, &subtitle, &got_output);
+}
+
+static int trigger_fix_sub_duration_heartbeat(OutputStream *ost, const AVPacket *pkt)
+{
+ int64_t signal_pts = av_rescale_q(pkt->pts, ost->mux_timebase,
+ AV_TIME_BASE_Q);
+
+ if (!ost->fix_sub_duration_heartbeat || !(pkt->flags & AV_PKT_FLAG_KEY))
+ // we are only interested in heartbeats on streams configured, and
+ // only on random access points.
+ return 0;
+
+ for (int index = 0; index < nb_input_streams; index++) {
+ InputStream *subtitle_ist = input_streams[index];
+ int ret = AVERROR_BUG;
+
+ if (!subtitle_ist->decoding_needed ||
+ subtitle_ist->dec_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE)
+ continue;
+
+ if ((ret = fix_sub_duration_heartbeat(subtitle_ist, signal_pts)) < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
int *decode_failed)
{
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 9b200b806a..b1e67a94e9 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -204,6 +204,8 @@ typedef struct OptionsContext {
int nb_reinit_filters;
SpecifierOpt *fix_sub_duration;
int nb_fix_sub_duration;
+ SpecifierOpt *fix_sub_duration_heartbeat;
+ int nb_fix_sub_duration_heartbeat;
SpecifierOpt *canvas_sizes;
int nb_canvas_sizes;
SpecifierOpt *pass;
@@ -571,6 +573,12 @@ typedef struct OutputStream {
/* frame encode sum of squared error values */
int64_t error[4];
+
+ /*
+ * bool on whether this stream should be utilized for splitting
+ * subtitles utilizing fix_sub_duration at random access points.
+ */
+ unsigned int fix_sub_duration_heartbeat;
} OutputStream;
typedef struct OutputFile {
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index 9c820ab73f..64a08749ac 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -86,6 +86,7 @@ static const char *const opt_name_filters[] = {"filter", "af",
static const char *const opt_name_filter_scripts[] = {"filter_script", NULL};
static const char *const opt_name_reinit_filters[] = {"reinit_filter", NULL};
static const char *const opt_name_fix_sub_duration[] = {"fix_sub_duration", NULL};
+static const char *const opt_name_fix_sub_duration_heartbeat[] = {"fix_sub_duration_heartbeat", NULL};
static const char *const opt_name_canvas_sizes[] = {"canvas_size", NULL};
static const char *const opt_name_pass[] = {"pass", NULL};
static const char *const opt_name_passlogfiles[] = {"passlogfile", NULL};
@@ -1600,6 +1601,9 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e
MATCH_PER_STREAM_OPT(bits_per_raw_sample, i, ost->bits_per_raw_sample,
oc, st);
+ MATCH_PER_STREAM_OPT(fix_sub_duration_heartbeat, i, ost->fix_sub_duration_heartbeat,
+ oc, st);
+
if (oc->oformat->flags & AVFMT_GLOBALHEADER)
ost->enc_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
@@ -3793,6 +3797,11 @@ const OptionDef options[] = {
{ "autoscale", HAS_ARG | OPT_BOOL | OPT_SPEC |
OPT_EXPERT | OPT_OUTPUT, { .off = OFFSET(autoscale) },
"automatically insert a scale filter at the end of the filter graph" },
+ { "fix_sub_duration_heartbeat", OPT_VIDEO | OPT_BOOL | OPT_EXPERT |
+ OPT_SPEC | OPT_OUTPUT, { .off = OFFSET(fix_sub_duration_heartbeat) },
+ "set this video output stream to be a heartbeat stream for "
+ "fix_sub_duration, according to which subtitles should be split at "
+ "random access points" },
/* audio options */
{ "aframes", OPT_AUDIO | HAS_ARG | OPT_PERFILE | OPT_OUTPUT, { .func_arg = opt_audio_frames },
--
2.33.1
More information about the ffmpeg-devel
mailing list