[FFmpeg-devel] [RFC][GSoC][PATCH v1 3/6] avformat/hls: use abr to switch streams

Steven Liu lingjiujianke at gmail.com
Wed Jul 15 12:24:01 EEST 2020


Hongcheng Zhong <sj.hc_Zhong at sjtu.edu.cn> 于2020年7月15日周三 下午4:38写道:
>
> From: spartazhc <spartazhc at gmail.com>
>
> When abr is enable, it will take over the task to call http to
> download segments, and will return a switch-request for hls to
> switch streams.
> For reason not to waste segments that have been downloaded,
> switch will become effective after old segments is used out.
> Abr cannot work with http_persistent option, and currently use
> http_multiple.
>
> v1 fixed:
> 1. fix memory leak
>
> Signed-off-by: spartazhc <spartazhc at gmail.com>
> ---
>  doc/demuxers.texi |   3 +
>  libavformat/hls.c | 232 ++++++++++++++++++++++++++++++++++++++++++++--
>  2 files changed, 229 insertions(+), 6 deletions(-)
>
> diff --git a/doc/demuxers.texi b/doc/demuxers.texi
> index 3c15ab9eee..4cdbd95962 100644
> --- a/doc/demuxers.texi
> +++ b/doc/demuxers.texi
> @@ -321,6 +321,9 @@ available in a metadata key named "variant_bitrate".
>  It accepts the following options:
>
>  @table @option
> + at item abr
> +enable abr to switch streams.
> +
>  @item live_start_index
>  segment index to start live streams at (negative values are from the end).
>
> diff --git a/libavformat/hls.c b/libavformat/hls.c
> index ba17c4ed96..877bd8c677 100644
> --- a/libavformat/hls.c
> +++ b/libavformat/hls.c
> @@ -189,6 +189,17 @@ struct variant {
>      char subtitles_group[MAX_FIELD_LEN];
>  };
>
> +struct throughput {
> +    int n_throughputs;
> +
> +    /* throughputs are in kbps, always record the last 20 times
> +     * set record 20 times refer to paper FESTIVE https://doi.org/10.1145/2413176.2413189
> +     */
> +    float throughput_fifo[20];
> +    int head;
> +    int tail;
> +};

what about add an #define blablabla_name 20 here if use the record 20
times, and use the blablabla_name at the below code which use the 20
value?
> +
>  typedef struct HLSContext {
>      AVClass *class;
>      AVFormatContext *ctx;
> @@ -213,8 +224,36 @@ typedef struct HLSContext {
>      int http_multiple;
>      int http_seekable;
>      AVIOContext *playlist_pb;
> +
> +    int abr;
> +    struct throughput *throughputs;
> +    int can_switch;
> +    int switch_request2;
> +    int switch_delay;
> +    int64_t switch_timestamp;
> +    int64_t delta_timestamp;
> +    int cur_pls;
>  } HLSContext;
>
> +static struct segment *next_segment(struct playlist *pls);
> +static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg, AVIOContext **in);
> +
> +static void sync_cur_seq(HLSContext *c) {
> +    int i;
> +    for (i = 0; i < c->n_playlists; i++) {
> +        struct playlist *pls = c->playlists[i];
> +        pls->cur_seq_no = c->cur_seq_no;
> +    }
> +}
> +
> +static struct segment *next2_segment(struct playlist *pls)
> +{
> +    int n = pls->cur_seq_no - pls->start_seq_no + 2;
> +    if (n >= pls->n_segments)
> +        return NULL;
> +    return pls->segments[n];
> +}
> +
>  static void free_segment_dynarray(struct segment **segments, int n_segments)
>  {
>      int i;
> @@ -624,6 +663,31 @@ static int open_url_keepalive(AVFormatContext *s, AVIOContext **pb,
>  #endif
>  }
>
> +static int update_throughputs(struct throughput *thr, float time, int pb_size)
> +{
> +    if (pb_size <= 0 || time <= 0)
> +        return -1;
EINVAL?
> +    if (thr->n_throughputs < 20) {
> +        ++thr->n_throughputs;
> +    } else {
> +        ++thr->head;
> +    }
> +    thr->throughput_fifo[thr->tail] = (float)(pb_size) / time;
> +    thr->tail = (thr->tail + 1) % 20;
> +    return 0;
> +}
> +
> +static int64_t get_switch_timestamp(HLSContext *c, struct playlist *pls)
> +{
> +    int64_t pos = c->first_timestamp == AV_NOPTS_VALUE ?
> +                  0 : c->first_timestamp;
> +
> +    for (int i = 0; i < pls->cur_seq_no + 2; i++) {
> +        pos += pls->segments[i]->duration;
> +    }
> +    return pos;
> +}
> +
>  static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>                      AVDictionary *opts, AVDictionary *opts2, int *is_http_out)
>  {
> @@ -639,6 +703,9 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>      } else if (av_strstart(url, "data", NULL)) {
>          if (url[4] == '+' || url[4] == ':')
>              proto_name = avio_find_protocol_name(url + 5);
> +    } else if (av_strstart(url, "ffabr", NULL)) {
> +        if (url[5] == '+' || url[5] == ':')
> +            proto_name = avio_find_protocol_name(url + 6);
>      }
>
>      if (!proto_name)
> @@ -669,6 +736,8 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>          ;
>      else if (av_strstart(url, "data", NULL) && !strncmp(proto_name, url + 5, strlen(proto_name)) && url[5 + strlen(proto_name)] == ':')
>          ;
> +    else if (av_strstart(url, "ffabr", NULL) && !strncmp(proto_name, url + 6, strlen(proto_name)) && url[6 + strlen(proto_name)] == ':')
> +        ;
>      else if (strcmp(proto_name, "file") || !strncmp(url, "file,", 5))
>          return AVERROR_INVALIDDATA;
>
> @@ -690,6 +759,43 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>      } else {
>          ret = s->io_open(s, pb, url, AVIO_FLAG_READ, &tmp);
>      }
> +    if (c->abr && ret >= 0) {
> +        AVDictionary *abr_ret = NULL;
> +        AVDictionaryEntry *en = NULL;
> +        struct segment *seg;
> +        int pb_size, switch_request;
> +        av_opt_get_dict_val(*pb, "abr-metadata", AV_OPT_SEARCH_CHILDREN, &abr_ret);
> +        if (abr_ret) {
> +            en = av_dict_get(abr_ret, "download_time", NULL, 0);
> +            if (en) {
> +                pb_size = avio_size(*pb);
> +                update_throughputs(c->throughputs, atoi(en->value) / 1000.0, pb_size);
> +                av_log(s, AV_LOG_VERBOSE, "[abr] time=%.2fms, size=%.2fkb\n", atoi(en->value) / 1000.0, pb_size / 1000.0);
> +            }
> +            en = av_dict_get(abr_ret, "switch_request", NULL, 0);
> +            if (en) {
> +                switch_request = atoi(en->value);
> +                av_log(s, AV_LOG_VERBOSE, "[abr] switch request: %s\n", en->value);
> +            }
> +            if (switch_request != -1) {
> +                c->switch_request2 = switch_request;
> +                c->switch_delay = 2;
> +                c->can_switch = 0;
> +                sync_cur_seq(c);
> +                seg = next2_segment(c->playlists[c->switch_request2]);
> +                if (!seg) {
> +                    c->switch_request2 = -1;
> +                    av_log(s, AV_LOG_INFO, "[abr] no more segment need to switch\n");
> +                } else {
> +                    c->switch_timestamp = get_switch_timestamp(c,  c->playlists[c->switch_request2]);
> +                    av_log(s, AV_LOG_VERBOSE, "[abr] switch timestamp: %ld\n", c->switch_timestamp);
> +                    c->playlists[c->switch_request2]->input_next_requested = 1;
> +                    ret = open_input(c, c->playlists[c->switch_request2], seg, &c->playlists[c->switch_request2]->input_next);
> +                }
> +            }
> +            av_dict_free(&abr_ret);
> +        }
> +    }
>      if (ret >= 0) {
>          // update cookies on http response with setcookies.
>          char *new_cookies = NULL;
> @@ -1219,6 +1325,33 @@ static void intercept_id3(struct playlist *pls, uint8_t *buf,
>          pls->is_id3_timestamped = (pls->id3_mpegts_timestamp != AV_NOPTS_VALUE);
>  }
>
> +static int abrinfo_to_dict(HLSContext *c, char **abr_info)
> +{
> +    struct throughput *thr = c->throughputs;
> +    char buffer[MAX_URL_SIZE];
> +    int size, i;
> +    size = snprintf(buffer, sizeof(buffer), "format=hls:");
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "cur_pls=%d:", c->cur_pls);
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "can_switch=%d:", c->can_switch);
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "n_variants=%d:", c->n_variants);
> +    for (i = 0; i < c->n_variants; i++) {
> +        struct variant *v = c->variants[i];
> +        size += snprintf(buffer + size, sizeof(buffer) - size, "variant_bitrate%d=%d:", i, v->bandwidth);
> +    }
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "n_throughputs=%d:", thr->n_throughputs);
> +    if (thr->n_throughputs > 0) {
> +        i = thr->head;
> +        do {
> +            size += snprintf(buffer + size, sizeof(buffer) - size, "throughputs%d=%.2f:", i, thr->throughput_fifo[i]);
> +            i = (i + 1) % 20;
> +        } while (i != thr->tail);
> +    }
> +    *abr_info = av_malloc(size);
check the return NULL of av_malloc.
> +    snprintf(*abr_info, size, "%s", buffer);
> +    av_log(c, AV_LOG_DEBUG, "[abr] abr_info: %s\n", *abr_info);
> +    return 0;
> +}
> +
>  static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg, AVIOContext **in)
>  {
>      AVDictionary *opts = NULL;
> @@ -1238,8 +1371,20 @@ static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg,
>      av_log(pls->parent, AV_LOG_VERBOSE, "HLS request for url '%s', offset %"PRId64", playlist %d\n",
>             seg->url, seg->url_offset, pls->index);
>
> +    if (c->abr) {
> +        char *abr_opts;
> +        abrinfo_to_dict(c, &abr_opts);
> +        av_dict_set(&opts, "abr-params", abr_opts, 0); // how to setup flag?
> +        av_free(abr_opts);
> +    }
>      if (seg->key_type == KEY_NONE) {
> -        ret = open_url(pls->parent, in, seg->url, c->avio_opts, opts, &is_http);
> +        char url_abr[MAX_URL_SIZE];
> +        if (c->abr) {
> +            snprintf(url_abr, sizeof(url_abr), "ffabr:%s", seg->url); // + or : tbd
> +            ret = open_url(pls->parent, in, url_abr, c->avio_opts, opts, &is_http);
> +        } else {
> +            ret = open_url(pls->parent, in, seg->url, c->avio_opts, opts, &is_http);
> +        }
>      } else if (seg->key_type == KEY_AES_128) {
>          char iv[33], key[33], url[MAX_URL_SIZE];
>          if (strcmp(seg->key, pls->key_url)) {
> @@ -1273,6 +1418,7 @@ static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg,
>              goto cleanup;
>          }
>          ret = 0;
> +        // TODO: Add abr prefix for crypto
>      } else if (seg->key_type == KEY_SAMPLE_AES) {
>          av_log(pls->parent, AV_LOG_ERROR,
>                 "SAMPLE-AES encryption is not supported yet\n");
> @@ -1435,7 +1581,17 @@ restart:
>          /* Check that the playlist is still needed before opening a new
>           * segment. */
>          v->needed = playlist_needed(v);
> -
> +        if (c->abr) {
> +            if (c->switch_request2 == -1)
> +                ;
> +            else if (v->needed && c->switch_request2 != v->index && c->switch_delay <= 0) {
> +                av_log(v->parent, AV_LOG_VERBOSE, "read_data: needed but not download playlist %d ('%s')\n",
> +                       v->index, v->url);
> +                return AVERROR_EOF;
> +            }
> +            if (!c->can_switch)
> +                c->can_switch = 1;
> +        }
>          if (!v->needed) {
>              av_log(v->parent, AV_LOG_INFO, "No longer receiving playlist %d ('%s')\n",
>                     v->index, v->url);
> @@ -1530,7 +1686,12 @@ reload:
>      }
>
>      seg = next_segment(v);
> -    if (c->http_multiple == 1 && !v->input_next_requested &&
> +
> +    // when get switch_request, old stream should stop downloading next seg
> +    if (c->abr && c->switch_request2 != v->index && c->switch_timestamp != AV_NOPTS_VALUE
> +         && ((c->cur_timestamp + seg->duration * 2) >= c->switch_timestamp) )
> +        ;
> +    else if (c->http_multiple == 1 && !v->input_next_requested &&
else if (c->http_multiple && !v->input_next_requested &&
>          seg && seg->key_type == KEY_NONE && av_strstart(seg->url, "http", NULL)) {
>          ret = open_input(c, v, seg, &v->input_next);
>          if (ret < 0) {
> @@ -1563,11 +1724,17 @@ reload:
>
>          return ret;
>      }
> -    if (c->http_persistent &&
> +    if (c->http_persistent && !c->abr &&
>          seg->key_type == KEY_NONE && av_strstart(seg->url, "http", NULL)) {
>          v->input_read_done = 1;
>      } else {
>          ff_format_io_close(v->parent, &v->input);
> +        if (c->abr) {
> +            if (c->switch_delay > 0)
> +                c->switch_delay--;
> +            av_log(v->parent, AV_LOG_VERBOSE, "read_data: close pls[%d]->input, v->cur_seq_no=%d, c->switch_delay= %d\n",
> +                    v->index, v->cur_seq_no, c->switch_delay);
> +        }
>      }
>      v->cur_seq_no++;
>
> @@ -1860,6 +2027,17 @@ static int hls_read_header(AVFormatContext *s)
>      c->first_packet = 1;
>      c->first_timestamp = AV_NOPTS_VALUE;
>      c->cur_timestamp = AV_NOPTS_VALUE;
> +    c->switch_request2 = -1;
> +    c->switch_delay = 0;
> +    c->switch_timestamp = AV_NOPTS_VALUE;
> +    c->delta_timestamp = AV_NOPTS_VALUE;
> +    c->can_switch = -1;
> +
> +    if (c->abr) {
> +        c->http_persistent = 0;
> +        c->http_multiple = 1;
> +        c->throughputs = av_mallocz(sizeof(struct throughput));
check return NULL;
> +    }
>
>      if ((ret = save_avio_options(s)) < 0)
>          goto fail;
> @@ -2049,6 +2227,9 @@ static int hls_read_header(AVFormatContext *s)
>          add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_SUBTITLE);
>      }
>
> +    if (c->abr && c->can_switch == -1)
> +        c->can_switch = 1;
> +
>      update_noheader_flag(s);
>
>      return 0;
> @@ -2057,6 +2238,13 @@ fail:
>      return ret;
>  }
>
> +static void change_discard_flags(struct playlist *pls, int flag)
> +{
> +    for (int i = 0; i < pls->n_main_streams; i++) {
> +        pls->main_streams[i]->discard = flag;
> +    }
> +}
> +
>  static int recheck_discard_flags(AVFormatContext *s, int first)
>  {
>      HLSContext *c = s->priv_data;
> @@ -2067,6 +2255,22 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>      for (i = 0; i < c->n_playlists; i++) {
>          struct playlist *pls = c->playlists[i];
>
> +        if (c->abr) {
> +            if (c->switch_request2 == -1)
> +                ;
> +            else if (c->switch_request2 == i && c->switch_delay <= 0
> +                    && c->cur_timestamp + c->delta_timestamp >= c->switch_timestamp
> +                    && c->switch_timestamp != AV_NOPTS_VALUE) {
> +                av_log(s, AV_LOG_VERBOSE, "[switch point] cur_timestamp:%ld\n", c->cur_timestamp);
> +                change_discard_flags(pls, AVDISCARD_DEFAULT);
> +                c->switch_timestamp = AV_NOPTS_VALUE;
> +            } else if (c->switch_request2 != -1 && c->switch_request2 != i && c->switch_delay <= 0
> +                    && c->cur_timestamp + c->delta_timestamp >= c->switch_timestamp
> +                    && c->switch_timestamp != AV_NOPTS_VALUE) {
> +                change_discard_flags(pls, AVDISCARD_ALL);
> +            }
> +        }
> +
>          cur_needed = playlist_needed(c->playlists[i]);
>
>          if (pls->broken) {
> @@ -2077,6 +2281,11 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>              changed = 1;
>              pls->cur_seq_no = select_cur_seq_no(c, pls);
>              pls->pb.eof_reached = 0;
> +            if (c->abr) {
> +                pls->cur_seq_no = select_cur_seq_no(c, pls) + 1;
> +                avio_flush(&pls->pb);
> +                avformat_flush(pls->ctx);
> +            }
>              if (c->cur_timestamp != AV_NOPTS_VALUE) {
>                  /* catch up */
>                  pls->seek_timestamp = c->cur_timestamp;
> @@ -2084,7 +2293,7 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>                  pls->seek_stream_index = -1;
>              }
>              av_log(s, AV_LOG_INFO, "Now receiving playlist %d, segment %d\n", i, pls->cur_seq_no);
> -        } else if (first && !cur_needed && pls->needed) {
> +        } else if ((first || c->abr) && !cur_needed && pls->needed) {
>              ff_format_io_close(pls->parent, &pls->input);
>              pls->input_read_done = 0;
>              ff_format_io_close(pls->parent, &pls->input_next);
> @@ -2093,6 +2302,9 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>              changed = 1;
>              av_log(s, AV_LOG_INFO, "No longer receiving playlist %d\n", i);
>          }
> +
> +        if (c->abr && changed && cur_needed)
> +            c->cur_pls = i;
>      }
>      return changed;
>  }
> @@ -2167,7 +2379,13 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
>                          /* audio elementary streams are id3 timestamped */
>                          fill_timing_for_id3_timestamped_stream(pls);
>                      }
> -
> +                    if (c->abr && c->first_timestamp != AV_NOPTS_VALUE && c->delta_timestamp == AV_NOPTS_VALUE && !i) {
> +                        c->delta_timestamp = av_rescale_q(pls->pkt.dts,
> +                            get_timebase(pls), AV_TIME_BASE_Q) - c->first_timestamp + 1;
> +                        av_log(s, AV_LOG_VERBOSE, "[abr] delta_timestamp=%ld, %ld, %ld\n",
> +                             c->delta_timestamp, c->first_timestamp,av_rescale_q(pls->pkt.dts,
> +                            get_timebase(pls), AV_TIME_BASE_Q));
> +                    }
>                      if (c->first_timestamp == AV_NOPTS_VALUE &&
>                          pls->pkt.dts       != AV_NOPTS_VALUE)
>                          c->first_timestamp = av_rescale_q(pls->pkt.dts,
> @@ -2375,6 +2593,8 @@ static int hls_probe(const AVProbeData *p)
>  #define OFFSET(x) offsetof(HLSContext, x)
>  #define FLAGS AV_OPT_FLAG_DECODING_PARAM
>  static const AVOption hls_options[] = {
> +    {"abr", "enable abr to switch streams",
> +        OFFSET(abr), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS },
>      {"live_start_index", "segment index to start live streams at (negative values are from the end)",
>          OFFSET(live_start_index), AV_OPT_TYPE_INT, {.i64 = -3}, INT_MIN, INT_MAX, FLAGS},
>      {"allowed_extensions", "List of file extensions that hls is allowed to access",
> --
> 2.27.0
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".


More information about the ffmpeg-devel mailing list