[FFmpeg-devel] [PATCH 2/2] avformat/hlsenc: reopen new http session for http_persistent when upload file failed

Steven Liu lq at chinaffmpeg.org
Sun Aug 25 17:57:40 EEST 2019



> 在 2019年8月25日,22:53,Steven Liu <lq at chinaffmpeg.org> 写道:
> 
> fix ticket: 7975
> 
> Signed-off-by: Steven Liu <lq at chinaffmpeg.org>
> ---
> libavformat/hlsenc.c | 66 ++++++++++++++++++++++++++++++++++++++++++----------
> 1 file changed, 54 insertions(+), 12 deletions(-)
> 
> diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
> index 18173cdce1..a4cb832766 100644
> --- a/libavformat/hlsenc.c
> +++ b/libavformat/hlsenc.c
> @@ -118,6 +118,7 @@ typedef struct VariantStream {
>     AVIOContext *out;
>     int packets_written;
>     int init_range_length;
> +    uint8_t *temp_buffer;
> 
>     AVFormatContext *avf;
>     AVFormatContext *vtt_avf;
> @@ -262,11 +263,12 @@ static int hlsenc_io_open(AVFormatContext *s, AVIOContext **pb, char *filename,
>     return err;
> }
> 
> -static void hlsenc_io_close(AVFormatContext *s, AVIOContext **pb, char *filename) {
> +static int hlsenc_io_close(AVFormatContext *s, AVIOContext **pb, char *filename) {
>     HLSContext *hls = s->priv_data;
>     int http_base_proto = filename ? ff_is_http_proto(filename) : 0;
> +    int ret = 0;
>     if (!*pb)
> -        return;
> +        return ret;
>     if (!http_base_proto || !hls->http_persistent || hls->key_info_file || hls->encrypt) {
>         ff_format_io_close(s, pb);
> #if CONFIG_HTTP_PROTOCOL
> @@ -275,8 +277,10 @@ static void hlsenc_io_close(AVFormatContext *s, AVIOContext **pb, char *filename
>         av_assert0(http_url_context);
>         avio_flush(*pb);
>         ffurl_shutdown(http_url_context, AVIO_FLAG_WRITE);
> +        ret = ff_http_get_shutdown_status(http_url_context);
> #endif
>     }
> +    return ret;
> }
> 
> static void set_http_options(AVFormatContext *s, AVDictionary **options, HLSContext *c)
> @@ -447,7 +451,6 @@ static void write_styp(AVIOContext *pb)
> static int flush_dynbuf(VariantStream *vs, int *range_length)
> {
>     AVFormatContext *ctx = vs->avf;
> -    uint8_t *buffer;
> 
>     if (!ctx->pb) {
>         return AVERROR(EINVAL);
> @@ -458,15 +461,20 @@ static int flush_dynbuf(VariantStream *vs, int *range_length)
>     avio_flush(ctx->pb);
> 
>     // write out to file
> -    *range_length = avio_close_dyn_buf(ctx->pb, &buffer);
> +    *range_length = avio_close_dyn_buf(ctx->pb, &vs->temp_buffer);
>     ctx->pb = NULL;
> -    avio_write(vs->out, buffer, *range_length);
> -    av_free(buffer);
> +    avio_write(vs->out, vs->temp_buffer, *range_length);
> 
>     // re-open buffer
>     return avio_open_dyn_buf(&ctx->pb);
> }
> 
> +static void reflush_dynbuf(VariantStream *vs, int *range_length)
> +{
> +    // re-open buffer
> +    avio_write(vs->out, vs->temp_buffer, *range_length);;
> +}
> +
> static int hls_delete_old_segments(AVFormatContext *s, HLSContext *hls,
>                                    VariantStream *vs) {
> 
> @@ -1544,7 +1552,10 @@ static int hls_window(AVFormatContext *s, int last, VariantStream *vs)
> 
> fail:
>     av_dict_free(&options);
> -    hlsenc_io_close(s, (byterange_mode || hls->segment_type == SEGMENT_TYPE_FMP4) ? &hls->m3u8_out : &vs->out, temp_filename);
> +    ret = hlsenc_io_close(s, (byterange_mode || hls->segment_type == SEGMENT_TYPE_FMP4) ? &hls->m3u8_out : &vs->out, temp_filename);
> +    if (ret < 0) {
> +        return ret;
> +    }
>     hlsenc_io_close(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name);
>     if (use_temp_file) {
>         ff_rename(temp_filename, vs->m3u8_name, s);
> @@ -2399,7 +2410,16 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
>                 if (ret < 0) {
>                     return ret;
>                 }
> -                hlsenc_io_close(s, &vs->out, filename);
> +                ret = hlsenc_io_close(s, &vs->out, filename);
> +                if (ret < 0) {
> +                    av_log(s, AV_LOG_WARNING, "upload segment failed,"
> +                           "and will retry upload by a new http session.\n");
> +                    ff_format_io_close(s, &vs->out);
> +                    ret = hlsenc_io_open(s, &vs->out, filename, &options);
> +                    reflush_dynbuf(vs, &range_length);
> +                    ret = hlsenc_io_close(s, &vs->out, filename);
> +                }
> +                av_free(vs->temp_buffer);
>                 av_free(filename);
>             }
>         }
> @@ -2426,8 +2446,13 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
>         // if we're building a VOD playlist, skip writing the manifest multiple times, and just wait until the end
>         if (hls->pl_type != PLAYLIST_TYPE_VOD) {
>             if ((ret = hls_window(s, 0, vs)) < 0) {
> -                av_free(old_filename);
> -                return ret;
> +                av_log(s, AV_LOG_WARNING, "update playlist failed, will retry once time\n");
> +                ff_format_io_close(s, &vs->out);
> +                vs->out = NULL;
> +                if ((ret = hls_window(s, 0, vs)) < 0) {
> +                    av_free(old_filename);
> +                    return ret;
> +                }
>             }
>         }
> 
> @@ -2577,7 +2602,19 @@ static int hls_write_trailer(struct AVFormatContext *s)
>             goto failed;
> 
>         vs->size = range_length;
> -        hlsenc_io_close(s, &vs->out, filename);
> +        ret = hlsenc_io_close(s, &vs->out, filename);
> +        if (ret < 0) {
> +            av_log(s, AV_LOG_WARNING, "write segment failed, will close old handle and retry once time\n");
> +            ff_format_io_close(s, &vs->out);
> +            ret = hlsenc_io_open(s, &vs->out, filename, &options);
> +            if (ret < 0) {
> +                av_log(s, AV_LOG_ERROR, "Failed to open file '%s'\n", vs->avf->url);
> +                goto failed;
> +            }
> +            reflush_dynbuf(vs, &range_length);
> +            ret = hlsenc_io_close(s, &vs->out, filename);
> +        }
> +        av_free(vs->temp_buffer);
>         av_free(filename);
> 
> failed:
> @@ -2610,7 +2647,12 @@ failed:
>             ff_format_io_close(s, &vtt_oc->pb);
>             avformat_free_context(vtt_oc);
>         }
> -        hls_window(s, 1, vs);
> +        ret = hls_window(s, 1, vs);
> +        if (ret < 0) {
> +            av_log(s, AV_LOG_WARNING, "update playlist failed, will retry once time\n");
> +            ff_format_io_close(s, &vs->out);
> +            hls_window(s, 1, vs);
> +        }
>         avformat_free_context(oc);
> 
>         vs->avf = NULL;
> -- 
> 2.15.1
> 


This patchiest test result:

liuqideMBP:dash liuqi$ ./ffmpeg -f lavfi -i testsrc2=s=176x144 -vcodec libx264 -g 50 -r:v 25 -f hls -hls_time 2 -http_persistent 1 -method PUT -t 20 http://127.0.0.1/output.m3u8
ffmpeg version N-94621-g311803b899 Copyright (c) 2000-2019 the FFmpeg developers
  built with Apple LLVM version 10.0.0 (clang-1000.11.45.5)
  configuration: --enable-fontconfig --enable-gpl --enable-libass --enable-libbluray --enable-libfreetype --enable-libmp3lame --enable-libspeex --enable-libx264 --enable-libx265 --enable-libfdk-aac --enable-version3 --enable-nonfree --enable-videotoolbox --enable-libxml2 --samples=fate-suite/ --enable-libopencv
  libavutil      56. 33.100 / 56. 33.100
  libavcodec     58. 55.100 / 58. 55.100
  libavformat    58. 31.101 / 58. 31.101
  libavdevice    58.  9.100 / 58.  9.100
  libavfilter     7. 58.101 /  7. 58.101
  libswscale      5.  6.100 /  5.  6.100
  libswresample   3.  6.100 /  3.  6.100
  libpostproc    55.  6.100 / 55.  6.100
Input #0, lavfi, from 'testsrc2=s=176x144':
  Duration: N/A, start: 0.000000, bitrate: N/A
    Stream #0:0: Video: rawvideo (I420 / 0x30323449), yuv420p, 176x144 [SAR 1:1 DAR 11:9], 25 tbr, 25 tbn, 25 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (rawvideo (native) -> h264 (libx264))
Press [q] to stop, [?] for help
[libx264 @ 0x7f888281b800] using SAR=1/1
[libx264 @ 0x7f888281b800] using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX
[libx264 @ 0x7f888281b800] profile High, level 1.1
[libx264 @ 0x7f888281b800] 264 - core 148 r2694 3b70645 - H.264/MPEG-4 AVC codec - Copyleft 2003-2016 - http://www.videolan.org/x264.html - options: cabac=1 ref=3 deblock=1:0:0 analyse=0x3:0x113 me=hex subme=7 psy=1 psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=1 trellis=1 8x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=4 lookahead_threads=1 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=3 b_pyramid=2 b_adapt=1 b_bias=0 direct=1 weightb=1 open_gop=0 weightp=2 keyint=50 keyint_min=5 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=23.0 qcomp=0.60 qpmin=0 qpmax=69 qpstep=4 ip_ratio=1.40 aq=1:1.00
Output #0, hls, to 'http://127.0.0.1/output.m3u8':
  Metadata:
    encoder         : Lavf58.31.101
    Stream #0:0: Video: h264 (libx264), yuv420p, 176x144 [SAR 1:1 DAR 11:9], q=-1--1, 25 fps, 90k tbn, 25 tbc
    Metadata:
      encoder         : Lavc58.55.100 libx264
    Side data:
      cpb: bitrate max/min/avg: 0/0/0 buffer size: 0 vbv_delay: 18446744073709551615
[hls @ 0x7f8882818400] Opening 'http://127.0.0.1/output0.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output1.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output2.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output3.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output4.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output5.ts' for writing
[http @ 0x7f8882406080] URL read error: End of file
[hls @ 0x7f8882818400] upload segment failed,and will retry upload by a new http session.
[hls @ 0x7f8882818400] Opening 'http://127.0.0.1/output5.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output6.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output7.ts' for writing.2x
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output8.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output9.ts' for writing
[http @ 0x7f88832e7400] Opening 'http://127.0.0.1/output.m3u8' for writing
frame=  500 fps=0.0 q=-1.0 Lsize=N/A time=00:00:19.92 bitrate=N/A speed=29.4x
video:413kB audio:0kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: unknown
[libx264 @ 0x7f888281b800] frame I:10    Avg QP:20.41  size:  3776
[libx264 @ 0x7f888281b800] frame P:138   Avg QP:26.88  size:  1239
[libx264 @ 0x7f888281b800] frame B:352   Avg QP:32.05  size:   606
[libx264 @ 0x7f888281b800] consecutive B-frames:  3.8%  4.4%  7.8% 84.0%
[libx264 @ 0x7f888281b800] mb I  I16..4: 32.3% 15.9% 51.8%
[libx264 @ 0x7f888281b800] mb P  I16..4:  3.5%  2.6%  2.6%  P16..4: 18.0% 16.0% 13.9%  0.0%  0.0%    skip:43.5%
[libx264 @ 0x7f888281b800] mb B  I16..4:  0.3%  0.2%  0.1%  B16..8: 26.7% 13.0%  4.4%  direct: 4.0%  skip:51.4%  L0:46.8% L1:46.6% BI: 6.6%
[libx264 @ 0x7f888281b800] 8x8 transform intra:24.0% inter:24.8%
[libx264 @ 0x7f888281b800] coded y,uvDC,uvAC intra: 26.1% 56.5% 46.9% inter: 10.0% 25.3% 21.1%
[libx264 @ 0x7f888281b800] i16 v,h,dc,p: 74% 17%  9%  0%
[libx264 @ 0x7f888281b800] i8 v,h,dc,ddl,ddr,vr,hd,vl,hu: 16%  9% 74%  1%  0%  0%  0%  0%  1%
[libx264 @ 0x7f888281b800] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 47% 25% 20%  1%  1%  1%  2%  2%  2%
[libx264 @ 0x7f888281b800] i8c dc,h,v,p: 39% 20% 39%  2%
[libx264 @ 0x7f888281b800] Weighted P-Frames: Y:0.0% UV:0.0%
[libx264 @ 0x7f888281b800] ref P L0: 60.1%  8.2% 20.1% 11.6%
[libx264 @ 0x7f888281b800] ref B L0: 78.3% 17.9%  3.8%
[libx264 @ 0x7f888281b800] ref B L1: 90.6%  9.4%
[libx264 @ 0x7f888281b800] kb/s:168.78
liuqideMBP:dash liuqi$



Nginx log:

127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output0.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output1.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output2.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output3.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output4.ts HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output5.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output6.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output7.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output8.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output9.ts HTTP/1.1" 201 0 "-" "Lavf/58.31.101"
127.0.0.1 - - [25/Aug/2019:22:56:07 +0800] "PUT /output.m3u8 HTTP/1.1" 204 0 "-" "Lavf/58.31.101"






the files in http server:

liuqideMBP:~ liuqi$ ls /usr/local/nginx/html/output*
/usr/local/nginx/html/output.m3u8 /usr/local/nginx/html/output2.ts  /usr/local/nginx/html/output5.ts  /usr/local/nginx/html/output8.ts
/usr/local/nginx/html/output0.ts  /usr/local/nginx/html/output3.ts  /usr/local/nginx/html/output6.ts  /usr/local/nginx/html/output9.ts
/usr/local/nginx/html/output1.ts  /usr/local/nginx/html/output4.ts  /usr/local/nginx/html/output7.ts
liuqideMBP:~ liuqi$



http config file:

http {
    include       mime.types;
    default_type  application/octet-stream;
    keepalive_timeout  100;
    keepalive_requests 10;





Thanks
Steven







More information about the ffmpeg-devel mailing list