[FFmpeg-devel] [PATCH v2] avformat: add a concat protocol that takes a line break delimited list of resources
James Almer
jamrial at gmail.com
Sun Jun 27 21:51:17 EEST 2021
On 6/27/2021 3:26 PM, Nicolas George wrote:
> James Almer (12021-06-26):
>> Suggested-by: ffmpeg at fb.com
>> Signed-off-by: James Almer <jamrial at gmail.com>
>> ---
>> Updated documentation, and line breaks can now be part of the filename.
>>
>> doc/protocols.texi | 33 +++++++++
>> libavformat/Makefile | 1 +
>> libavformat/concat.c | 146 ++++++++++++++++++++++++++++++++++++++++
>> libavformat/protocols.c | 1 +
>> 4 files changed, 181 insertions(+)
>>
>> diff --git a/doc/protocols.texi b/doc/protocols.texi
>> index ccdfb6e439..11de674225 100644
>> --- a/doc/protocols.texi
>> +++ b/doc/protocols.texi
>> @@ -215,6 +215,39 @@ ffplay concat:split1.mpeg\|split2.mpeg\|split3.mpeg
>> Note that you may need to escape the character "|" which is special for
>> many shells.
>>
>> + at section concatf
>> +
>> +Physical concatenation protocol using a line break delimited list of
>> +resources.
>> +
>> +Read and seek from many resources in sequence as if they were
>> +a unique resource.
>> +
>> +A URL accepted by this protocol has the syntax:
>> + at example
>> +concatf:@var{URL}
>> + at end example
>> +
>> +where @var{URL} is the url containing a line break delimited list of
>> +resources to be concatenated, each one possibly specifying a distinct
>> +protocol.
>> +
>> +For example to read a sequence of files @file{split1.mpeg},
>> + at file{split2.mpeg}, @file{split3.mpeg} listed in separate lines within
>> +a file @file{split.txt} with @command{ffplay} use the command:
>> + at example
>> +ffplay concatf:split.txt
>> + at end example
>> +Where @file{split.txt} contains the lines:
>> + at example
>> +split1.mpeg
>> +split2.mpeg
>> +split3.mpeg
>> + at end example
>> +
>
>> +Note that if any of the entries in the list contain a line break as part
>> +of their name, you'll need to escape it with a preceding "\" character.
>
> This is not detailed and accurate enough. And it should be a link to the
> common description of our escaping syntax anyway.
How is it not accurate enough? If the file contains the following
foobar.h264
foo\
bar.h264
It will read the resources
foobar.h264
and
foo
bar.h264
But without that backslash, it will try to read foo and bar.h264 as
separate resources because it interpreted the line break as a delimiter
character.
I can add a line to the documentation stating that no other character
needs to be escaped if that will make it more clear.
>
>> +
>> @section crypto
>>
>> AES-encrypted stream reading protocol.
>> diff --git a/libavformat/Makefile b/libavformat/Makefile
>> index c9ef564523..caca95802a 100644
>> --- a/libavformat/Makefile
>> +++ b/libavformat/Makefile
>> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_APPLEHTTP_PROTOCOL) += hlsproto.o
>> OBJS-$(CONFIG_BLURAY_PROTOCOL) += bluray.o
>> OBJS-$(CONFIG_CACHE_PROTOCOL) += cache.o
>> OBJS-$(CONFIG_CONCAT_PROTOCOL) += concat.o
>> +OBJS-$(CONFIG_CONCATF_PROTOCOL) += concat.o
>> OBJS-$(CONFIG_CRYPTO_PROTOCOL) += crypto.o
>> OBJS-$(CONFIG_DATA_PROTOCOL) += data_uri.o
>> OBJS-$(CONFIG_FFRTMPCRYPT_PROTOCOL) += rtmpcrypt.o rtmpdigest.o rtmpdh.o
>> diff --git a/libavformat/concat.c b/libavformat/concat.c
>> index 278afd997d..b66e3b9e01 100644
>> --- a/libavformat/concat.c
>> +++ b/libavformat/concat.c
>> @@ -22,9 +22,11 @@
>> */
>>
>> #include "libavutil/avstring.h"
>> +#include "libavutil/bprint.h"
>> #include "libavutil/mem.h"
>>
>> #include "avformat.h"
>> +#include "avio_internal.h"
>> #include "url.h"
>>
>> #define AV_CAT_SEPARATOR "|"
>> @@ -56,6 +58,7 @@ static av_cold int concat_close(URLContext *h)
>> return err < 0 ? -1 : 0;
>> }
>>
>> +#if CONFIG_CONCAT_PROTOCOL
>> static av_cold int concat_open(URLContext *h, const char *uri, int flags)
>> {
>> char *node_uri = NULL;
>> @@ -124,6 +127,7 @@ static av_cold int concat_open(URLContext *h, const char *uri, int flags)
>> data->total_size = total_size;
>> return err;
>> }
>> +#endif
>>
>> static int concat_read(URLContext *h, unsigned char *buf, int size)
>> {
>> @@ -188,6 +192,7 @@ static int64_t concat_seek(URLContext *h, int64_t pos, int whence)
>> return result;
>> }
>>
>> +#if CONFIG_CONCAT_PROTOCOL
>> const URLProtocol ff_concat_protocol = {
>> .name = "concat",
>> .url_open = concat_open,
>> @@ -197,3 +202,144 @@ const URLProtocol ff_concat_protocol = {
>> .priv_data_size = sizeof(struct concat_data),
>> .default_whitelist = "concat,file,subfile",
>> };
>> +#endif
>> +
>
>> +#if CONFIG_CONCATF_PROTOCOL
>> +// Custom ff_read_line_to_bprint() implementation where line breaks can be
>> +// part of the line being read if escaped.
>> +static int64_t read_line_to_bprint(AVIOContext *s, AVBPrint *bp)
>> +{
>> + int len, end;
>> + int64_t read = 0;
>> + char tmp[1024];
>> + char c;
>> +
>> + do {
>> + len = 0;
>> + do {
>> + char escape = c = avio_r8(s);
>> + if (c == '\\')
>> + c = avio_r8(s);
>> + end = (c == '\r' || c == '\n' || c == '\0');
>> + if (end && escape == '\\') {
>> + if (c != '\0') {
>> + tmp[len++] = c;
>> + end = 0;
>> + } else
>> + tmp[len++] = escape;
>> + } else if (!end) {
>> + if (escape == '\\') {
>> + tmp[len++] = escape;
>> + avio_skip(s, -1);
>> + } else
>> + tmp[len++] = c;
>> + }
>> + } while (!end && len < sizeof(tmp));
>> + av_bprint_append_data(bp, tmp, len);
>> + read += len;
>> + } while (!end);
>> +
>> + if (c == '\r' && avio_r8(s) != '\n' && !avio_feof(s))
>> + avio_skip(s, -1);
>> +
>> + if (!c && s->error)
>> + return s->error;
>> +
>> + if (!c && !read && avio_feof(s))
>> + return AVERROR_EOF;
>> +
>> + return read;
>> +}
>
> Re-implementing yet another de-escaping and splitting function is a
> terrible idea, I am strongly against it.
>
> It would be easier if we had a good string API already. Barring that, I
> think you need to load the whole file and use the existing de-escaping
> functions.
The existing de-escaping functions will just remove backslashes, and the
line break character will be parsed as a delimiter, which is why i wrote
the above to ensure it is read as part of the filename.
The contents of the file are meant to be taken as is, and if you want
the delimiter character to be part of the filename, you need to let the
parser know about it. Every other character in the text file doesn't
need to be escaped since this is not the command line where they could
be interpreted as something else.
>
>> +
>> +static av_cold int concatf_open(URLContext *h, const char *uri, int flags)
>> +{
>> + AVBPrint bp;
>> + struct concat_data *data = h->priv_data;
>> + struct concat_nodes *nodes = NULL;
>> + AVIOContext *in = NULL;
>> + URLContext *uc;
>> + int64_t size, total_size = 0;
>> + unsigned int nodes_size = 0;
>> + size_t i = 0;
>> + int err = 0;
>> +
>> + if (!av_strstart(uri, "concatf:", &uri)) {
>> + av_log(h, AV_LOG_ERROR, "URL %s lacks prefix\n", uri);
>> + return AVERROR(EINVAL);
>> + }
>> +
>> + /* handle input */
>> + if (!*uri)
>> + return AVERROR(ENOENT);
>> +
>> + err = ffio_open_whitelist(&in, uri, AVIO_FLAG_READ, &h->interrupt_callback,
>> + NULL, h->protocol_whitelist, h->protocol_blacklist);
>> + if (err < 0)
>> + return err;
>> +
>> + av_bprint_init(&bp, 0, AV_BPRINT_SIZE_UNLIMITED);
>> +
>> + for (i = 0; !avio_feof(in); i++) {
>> + size_t len = i;
>> +
>> + av_bprint_clear(&bp);
>> + if ((err = read_line_to_bprint(in, &bp)) <= 0) {
>> + if (err == 0 && i == 0)
>> + err = AVERROR_INVALIDDATA;
>> + else if (err == AVERROR_EOF)
>> + err = 0;
>> + break;
>> + }
>> +
>> + if (++len == SIZE_MAX / sizeof(*nodes)) {
>> + err = AVERROR(ENAMETOOLONG);
>> + break;
>> + }
>> +
>> + /* creating URLContext */
>> + err = ffurl_open_whitelist(&uc, bp.str, flags,
>> + &h->interrupt_callback, NULL, h->protocol_whitelist, h->protocol_blacklist, h);
>> + if (err < 0)
>> + break;
>> +
>> + /* creating size */
>> + if ((size = ffurl_size(uc)) < 0) {
>> + ffurl_close(uc);
>> + err = AVERROR(ENOSYS);
>> + break;
>> + }
>> +
>> + nodes = av_fast_realloc(data->nodes, &nodes_size, sizeof(*nodes) * len);
>> + if (!nodes) {
>> + ffurl_close(uc);
>> + err = AVERROR(ENOMEM);
>> + break;
>> + }
>> + data->nodes = nodes;
>> +
>> + /* assembling */
>> + data->nodes[i].uc = uc;
>> + data->nodes[i].size = size;
>> + total_size += size;
>> + }
>> + avio_closep(&in);
>> + av_bprint_finalize(&bp, NULL);
>> + data->length = i;
>> +
>> + if (err < 0)
>> + concat_close(h);
>> +
>> + data->total_size = total_size;
>> + return err;
>> +}
>> +
>> +const URLProtocol ff_concatf_protocol = {
>> + .name = "concatf",
>> + .url_open = concatf_open,
>> + .url_read = concat_read,
>> + .url_seek = concat_seek,
>> + .url_close = concat_close,
>> + .priv_data_size = sizeof(struct concat_data),
>> + .default_whitelist = "concatf,concat,file,subfile",
>> +};
>> +#endif
>> diff --git a/libavformat/protocols.c b/libavformat/protocols.c
>> index 4b6b1c8e98..7f08f151b6 100644
>> --- a/libavformat/protocols.c
>> +++ b/libavformat/protocols.c
>> @@ -27,6 +27,7 @@ extern const URLProtocol ff_async_protocol;
>> extern const URLProtocol ff_bluray_protocol;
>> extern const URLProtocol ff_cache_protocol;
>> extern const URLProtocol ff_concat_protocol;
>> +extern const URLProtocol ff_concatf_protocol;
>> extern const URLProtocol ff_crypto_protocol;
>> extern const URLProtocol ff_data_protocol;
>> extern const URLProtocol ff_ffrtmpcrypt_protocol;
>
> Regards,
>
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
>
More information about the ffmpeg-devel
mailing list