- NUT-devel - lists.mplayerhq.hu

[NUT] (ods15): r102 - /trunk/nututils/nutindex.c
by syncmail＠mplayerhq.hu 12 Mar '06

12 Mar '06

Author: ods15 Date: Sun Mar 12 06:07:10 2006 New Revision: 102 Log: some more memory optimizations to nutindex Modified: trunk/nututils/nutindex.c Modified: trunk/nututils/nutindex.c ============================================================================== --- trunk/nututils/nutindex.c (original) +++ trunk/nututils/nutindex.c Sun Mar 12 06:07:10 2006 @@ -72,17 +72,16 @@ } } -static output_buffer_t * new_mem_buffer() { - output_buffer_t * bc = malloc(sizeof(output_buffer_t)); - bc->write_len = PREALLOC_SIZE; +static output_buffer_t * new_mem_buffer(output_buffer_t * bc) { + bc->write_len = 0; bc->is_mem = 1; bc->file_pos = 0; - bc->buf_ptr = bc->buf = malloc(bc->write_len); + bc->buf_ptr = bc->buf = NULL; return bc; } -static output_buffer_t * new_output_buffer(FILE * out) { - output_buffer_t * bc = new_mem_buffer(); +static output_buffer_t * new_output_buffer(output_buffer_t * bc, FILE * out) { + new_mem_buffer(bc); bc->is_mem = 0; bc->out = out; return bc; @@ -105,7 +104,7 @@ } static void put_data(output_buffer_t * bc, int len, const uint8_t * data) { - if (len < PREALLOC_SIZE || bc->is_mem) { + if (len + (bc->buf_ptr - bc->buf) < bc->write_len || bc->is_mem) { ready_write_buf(bc, len); memcpy(bc->buf_ptr, data, len); bc->buf_ptr += len; @@ -119,7 +118,6 @@ if (!bc) return; if (!bc->is_mem) flush_out_buf(bc); free(bc->buf); - free(bc); } // ============ INPUT BUFFER ============== @@ -164,8 +162,7 @@ if (whence == SEEK_END) bc->filesize = bc->file_pos - pos; } -static input_buffer_t * new_input_buffer(FILE * in) { - input_buffer_t * bc = malloc(sizeof(input_buffer_t)); +static input_buffer_t * new_input_buffer(input_buffer_t * bc, FILE * in) { bc->read_len = 0; bc->write_len = 0; bc->file_pos = 0; @@ -184,7 +181,6 @@ static void free_buffer(input_buffer_t * bc) { if (!bc) return; free(bc->buf); - free(bc); } static int get_bytes(input_buffer_t * bc, int count, uint64_t * val) { @@ -296,6 +292,8 @@ (v_len(forward_ptr + new_idx_len-idx_len) - v_len(forward_ptr)) > new_idx_len - idx_len) new_idx_len += 16; + ready_write_buf(out, new_idx_len); // prealloc + put_bytes(out, 8, INDEX_STARTCODE); put_v(out, forward_ptr + new_idx_len-idx_len); if (forward_ptr + new_idx_len-idx_len > 4096) put_bytes(out, 4, crc32(out->buf, bctello(out))); @@ -310,7 +308,6 @@ printf("%d => %d (%d)\n", (int)tmp, (int)(tmp + new_idx_len/16), new_idx_len); idx_len = (in->filesize - 12) - bctello(in); // from where we are, until the index_ptr and checksum, copy everything - ready_write_buf(out, idx_len); if (get_data(in, idx_len, out->buf_ptr)) return 1; out->buf_ptr += idx_len; @@ -323,22 +320,26 @@ int main(int argc, char * argv[]) { FILE * fin = argc>2 ? fopen(argv[1], "rb") : NULL; FILE * fout = argc>2 ? fopen(argv[2], "wb") : NULL; - input_buffer_t * in; - output_buffer_t * out, * mem = new_mem_buffer(); + input_buffer_t iin, * in; + output_buffer_t oout, * out, omem, * mem = new_mem_buffer(&omem); off_t end; if (!fin || !fout) { fprintf(stderr, "bleh\n"); return 1; } - in = new_input_buffer(fin); - out = new_output_buffer(fout); + in = new_input_buffer(&iin, fin); + out = new_output_buffer(&oout, fout); CHECK(find_copy_index(in, mem, &end)); seek_buf(in, 0, SEEK_SET); CHECK(read_headers(in)); + printf("headers: %d\n", (int)bctello(in)); + put_data(out, bctello(in), in->buf); // write headers put_data(out, bctello(mem), mem->buf); // write index + + flush_buf(in); // copy all data while (bctello(in) < end) {

1 0

[NUT] (ods15): r101 - /trunk/demux_nut.c
by syncmail＠mplayerhq.hu 12 Mar '06

12 Mar '06

Author: ods15 Date: Sun Mar 12 05:15:39 2006 New Revision: 101 Log: fix mplayer demux_nut Modified: trunk/demux_nut.c Modified: trunk/demux_nut.c ============================================================================== --- trunk/demux_nut.c (original) +++ trunk/demux_nut.c Sun Mar 12 05:15:39 2006 @@ -205,7 +205,7 @@ dp->pts = pts; dp->pos = demuxer->filepos; - dp->flags= (pd.flags & NUT_KEY_STREAM_FLAG) ? 0x10 : 0; + dp->flags= (pd.flags & NUT_FLAG_KEY) ? 0x10 : 0; while ((ret = nut_read_frame(nut, &pd.len, dp->buffer))) { if (ret < 0) { @@ -218,7 +218,7 @@ return 1; } -static void demux_seek_nut(demuxer_t * demuxer, float time_pos, int flags) { +static void demux_seek_nut(demuxer_t * demuxer, float time_pos, float audio_delay, int flags) { nut_context_t * nut = ((nut_priv_t*)demuxer->priv)->nut; nut_priv_t * priv = demuxer->priv; sh_audio_t * sh_audio = demuxer->audio->sh;

1 0

max pts/distance stuff
by Michael Niedermayer 11 Mar '06

11 Mar '06

Hi currenbtly we have " max_distance max distance of syncpoints, the distance may only be larger if there is no more than a single frame between the two syncpoints. This can be used by the demuxer to detect damaged frame headers if the damage results in too long of a chain syncpoints SHOULD be placed immediately before a keyframe if the previous frame of the same stream was a non-keyframe, unless such non-keyframe - keyframe transitions are very frequent SHOULD be set to <=32768 if the stored value is >65536 then max_distance MUST be set to 65536 max_pts_distance max absoloute difference of pts of new frame from last_pts in the timebase of the stream, without a checksum after the frameheader Note that last_pts is not necessarily the pts of the last frame on the same stream, as it is altered by syncpoint timestamps. max_frame_size max frame size without a checksum after the frameheader. MUST be <=4*max_distance. If stored value is bigger then max_frame_size MUST be set to 4*max_distance " in the spec, there are several issues which need disscussion 1. no limit for max_pts_distance, users could set it to insane values and cause troubble on the demuxer side, yes theres the time rescaling issue 2. max_frame_size limit is too large, its practically 256k, i think 64k or less is more reasonable 3. what about droping one varaible and max_distance=max_frame_size? -- Michael

3 6

[NUT] (ods15): r100 - in /trunk/libnut: demuxer.c muxer.c
by syncmail＠mplayerhq.hu 11 Mar '06

11 Mar '06

Author: ods15 Date: Sat Mar 11 20:49:07 2006 New Revision: 100 Log: sync to spec, 128kb for max_distance instead of 64kb Modified: trunk/libnut/demuxer.c trunk/libnut/muxer.c Modified: trunk/libnut/demuxer.c ============================================================================== --- trunk/libnut/demuxer.c (original) +++ trunk/libnut/demuxer.c Sat Mar 11 20:49:07 2006 @@ -236,7 +236,7 @@ ERROR(i != NUT_VERSION, -ERR_BAD_VERSION); GET_V(tmp, nut->stream_count); GET_V(tmp, nut->max_distance); - if (nut->max_distance > 65536) nut->max_distance = 65536; + if (nut->max_distance > 131072) nut->max_distance = 131072; for(i = 0; i < 256; ) { int scrap; Modified: trunk/libnut/muxer.c ============================================================================== --- trunk/libnut/muxer.c (original) +++ trunk/libnut/muxer.c Sat Mar 11 20:49:07 2006 @@ -488,7 +488,7 @@ nut->max_distance = mopts->max_distance; nut->mopts = *mopts; - if (nut->max_distance > 65536) nut->max_distance = 65536; + if (nut->max_distance > 131072) nut->max_distance = 131072; { int j, n;

1 0

[NUT] (ods15): r99 - /trunk/README
by syncmail＠mplayerhq.hu 11 Mar '06

11 Mar '06

Author: ods15 Date: Sat Mar 11 20:26:24 2006 New Revision: 99 Log: document nutindex Modified: trunk/README Modified: trunk/README ============================================================================== --- trunk/README (original) +++ trunk/README Sat Mar 11 20:26:24 2006 @@ -34,6 +34,10 @@ auto detction should work fine. +to copy index to begginning of file: +./nutindex old.nut new.nut + + If you have weird endianess, You'll need to compile with '-DWORDS_BIGENDIAN', just uncomment the appropriate file in config.mak .

1 0

[NUT] (ods15): r98 - /trunk/nututils/nutindex.c
by syncmail＠mplayerhq.hu 11 Mar '06

11 Mar '06

Author: ods15 Date: Sat Mar 11 20:24:03 2006 New Revision: 98 Log: nutindex: sync to spec, /16 not /8 forward_ptr have their own checksum optimizations in memory Modified: trunk/nututils/nutindex.c Modified: trunk/nututils/nutindex.c ============================================================================== --- trunk/nututils/nutindex.c (original) +++ trunk/nututils/nutindex.c Sat Mar 11 20:24:03 2006 @@ -212,36 +212,49 @@ } static int get_data(input_buffer_t * bc, int len, uint8_t * buf) { - int tmp; - tmp = ready_read_buf(bc, len); - len = MIN(len, tmp); - memcpy(buf, bc->buf_ptr, len); - bc->buf_ptr += len; + int tmp = MIN(len, bc->read_len - (bc->buf_ptr - bc->buf)); + if (tmp) { + memcpy(buf, bc->buf_ptr, tmp); + bc->buf_ptr += tmp; + len -= tmp; + } + if (len) { + int read = fread(buf + tmp, 1, len, bc->in); + bc->file_pos += read; + len -= read; + } + flush_buf(bc); return len; } #define CHECK(expr) do { int _a; if ((_a = (expr))) return _a; } while(0) #define GET_V(bc, v) do { uint64_t _tmp; CHECK(get_v((bc), &_tmp)); (v) = _tmp; } while(0) +static int get_header(input_buffer_t * in) { + off_t start = bctello(in) - 8; // startcode + int forward_ptr; + + GET_V(in, forward_ptr); + if (forward_ptr > 4096) { + CHECK(skip_buffer(in, 4)); // header_checksum + if (crc32(in->buf_ptr - (bctello(in) - start), bctello(in) - start)) return 3; + } + + CHECK(skip_buffer(in, forward_ptr)); + if (crc32(in->buf_ptr - forward_ptr, forward_ptr)) return 3; + return 0; +} + static int read_headers(input_buffer_t * in) { int len = strlen(ID_STRING) + 1; - char str[len]; uint64_t tmp; - if (get_data(in, len, str) != len) return 1; - if (memcmp(str, ID_STRING, len)) return 1; + assert(in->buf == in->buf_ptr); + if (ready_read_buf(in, len) < len) return 1; + if (memcmp(in->buf_ptr, ID_STRING, len)) return 1; + in->buf_ptr += len; CHECK(get_bytes(in, 8, &tmp)); - if (tmp != MAIN_STARTCODE) return 1; - GET_V(in, len); - CHECK(skip_buffer(in, len)); - CHECK(get_bytes(in, 8, &tmp)); - while (tmp == STREAM_STARTCODE) { - GET_V(in, len); - CHECK(skip_buffer(in, len)); - CHECK(get_bytes(in, 8, &tmp)); - } - while (tmp == INFO_STARTCODE) { - GET_V(in, len); - CHECK(skip_buffer(in, len)); + while (tmp != INDEX_STARTCODE && tmp != SYNCPOINT_STARTCODE) { + CHECK(get_header(in)); CHECK(get_bytes(in, 8, &tmp)); } if (tmp == INDEX_STARTCODE) return 2; @@ -252,6 +265,7 @@ static int find_copy_index(input_buffer_t * in, output_buffer_t * out, off_t * end) { uint64_t tmp; uint64_t idx_len; + uint64_t max_pts, syncpoints; int new_idx_len, forward_ptr; int i, padding; assert(out->is_mem); @@ -265,47 +279,45 @@ if (tmp != INDEX_STARTCODE) return 2; GET_V(in, forward_ptr); - - GET_V(in, tmp); put_v(out, tmp); // max_pts - GET_V(in, tmp); put_v(out, tmp); // syncpoints + if (forward_ptr > 4) { // checksum + skip_buffer(in, 4); + if (crc32(in->buf, in->buf_ptr - in->buf)) return 4; + } + + GET_V(in, max_pts); + GET_V(in, syncpoints); GET_V(in, tmp); // first syncpoint position - new_idx_len = ((idx_len + 7)/8)*8; + new_idx_len = ((idx_len + 15)/16)*16; // paranoia, this should either iterate once or not at all - while ( (v_len(tmp + new_idx_len/8 ) - v_len(tmp )) + + while ( (v_len(tmp + new_idx_len/16 ) - v_len(tmp )) + (v_len(forward_ptr + new_idx_len-idx_len) - v_len(forward_ptr)) - > new_idx_len - idx_len) new_idx_len += 8; + > new_idx_len - idx_len) new_idx_len += 16; + + put_bytes(out, 8, INDEX_STARTCODE); + put_v(out, forward_ptr + new_idx_len-idx_len); + if (forward_ptr + new_idx_len-idx_len > 4096) put_bytes(out, 4, crc32(out->buf, bctello(out))); + put_v(out, max_pts); + put_v(out, syncpoints); padding = new_idx_len - idx_len; - padding -= (v_len(tmp + new_idx_len/8) - v_len(tmp)); + padding -= (v_len(tmp + new_idx_len/16) - v_len(tmp)); padding -= (v_len(forward_ptr + new_idx_len-idx_len) - v_len(forward_ptr)); // extreme paranoia, this will be zero for (i = 0; i < padding; i++) put_bytes(out, 1, 0x80); - put_v(out, tmp + new_idx_len/8); // the new first syncpoint position - printf("%d => %d (%d)\n", (int)tmp, (int)(tmp + new_idx_len/8), new_idx_len); + put_v(out, tmp + new_idx_len/16); // the new first syncpoint position + printf("%d => %d (%d)\n", (int)tmp, (int)(tmp + new_idx_len/16), new_idx_len); idx_len = (in->filesize - 12) - bctello(in); // from where we are, until the index_ptr and checksum, copy everything - if (ready_read_buf(in, idx_len) < idx_len) return 1; ready_write_buf(out, idx_len); - memcpy(out->buf_ptr, in->buf_ptr, idx_len); + if (get_data(in, idx_len, out->buf_ptr)) return 1; out->buf_ptr += idx_len; - in->buf_ptr += idx_len; - - forward_ptr = out->buf_ptr - out->buf; - forward_ptr += 4 + 8; // checksum and index_ptr - put_bytes(out, 8, 8 + v_len(forward_ptr) + forward_ptr); // index_ptr - put_bytes(out, 4, crc32(out->buf, out->buf_ptr - out->buf)); // checksum - - return 0; -} - -static void put_index(output_buffer_t * bc, output_buffer_t * tmp) { - assert(tmp->is_mem); - - put_bytes(bc, 8, INDEX_STARTCODE); - put_v(bc, tmp->buf_ptr - tmp->buf); // forward_ptr - put_data(bc, tmp->buf_ptr - tmp->buf, tmp->buf); + + put_bytes(out, 8, new_idx_len); // index_ptr + put_bytes(out, 4, crc32(out->buf, bctello(out))); // checksum + + return 0; } int main(int argc, char * argv[]) { @@ -325,8 +337,8 @@ seek_buf(in, 0, SEEK_SET); CHECK(read_headers(in)); - put_data(out, bctello(in), in->buf); - put_index(out, mem); + put_data(out, bctello(in), in->buf); // write headers + put_data(out, bctello(mem), mem->buf); // write index // copy all data while (bctello(in) < end) { @@ -337,7 +349,7 @@ flush_buf(in); printf("%d\r", (int)bctello(in)); } - put_index(out, mem); + put_data(out, bctello(mem), mem->buf); // write index free_buffer(in); free_out_buffer(out);

1 0

[NUT] (ods15): r97 - in /trunk/libnut: demuxer.c priv.h
by syncmail＠mplayerhq.hu 11 Mar '06

11 Mar '06

Author: ods15 Date: Sat Mar 11 20:04:14 2006 New Revision: 97 Log: more verbose errors Modified: trunk/libnut/demuxer.c trunk/libnut/priv.h Modified: trunk/libnut/demuxer.c ============================================================================== --- trunk/libnut/demuxer.c (original) +++ trunk/libnut/demuxer.c Sat Mar 11 20:04:14 2006 @@ -552,7 +552,7 @@ } // error checking - max distance - ERROR(!after_sync && bctello(nut->i) + pd->len - nut->last_syncpoint > nut->max_distance, -ERR_MAX_DISTANCE); + ERROR(!after_sync && bctello(nut->i) + pd->len - nut->last_syncpoint > nut->max_distance, -ERR_MAX_SYNCPOINT_DISTANCE); ERROR(!checksum && pd->len > nut->max_distance, -ERR_MAX_DISTANCE); // error checking - max pts distance ERROR(!checksum && ABS((int64_t)pd->pts - (int64_t)nut->sc[pd->stream].last_pts) > nut->sc[pd->stream].max_pts_distance, -ERR_MAX_PTS_DISTANCE); @@ -1273,6 +1273,7 @@ case ERR_BAD_VERSION: return "Bad NUT Version."; case ERR_NOT_FRAME_NOT_N: return "Invalid Framecode."; case ERR_BAD_CHECKSUM: return "Bad Checksum."; + case ERR_MAX_SYNCPOINT_DISTANCE: return "max_distance syncpoint"; case ERR_MAX_DISTANCE: return "max_distance"; case ERR_NO_HEADERS: return "No headers found!"; case ERR_NOT_SEEKABLE: return "Cannot seek to that position."; @@ -1281,6 +1282,7 @@ case ERR_BAD_STREAM_ORDER: return "Stream headers are stored in wrong order."; case ERR_NOSTREAM_STARTCODE: return "Expected stream startcode not found."; case ERR_BAD_EOF: return "Invalid forward_ptr!"; + case ERR_VLC_TOO_LONG: return "VLC too long"; } return NULL; } Modified: trunk/libnut/priv.h ============================================================================== --- trunk/libnut/priv.h (original) +++ trunk/libnut/priv.h Sat Mar 11 20:04:14 2006 @@ -38,6 +38,7 @@ ERR_BAD_VERSION, ERR_NOT_FRAME_NOT_N, ERR_BAD_CHECKSUM, + ERR_MAX_SYNCPOINT_DISTANCE, ERR_MAX_DISTANCE, ERR_NO_HEADERS, ERR_NOT_SEEKABLE,

1 0

[NUT] (ods15): r96 - in /trunk/libnut: demuxer.c priv.h
by syncmail＠mplayerhq.hu 11 Mar '06

11 Mar '06

Author: ods15 Date: Sat Mar 11 19:16:18 2006 New Revision: 96 Log: sync to spec, unlimited stuffing not allowed when in "raw" (not tmp buffer) mode Modified: trunk/libnut/demuxer.c trunk/libnut/priv.h Modified: trunk/libnut/demuxer.c ============================================================================== --- trunk/libnut/demuxer.c (original) +++ trunk/libnut/demuxer.c Sat Mar 11 19:16:18 2006 @@ -120,15 +120,23 @@ static int get_v(input_buffer_t * bc, uint64_t * val) { int i, len; *val = 0; + do { len = ready_read_buf(bc, 16); - for (i = 0; i < len; i++) { - uint8_t tmp= *(bc->buf_ptr++); - *val = (*val << 7) | (tmp & 0x7F); - if (!(tmp & 0x80)) return 0; - } + for (i = 0; i < len; i++) if (*bc->buf_ptr++ != 0x80) break; + + if (i == len) { if (len >= 16 && !bc->is_mem) return -ERR_VLC_TOO_LONG; } + else { bc->buf_ptr--; break; } } while (len >= 16); - return buf_eof(bc); + + len = ready_read_buf(bc, 16); + for (i = 0; i < len; i++) { + uint8_t tmp= *(bc->buf_ptr++); + *val = (*val << 7) | (tmp & 0x7F); + if (!(tmp & 0x80)) return 0; + } + if (len >= 16) return -ERR_VLC_TOO_LONG; + else return buf_eof(bc); } static int get_s(input_buffer_t * bc, int64_t * val) { Modified: trunk/libnut/priv.h ============================================================================== --- trunk/libnut/priv.h (original) +++ trunk/libnut/priv.h Sat Mar 11 19:16:18 2006 @@ -35,17 +35,18 @@ enum errors { ERR_GENERAL_ERROR = 1, - ERR_BAD_VERSION = 2, - ERR_NOT_FRAME_NOT_N = 3, - ERR_BAD_CHECKSUM = 4, - ERR_MAX_DISTANCE = 5, - ERR_NO_HEADERS = 6, - ERR_NOT_SEEKABLE = 7, - ERR_OUT_OF_ORDER = 8, - ERR_MAX_PTS_DISTANCE = 9, - ERR_BAD_STREAM_ORDER = 11, - ERR_NOSTREAM_STARTCODE = 12, - ERR_BAD_EOF = 13, + ERR_BAD_VERSION, + ERR_NOT_FRAME_NOT_N, + ERR_BAD_CHECKSUM, + ERR_MAX_DISTANCE, + ERR_NO_HEADERS, + ERR_NOT_SEEKABLE, + ERR_OUT_OF_ORDER, + ERR_MAX_PTS_DISTANCE, + ERR_VLC_TOO_LONG, + ERR_BAD_STREAM_ORDER, + ERR_NOSTREAM_STARTCODE, + ERR_BAD_EOF, }; typedef struct {

1 0

Split index per stream?
by Michael Niedermayer 11 Mar '06

11 Mar '06

Hi one index for syncpoint pos, one for each stream? comments? -- Michael

2 6

[NUT] (ods15): r95 - /trunk/libnut/demuxer.c
by syncmail＠mplayerhq.hu 11 Mar '06

11 Mar '06

Author: ods15 Date: Sat Mar 11 18:13:42 2006 New Revision: 95 Log: Add extendibility, there can be new headers anywhere Modified: trunk/libnut/demuxer.c Modified: trunk/libnut/demuxer.c ============================================================================== --- trunk/libnut/demuxer.c (original) +++ trunk/libnut/demuxer.c Sat Mar 11 18:13:42 2006 @@ -708,7 +708,11 @@ uint64_t tmp; int j; CHECK(get_bytes(nut->i, 8, &tmp)); - if (tmp != STREAM_STARTCODE) return -ERR_NOSTREAM_STARTCODE; + while (tmp != STREAM_STARTCODE) { + ERROR(tmp >> 56 != 'N', -ERR_NOSTREAM_STARTCODE); + CHECK(get_header(nut->i, NULL)); + CHECK(get_bytes(nut->i, 8, &tmp)); + } CHECK(get_stream_header(nut, i)); if (!nut->sc[i].pts_cache) { nut->sc[i].pts_cache = malloc(nut->sc[i].sh.decode_delay * sizeof(int64_t)); @@ -719,6 +723,11 @@ if (nut->dopts.read_index) { uint64_t tmp; CHECK(get_bytes(nut->i, 8, &tmp)); + while (tmp >> 56 == 'N') { + if (tmp == INDEX_STARTCODE) break; + CHECK(get_header(nut->i, NULL)); + CHECK(get_bytes(nut->i, 8, &tmp)); + } if (tmp == INDEX_STARTCODE) nut->seek_status = 2; nut->i->buf_ptr -= 8; flush_buf(nut->i);

1 0