[FFmpeg-devel] [PATCH] Implement PAFF in H.264
Jeff Downs
heydowns
Thu Oct 4 21:07:09 CEST 2007
On Thu, 4 Oct 2007, Jeff Downs wrote:
> I'll continue to split as I have time, given how productive this round
> was.
>
OK, here is the next round of split patches. I took these diffs against a
tree with the defreflist and reordering patches (posted short time ago)
applied, but they will likely also apply clean to current svn.
Again, in suggested application order with commit message.
1. paff-mbaddress.patch
Modifies macroblock addressing and current macroblock y-position for
field decoding. Part of PAFF implementation.
2. paff-poc.patch
Set Picture.poc for fields and field pairs. Part of PAFF implementation.
3. paff-emuedge.patch
Edge emulation for fields. Part of PAFF implementation
4. paff-mmcodecode.patch
Augment mmcodecoding process to work properly with fields. Part of PAFF
implementation
5. paff-mmcorefrename.patch
Cosmetic renaming variable so that it makes more sense for forthcoming
PAFF mmco patches.
6. paff-mmco.patch
Augment MMCO execution to work with both fields and frames. Part of PAFF
implementation
7. paff-mmco-indent.patch
Re-indent after PAFF MMCO implementation patch.
8. paff-picmgmt.patch
Manage Picture buffers for fields as well as frames. Pair complementary
fields into one MPV Picture. Part of PAFF implementation.
After these, there are two more -- one to indent after #8 and one to
actually enable PAFF handling by properly defining FIELD_PICTURE.
Since the indent one is quite large, I'll wait until these can be looked
at to post the last two.
-Jeff
-------------- next part --------------
--- ../ffmpeg-reordering/libavcodec/h264.c 2007-10-04 10:27:13.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 14:29:25.000000000 -0400
@@ -177,7 +177,7 @@
//wow what a mess, why didn't they simplify the interlacing&intra stuff, i can't imagine that these complex rules are worth it
- top_xy = mb_xy - s->mb_stride;
+ top_xy = mb_xy - (s->mb_stride << FIELD_PICTURE);
topleft_xy = top_xy - 1;
topright_xy= top_xy + 1;
left_xy[1] = left_xy[0] = mb_xy-1;
@@ -3928,6 +3928,11 @@
}
s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width;
s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame;
+ if (FIELD_PICTURE) {
+ s->resync_mb_y = s->mb_y = s->mb_y * 2;
+ if (s->picture_structure == PICT_BOTTOM_FIELD)
+ s->resync_mb_y = s->mb_y = s->mb_y + 1;
+ }
assert(s->mb_y < s->mb_height);
if(s->picture_structure==PICT_FRAME){
@@ -4987,7 +4992,7 @@
}else{
int mb_xy = mb_x + mb_y*s->mb_stride;
mba_xy = mb_xy - 1;
- mbb_xy = mb_xy - s->mb_stride;
+ mbb_xy = mb_xy - (s->mb_stride << FIELD_PICTURE);
}
if( h->slice_table[mba_xy] == h->slice_num && !IS_SKIP( s->current_picture.mb_type[mba_xy] ))
@@ -5431,6 +5436,8 @@
if (left_mb_frame_flag != curr_mb_frame_flag) {
h->left_mb_xy[0] = pair_xy - 1;
}
+ } else if (FIELD_PICTURE) {
+ h->top_mb_xy -= s->mb_stride;
}
return;
}
@@ -6666,7 +6673,7 @@
s->mb_x = 0;
ff_draw_horiz_band(s, 16*s->mb_y, 16);
++s->mb_y;
- if(FRAME_MBAFF) {
+ if(FRAME_MBAFF || FIELD_PICTURE) {
++s->mb_y;
}
}
@@ -6703,7 +6710,7 @@
s->mb_x=0;
ff_draw_horiz_band(s, 16*s->mb_y, 16);
++s->mb_y;
- if(FRAME_MBAFF) {
+ if(FRAME_MBAFF || FIELD_PICTURE) {
++s->mb_y;
}
if(s->mb_y >= s->mb_height){
-------------- next part --------------
--- ../ffmpeg-mbaddress/libavcodec/h264.c 2007-10-04 11:14:08.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 11:21:48.000000000 -0400
@@ -3682,11 +3682,15 @@
field_poc[1]= poc;
}
- if(s->picture_structure != PICT_BOTTOM_FIELD)
+ if(s->picture_structure != PICT_BOTTOM_FIELD) {
s->current_picture_ptr->field_poc[0]= field_poc[0];
- if(s->picture_structure != PICT_TOP_FIELD)
+ s->current_picture_ptr->poc = field_poc[0];
+ }
+ if(s->picture_structure != PICT_TOP_FIELD) {
s->current_picture_ptr->field_poc[1]= field_poc[1];
- if(s->picture_structure == PICT_FRAME) // FIXME field pix?
+ s->current_picture_ptr->poc = field_poc[1];
+ }
+ if(!FIELD_PICTURE || !s->first_field)
s->current_picture_ptr->poc= FFMIN(field_poc[0], field_poc[1]);
return 0;
-------------- next part --------------
--- ../ffmpeg-poc/libavcodec/h264.c 2007-10-04 13:15:16.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 13:15:29.000000000 -0400
@@ -4094,7 +4094,7 @@
h->slice_num = ++h0->current_slice;
h->emu_edge_width= (s->flags&CODEC_FLAG_EMU_EDGE) ? 0 : 16;
- h->emu_edge_height= FRAME_MBAFF ? 0 : h->emu_edge_width;
+ h->emu_edge_height= (FRAME_MBAFF || FIELD_PICTURE) ? 0 : h->emu_edge_width;
if(s->avctx->debug&FF_DEBUG_PICT_INFO){
av_log(h->s.avctx, AV_LOG_DEBUG, "slice:%d %s mb:%d %c pps:%u frame:%d poc:%d/%d ref:%d/%d qp:%d loop:%d:%d:%d weight:%d%s\n",
-------------- next part --------------
--- ../ffmpeg-emuedge/libavcodec/h264.c 2007-10-04 14:05:30.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 14:06:17.000000000 -0400
@@ -3565,7 +3565,7 @@
h->mmco[i].opcode= opcode;
if(opcode==MMCO_SHORT2UNUSED || opcode==MMCO_SHORT2LONG){
- h->mmco[i].short_pic_num= (h->frame_num - get_ue_golomb(gb) - 1) & ((1<<h->sps.log2_max_frame_num)-1); //FIXME fields
+ h->mmco[i].short_pic_num= (h->curr_pic_num - get_ue_golomb(gb) - 1) & (h->max_pic_num - 1);
/* if(h->mmco[i].short_pic_num >= h->short_ref_count || h->short_ref[ h->mmco[i].short_pic_num ] == NULL){
av_log(s->avctx, AV_LOG_ERROR, "illegal short ref in memory management control operation %d\n", mmco);
return -1;
@@ -3573,7 +3573,7 @@
}
if(opcode==MMCO_SHORT2LONG || opcode==MMCO_LONG2UNUSED || opcode==MMCO_LONG || opcode==MMCO_SET_MAX_LONG){
unsigned int long_arg= get_ue_golomb(gb);
- if(/*h->mmco[i].long_arg >= h->long_ref_count || h->long_ref[ h->mmco[i].long_arg ] == NULL*/ long_arg >= 16){
+ if(long_arg >= 32 || (long_arg >= 16 && !(opcode == MMCO_LONG2UNUSED && FIELD_PICTURE))){
av_log(h->s.avctx, AV_LOG_ERROR, "illegal long ref in memory management control operation %d\n", opcode);
return -1;
}
@@ -3591,10 +3591,17 @@
}else{
assert(h->long_ref_count + h->short_ref_count <= h->sps.ref_frame_count);
- if(h->long_ref_count + h->short_ref_count == h->sps.ref_frame_count){ //FIXME fields
+ if(h->long_ref_count + h->short_ref_count == h->sps.ref_frame_count &&
+ !(FIELD_PICTURE && !s->first_field && s->current_picture_ptr->reference)) {
h->mmco[0].opcode= MMCO_SHORT2UNUSED;
h->mmco[0].short_pic_num= h->short_ref[ h->short_ref_count - 1 ]->frame_num;
h->mmco_index= 1;
+ if (FIELD_PICTURE) {
+ h->mmco[0].short_pic_num *= 2;
+ h->mmco[1].opcode= MMCO_SHORT2UNUSED;
+ h->mmco[1].short_pic_num= h->mmco[0].short_pic_num + 1;
+ h->mmco_index= 2;
+ }
}else
h->mmco_index= 0;
}
-------------- next part --------------
--- ../ffmpeg-mmcodecode/libavcodec/h264.c 2007-10-04 14:11:04.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 14:12:58.000000000 -0400
@@ -3458,7 +3458,7 @@
static int execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count){
MpegEncContext * const s = &h->s;
int i, j;
- int current_is_long=0;
+ int current_ref_assigned=0;
Picture *pic;
if((s->avctx->debug&FF_DEBUG_MMCO) && mmco_count==0)
@@ -3501,7 +3501,7 @@
h->long_ref[ mmco[i].long_arg ]->long_ref=1;
h->long_ref_count++;
- current_is_long=1;
+ current_ref_assigned=1;
break;
case MMCO_SET_MAX_LONG:
assert(mmco[i].long_arg <= 16);
@@ -3525,7 +3525,7 @@
}
}
- if(!current_is_long){
+ if(!current_ref_assigned){
pic= remove_short(h, s->current_picture_ptr->frame_num);
if(pic){
unreference_pic(h, pic, 0);
-------------- next part --------------
--- ../ffmpeg-mmcorefrename/libavcodec/h264.c 2007-10-04 14:14:18.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 14:16:42.000000000 -0400
@@ -3465,42 +3465,80 @@
av_log(h->s.avctx, AV_LOG_DEBUG, "no mmco here\n");
for(i=0; i<mmco_count; i++){
+ int structure, frame_num;
if(s->avctx->debug&FF_DEBUG_MMCO)
av_log(h->s.avctx, AV_LOG_DEBUG, "mmco:%d %d %d\n", h->mmco[i].opcode, h->mmco[i].short_pic_num, h->mmco[i].long_arg);
switch(mmco[i].opcode){
case MMCO_SHORT2UNUSED:
- pic= remove_short(h, mmco[i].short_pic_num);
- if(pic)
- unreference_pic(h, pic, 0);
- else if(s->avctx->debug&FF_DEBUG_MMCO)
- av_log(h->s.avctx, AV_LOG_DEBUG, "mmco: remove_short() failure\n");
+ if(s->avctx->debug&FF_DEBUG_MMCO)
+ av_log(h->s.avctx, AV_LOG_DEBUG, "mmco: unref short %d count %d\n", h->mmco[i].short_pic_num, h->short_ref_count);
+ frame_num = pic_num_extract(h, mmco[i].short_pic_num, &structure);
+ pic = find_short(h, frame_num, &j);
+ if (pic) {
+ if (unreference_pic(h, pic, structure ^ PICT_FRAME))
+ remove_short_at_index(h, j);
+ } else if(s->avctx->debug&FF_DEBUG_MMCO)
+ av_log(h->s.avctx, AV_LOG_DEBUG, "mmco: unref short failure\n");
break;
case MMCO_SHORT2LONG:
- pic= remove_long(h, mmco[i].long_arg);
- if(pic) unreference_pic(h, pic, 0);
+ if (FIELD_PICTURE && mmco[i].long_arg < h->long_ref_count &&
+ h->long_ref[mmco[i].long_arg]->frame_num ==
+ mmco[i].short_pic_num / 2) {
+ /* do nothing, we've already moved this field pair. */
+ } else {
+ int frame_num = mmco[i].short_pic_num >> FIELD_PICTURE;
+
+ pic= remove_long(h, mmco[i].long_arg);
+ if(pic) unreference_pic(h, pic, 0);
- h->long_ref[ mmco[i].long_arg ]= remove_short(h, mmco[i].short_pic_num);
+ h->long_ref[ mmco[i].long_arg ]= remove_short(h, frame_num);
if (h->long_ref[ mmco[i].long_arg ]){
h->long_ref[ mmco[i].long_arg ]->long_ref=1;
h->long_ref_count++;
}
+ }
break;
case MMCO_LONG2UNUSED:
- pic= remove_long(h, mmco[i].long_arg);
- if(pic)
- unreference_pic(h, pic, 0);
- else if(s->avctx->debug&FF_DEBUG_MMCO)
- av_log(h->s.avctx, AV_LOG_DEBUG, "mmco: remove_long() failure\n");
+ j = pic_num_extract(h, mmco[i].long_arg, &structure);
+ pic = h->long_ref[j];
+ if (pic) {
+ if (unreference_pic(h, pic, structure ^ PICT_FRAME))
+ remove_long_at_index(h, j);
+ } else if(s->avctx->debug&FF_DEBUG_MMCO)
+ av_log(h->s.avctx, AV_LOG_DEBUG, "mmco: unref long failure\n");
break;
case MMCO_LONG:
+ j = 1;
+ if (FIELD_PICTURE && !s->first_field) {
+ if (h->long_ref[mmco[i].long_arg] == s->current_picture_ptr) {
+ /* Just mark second field as referenced */
+ j = 0;
+ } else if (s->current_picture_ptr->reference) {
+ /* First field in pair is in short term list or
+ * at a different long term index.
+ * This is not allowed; see 7.4.3, notes 2 and 3.
+ * Report the problem and keep the pair where it is,
+ * and mark this field valid.
+ */
+ av_log(h->s.avctx, AV_LOG_ERROR,
+ "illegal long term reference assignment for second "
+ "field in complementary field pair (first field is "
+ "short term or has non-matching long index)\n");
+ j = 0;
+ }
+ }
+
+ if (j) {
pic= remove_long(h, mmco[i].long_arg);
if(pic) unreference_pic(h, pic, 0);
h->long_ref[ mmco[i].long_arg ]= s->current_picture_ptr;
h->long_ref[ mmco[i].long_arg ]->long_ref=1;
h->long_ref_count++;
+ }
+ s->current_picture_ptr->reference |= s->picture_structure;
current_ref_assigned=1;
break;
case MMCO_SET_MAX_LONG:
@@ -3525,6 +3563,35 @@
}
}
+ if (!current_ref_assigned && FIELD_PICTURE &&
+ !s->first_field && s->current_picture_ptr->reference) {
+
+ /* Second field of complementary field pair; the first field of
+ * which is already referenced. If short referenced, it
+ * should be first entry in short_ref. If not, it must exist
+ * in long_ref; trying to put it on the short list here is an
+ * error in the encoded bit stream (ref: 7.4.3, NOTE 2 and 3).
+ * If on neither list, we have a problem elsewhere
+ */
+ if (h->short_ref_count && h->short_ref[0] == s->current_picture_ptr) {
+ /* Just mark the second field valid */
+ s->current_picture_ptr->reference = PICT_FRAME;
+ } else if (s->current_picture_ptr->long_ref) {
+ av_log(h->s.avctx, AV_LOG_ERROR, "illegal short term reference "
+ "assignment for second field "
+ "in complementary field pair "
+ "(first field is long term)\n");
+ } else {
+ av_log(h->s.avctx, AV_LOG_ERROR, "problem in internal reference "
+ "list handling; marking second "
+ "field in pair finds first field "
+ "in reference, but not in any "
+ "ref list\n");
+ s->current_picture_ptr->reference = 0;
+ }
+ current_ref_assigned = 1;
+ }
+
if(!current_ref_assigned){
pic= remove_short(h, s->current_picture_ptr->frame_num);
if(pic){
@@ -3538,6 +3605,7 @@
h->short_ref[0]= s->current_picture_ptr;
h->short_ref[0]->long_ref=0;
h->short_ref_count++;
+ s->current_picture_ptr->reference |= s->picture_structure;
}
print_short_term(h);
-------------- next part --------------
--- ../ffmpeg-mmco/libavcodec/h264.c 2007-10-04 14:18:28.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 14:19:50.000000000 -0400
@@ -3493,10 +3493,10 @@
if(pic) unreference_pic(h, pic, 0);
h->long_ref[ mmco[i].long_arg ]= remove_short(h, frame_num);
- if (h->long_ref[ mmco[i].long_arg ]){
- h->long_ref[ mmco[i].long_arg ]->long_ref=1;
- h->long_ref_count++;
- }
+ if (h->long_ref[ mmco[i].long_arg ]){
+ h->long_ref[ mmco[i].long_arg ]->long_ref=1;
+ h->long_ref_count++;
+ }
}
break;
case MMCO_LONG2UNUSED:
@@ -3530,12 +3530,12 @@
}
if (j) {
- pic= remove_long(h, mmco[i].long_arg);
- if(pic) unreference_pic(h, pic, 0);
+ pic= remove_long(h, mmco[i].long_arg);
+ if(pic) unreference_pic(h, pic, 0);
- h->long_ref[ mmco[i].long_arg ]= s->current_picture_ptr;
- h->long_ref[ mmco[i].long_arg ]->long_ref=1;
- h->long_ref_count++;
+ h->long_ref[ mmco[i].long_arg ]= s->current_picture_ptr;
+ h->long_ref[ mmco[i].long_arg ]->long_ref=1;
+ h->long_ref_count++;
}
s->current_picture_ptr->reference |= s->picture_structure;
-------------- next part --------------
--- ../ffmpeg-mmco-indent/libavcodec/h264.c 2007-10-04 14:30:08.000000000 -0400
+++ libavcodec/h264.c 2007-10-04 14:54:45.000000000 -0400
@@ -3339,6 +3339,7 @@
idr(h);
if(h->s.current_picture_ptr)
h->s.current_picture_ptr->reference= 0;
+ h->s.first_field= 0;
}
/**
@@ -3831,6 +3832,7 @@
dst->s.current_picture = src->s.current_picture;
dst->s.linesize = src->s.linesize;
dst->s.uvlinesize = src->s.uvlinesize;
+ dst->s.first_field = src->s.first_field;
dst->prev_poc_msb = src->prev_poc_msb;
dst->prev_poc_lsb = src->prev_poc_lsb;
@@ -3858,12 +3860,14 @@
*/
static int decode_slice_header(H264Context *h, H264Context *h0){
MpegEncContext * const s = &h->s;
+ MpegEncContext * const s0 = &h0->s;
unsigned int first_mb_in_slice;
unsigned int pps_id;
int num_ref_idx_active_override_flag;
static const uint8_t slice_type_map[5]= {P_TYPE, B_TYPE, I_TYPE, SP_TYPE, SI_TYPE};
unsigned int slice_type, tmp, i;
int default_ref_list_done = 0;
+ int last_pic_structure;
s->dropable= h->nal_ref_idc == 0;
@@ -3871,6 +3875,7 @@
if((s->flags2 & CODEC_FLAG2_CHUNKS) && first_mb_in_slice == 0){
h0->current_slice = 0;
+ if (!s0->first_field)
s->current_picture_ptr= NULL;
}
@@ -3940,6 +3945,7 @@
return -1; // we cant (re-)initialize context during parallel decoding
if (MPV_common_init(s) < 0)
return -1;
+ s->first_field = 0;
init_scan_tables(h);
alloc_tables(h);
@@ -3978,6 +3984,7 @@
h->mb_mbaff = 0;
h->mb_aff_frame = 0;
+ last_pic_structure = s0->picture_structure;
if(h->sps.frame_mbs_only_flag){
s->picture_structure= PICT_FRAME;
}else{
@@ -3991,8 +3998,50 @@
}
if(h0->current_slice == 0){
- if(frame_start(h) < 0)
+ /* See if we have a decoded first field looking for a pair... */
+ if (s0->first_field) {
+ assert(s0->current_picture_ptr);
+ assert(s0->current_picture_ptr->data[0]);
+ assert(s0->current_picture_ptr->reference != DELAYED_PIC_REF);
+
+ /* figure out if we have a complementary field pair */
+ if (!FIELD_PICTURE || s->picture_structure == last_pic_structure) {
+ /*
+ * Previous field is unmatched. Don't display it, but let it
+ * remain for reference if marked as such.
+ */
+ s0->current_picture_ptr = NULL;
+ s0->first_field = FIELD_PICTURE;
+
+ } else {
+ if (h->nal_ref_idc &&
+ s0->current_picture_ptr->reference &&
+ s0->current_picture_ptr->frame_num != h->frame_num) {
+ /*
+ * This and previous field were reference, but had
+ * different frame_nums. Consider this field first in
+ * pair. Throw away previous field except for reference
+ * purposes.
+ */
+ s0->first_field = 1;
+ s0->current_picture_ptr = NULL;
+
+ } else {
+ /* Second field in complementary pair */
+ s0->first_field = 0;
+ }
+ }
+
+ } else {
+ /* Frame or first field in a potentially complementary pair */
+ assert(!s0->current_picture_ptr);
+ s0->first_field = FIELD_PICTURE;
+ }
+
+ if((!FIELD_PICTURE || s0->first_field) && frame_start(h) < 0) {
+ s0->first_field = 0;
return -1;
+ }
}
if(h != h0)
clone_slice(h, h0);
@@ -7366,6 +7415,8 @@
hx = h->thread_context[context_count - 1];
s->mb_x = hx->s.mb_x;
s->mb_y = hx->s.mb_y;
+ s->dropable = hx->s.dropable;
+ s->picture_structure = hx->s.picture_structure;
for(i = 1; i < context_count; i++)
h->s.error_count += h->thread_context[i]->s.error_count;
}
@@ -7388,6 +7439,7 @@
#endif
if(!(s->flags2 & CODEC_FLAG2_CHUNKS)){
h->current_slice = 0;
+ if (!s->first_field)
s->current_picture_ptr= NULL;
}
@@ -7685,16 +7737,34 @@
h->prev_frame_num_offset= h->frame_num_offset;
h->prev_frame_num= h->frame_num;
- if(s->current_picture_ptr->reference & s->picture_structure){
+ if(!s->dropable) {
h->prev_poc_msb= h->poc_msb;
h->prev_poc_lsb= h->poc_lsb;
execute_ref_pic_marking(h, h->mmco, h->mmco_index);
}
+ /*
+ * FIXME: Error handling code does not seem to support interlaced
+ * when slices span multiple rows
+ * The ff_er_add_slice calls don't work right for bottom
+ * fields; they cause massive erroneous error concealing
+ * Error marking covers both fields (top and bottom).
+ * This causes a mismatched s->error_count
+ * and a bad error table. Further, the error count goes to
+ * INT_MAX when called for bottom field, because mb_y is
+ * past end by one (callers fault) and resync_mb_y != 0
+ * causes problems for the first MB line, too.
+ */
+ if (!FIELD_PICTURE)
ff_er_frame_end(s);
MPV_frame_end(s);
+ if (s->first_field) {
+ /* Wait for second field. */
+ *data_size = 0;
+
+ } else {
//FIXME do something with unavailable reference frames
#if 0 //decode order
@@ -7765,6 +7835,7 @@
*pict= *(AVFrame*)out;
else
av_log(avctx, AV_LOG_DEBUG, "no picture\n");
+ }
}
assert(pict->data[0] || !*data_size);
--- ../ffmpeg-mmco-indent/libavcodec/mpegvideo.c 2007-10-04 14:23:41.000000000 -0400
+++ libavcodec/mpegvideo.c 2007-10-04 14:52:47.000000000 -0400
@@ -954,7 +954,7 @@
assert(s->pict_type == I_TYPE || (s->last_picture_ptr && s->last_picture_ptr->data[0]));
- if(s->picture_structure!=PICT_FRAME){
+ if(s->picture_structure!=PICT_FRAME && s->out_format != FMT_H264){
int i;
for(i=0; i<4; i++){
if(s->picture_structure == PICT_BOTTOM_FIELD){
More information about the ffmpeg-devel
mailing list