[FFmpeg-devel] [PATCH] samplefmt: change syntax for av_samples_alloc() and av_samples_fill_arrays()

[Stefano Sabatini]

---
 libavutil/samplefmt.c |   34 ++++++++++++++++++----------------
 libavutil/samplefmt.h |   26 ++++++++++++++++----------
 2 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/libavutil/samplefmt.c b/libavutil/samplefmt.c
index ea004d9..346e44e 100644
--- a/libavutil/samplefmt.c
+++ b/libavutil/samplefmt.c
@@ -73,31 +73,29 @@ int av_get_bits_per_sample_fmt(enum AVSampleFormat sample_fmt)
 }
 
 int av_samples_fill_arrays(uint8_t *pointers[8], int linesizes[8],
-                           uint8_t *buf, int nb_channels, int nb_samples,
-                           enum AVSampleFormat sample_fmt, int planar, int align)
+                           uint8_t *buf, int linesize0, int nb_channels,
+                           enum AVSampleFormat sample_fmt, int planar)
 {
     int i, step_size = 0;
-    int sample_size = av_get_bits_per_sample_fmt(sample_fmt) >> 3;
-    int channel_step = planar ? FFALIGN(nb_samples*sample_size, align) : sample_size;
-
-    if(nb_channels * (uint64_t)nb_samples * sample_size >= INT_MAX - align*(uint64_t)nb_channels)
-        return AVERROR(EINVAL);
 
     if (pointers) {
         pointers[0] = buf;
         for (i = 0; i < nb_channels; i++) {
             pointers[i] = buf + step_size;
-            step_size += channel_step;
+            if (planar)
+                step_size += linesize0;
         }
         memset(&pointers[nb_channels], 0, (8-nb_channels) * sizeof(pointers[0]));
     }
 
     if (linesizes) {
-        linesizes[0] = planar ?  sample_size : nb_channels*sample_size;
-        memset(&linesizes[1], 0, (8-1) * sizeof(linesizes[0]));
+        linesizes[0] = linesize0;
+        for (i = 1; planar && pointers && i < nb_channels; i++)
+            linesizes[i] = pointers[i] - pointers[0];
+        memset(&linesizes[i], 0, (8-i) * sizeof(linesizes[0]));
     }
 
-    return planar ? channel_step * nb_channels : FFALIGN(nb_channels*sample_size*nb_samples, align);
+    return planar ? linesize0 * nb_channels : linesize0;
 }
 
 int av_samples_alloc(uint8_t *pointers[8], int linesizes[8],
@@ -106,15 +104,19 @@ int av_samples_alloc(uint8_t *pointers[8], int linesizes[8],
                      int align)
 {
     uint8_t *buf;
-    int size = av_samples_fill_arrays(NULL, NULL,
-                                      NULL, nb_channels, nb_samples,
-                                      sample_fmt, planar, align);
+    int size;
+    int sample_size = av_get_bits_per_sample_fmt(sample_fmt) >> 3;
+    int linesize0 = planar ? sample_size * nb_samples :
+                             sample_size * nb_samples * nb_channels; /* packed */
 
+    linesize0 = FFALIGN(linesize0, align);
+    size = planar ? FFALIGN(nb_channels * linesize0, align) :
+                    FFALIGN(linesize0, align);    /* packed */
     buf = av_mallocz(size);
     if (!buf)
         return AVERROR(ENOMEM);
 
     return av_samples_fill_arrays(pointers, linesizes,
-                                  buf, nb_channels, nb_samples,
-                                  sample_fmt, planar, align);
+                                  buf, linesize0, nb_channels,
+                                  sample_fmt, planar);
 }
diff --git a/libavutil/samplefmt.h b/libavutil/samplefmt.h
index 9b9c0d4..bb2793f 100644
--- a/libavutil/samplefmt.h
+++ b/libavutil/samplefmt.h
@@ -74,21 +74,27 @@ int av_get_bits_per_sample_fmt(enum AVSampleFormat sample_fmt);
  * format sample_fmt.
  *
  * The pointers array is filled with the pointers to the samples data:
- * data[c] points to the first sample of channel c.
- * data[c] + linesize[0] points to the second sample of channel c
+ * for planar, set the start point of each plane's data within the buffer,
+ * for packed, set the start point of the entire buffer only.
+ *
+ * linesize0 specifies the aligned size for each data plane.
+ * linesize[0] is set to linesize0, if the format is planar and i>0
+ * linesize[i] is set to data[i] - data[0] for all the other planes.
  *
  * @param pointers array to be filled with the pointer for each plane, may be NULL
- * @param linesizes array to be filled with the linesize, may be NULL
+ * @param linesizes array to be filled with the linesize for each plane, may be NULL
  * @param buf the pointer to a buffer containing the samples
- * @param nb_samples the number of samples in a single channel
- * @param planar 1 if the samples layout is planar, 0 if it is packed
+ * @param linesize0 the size of linesizes[0], usually the aligned
+ * size of each plane buffer
+ * @param planar >0 if the samples layout is planar, 0 if it is packed
  * @param nb_channels the number of channels
- * @return the total size of the buffer, a negative
- * error code in case of failure
+ * @return the total size of the buffer, a negative error code in case
+ * of failure
  */
 int av_samples_fill_arrays(uint8_t *pointers[8], int linesizes[8],
-                           uint8_t *buf, int nb_channels, int nb_samples,
-                           enum AVSampleFormat sample_fmt, int planar, int align);
+                           uint8_t *buf, int linesize0,
+                           enum AVSampleFormat sample_fmt, int planar,
+                           int nb_channels);
 
 /**
  * Allocate a samples buffer for nb_samples samples, and
@@ -97,7 +103,7 @@ int av_samples_fill_arrays(uint8_t *pointers[8], int linesizes[8],
  * av_freep(&pointers[0]).
  *
  * @param nb_samples number of samples per channel
- * @param planar 1 if the samples layout is planar, 0 if packed,
+ * @param planar >0 if the samples layout is planar, packed otherwise
  * @param align the value to use for buffer size alignment
  * @return the size in bytes required for the samples buffer, a negative
  * error code in case of failure
-- 
1.7.2.3