[FFmpeg-devel] [PATCH 3/5] lavfi: add ICC profile support via lcms2
Niklas Haas
ffmpeg at haasn.xyz
Mon Apr 11 18:36:52 EEST 2022
From: Niklas Haas <git at haasn.dev>
This introduces an optional dependency on lcms2 into FFmpeg. lcms2 is a
widely used library for ICC profile handling, which apart from being
used in almost all major image processing programs and video players,
has also been deployed in browsers. As such, it's both widely available
and well-tested.
Add a few helpers to cover our major use cases. This commit merely
introduces the helpers (and configure check), even though nothing uses
them yet.
It's worth pointing out that the reason the cmsToneCurves for each
AVCOL_TRC are cached inside the context, is because constructing a
cmsToneCurve requires evaluating the curve at 4096 (by default) grid
points and constructing a LUT. So, we ideally only want to do this once
per curve. This matters for e.g. ff_icc_profile_detect_transfer, which
essentially compares a profile against all of these generated LUTs.
Re-generating the LUTs for every iteration would be unnecessarily
wasteful.
The same consideration does not apply to e.g. cmsCreate*Profile, which
is a very lightweight operation just involving struct allocation and
setting a few pointers.
The cutoff value of 0.01 was determined by experimentation. The lowest
"false positive" delta I saw in practice was 0.13, and the largest
"false negative" delta was 0.0008. So a value of 0.01 sits comfortaby
almost exactly in the middle.
Signed-off-by: Niklas Haas <git at haasn.dev>
---
configure | 3 +
libavfilter/fflcms2.c | 311 ++++++++++++++++++++++++++++++++++++++++++
libavfilter/fflcms2.h | 87 ++++++++++++
3 files changed, 401 insertions(+)
create mode 100644 libavfilter/fflcms2.c
create mode 100644 libavfilter/fflcms2.h
diff --git a/configure b/configure
index 9c8965852b..1a9c3dcd3c 100755
--- a/configure
+++ b/configure
@@ -215,6 +215,7 @@ External library support:
--disable-iconv disable iconv [autodetect]
--enable-jni enable JNI support [no]
--enable-ladspa enable LADSPA audio filtering [no]
+ --enable-lcms2 enable ICC profile support via LittleCMS 2 [no]
--enable-libaom enable AV1 video encoding/decoding via libaom [no]
--enable-libaribb24 enable ARIB text and caption decoding via libaribb24 [no]
--enable-libass enable libass subtitles rendering,
@@ -1813,6 +1814,7 @@ EXTERNAL_LIBRARY_LIST="
gnutls
jni
ladspa
+ lcms2
libaom
libass
libbluray
@@ -6504,6 +6506,7 @@ enabled gmp && require gmp gmp.h mpz_export -lgmp
enabled gnutls && require_pkg_config gnutls gnutls gnutls/gnutls.h gnutls_global_init
enabled jni && { [ $target_os = "android" ] && check_headers jni.h && enabled pthreads || die "ERROR: jni not found"; }
enabled ladspa && require_headers "ladspa.h dlfcn.h"
+enabled lcms2 && require_pkg_config lcms2 "lcms2 >= 2.13" lcms2.h cmsCreateContext
enabled libaom && require_pkg_config libaom "aom >= 1.0.0" aom/aom_codec.h aom_codec_version
enabled libaribb24 && { check_pkg_config libaribb24 "aribb24 > 1.0.3" "aribb24/aribb24.h" arib_instance_new ||
{ enabled gpl && require_pkg_config libaribb24 aribb24 "aribb24/aribb24.h" arib_instance_new; } ||
diff --git a/libavfilter/fflcms2.c b/libavfilter/fflcms2.c
new file mode 100644
index 0000000000..afde5d87a8
--- /dev/null
+++ b/libavfilter/fflcms2.c
@@ -0,0 +1,311 @@
+/*
+ * Copyright (c) 2022 Niklas Haas
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/color_utils.h"
+
+#include "fflcms2.h"
+
+static void log_cb(cmsContext ctx, cmsUInt32Number error, const char *str)
+{
+ FFIccContext *s = cmsGetContextUserData(ctx);
+ av_log(s->avctx, AV_LOG_ERROR, "lcms2: [%"PRIu32"] %s\n", error, str);
+}
+
+int ff_icc_context_init(FFIccContext *s, void *avctx)
+{
+ memset(s, 0, sizeof(*s));
+ s->avctx = avctx;
+ s->ctx = cmsCreateContext(NULL, s);
+ if (!s->ctx)
+ return AVERROR(ENOMEM);
+
+ cmsSetLogErrorHandlerTHR(s->ctx, log_cb);
+ return 0;
+}
+
+void ff_icc_context_uninit(FFIccContext *s)
+{
+ for (int i = 0; i < FF_ARRAY_ELEMS(s->curves); i++)
+ cmsFreeToneCurve(s->curves[i]);
+ cmsDeleteContext(s->ctx);
+ memset(s, 0, sizeof(*s));
+}
+
+static int get_curve(FFIccContext *s, enum AVColorTransferCharacteristic trc,
+ cmsToneCurve **out_curve)
+{
+ if (trc >= AVCOL_TRC_NB)
+ return AVERROR_INVALIDDATA;
+
+ if (s->curves[trc])
+ goto done;
+
+ switch (trc) {
+ case AVCOL_TRC_LINEAR:
+ s->curves[trc] = cmsBuildGamma(s->ctx, 1.0);
+ break;
+ case AVCOL_TRC_GAMMA22:
+ s->curves[trc] = cmsBuildGamma(s->ctx, 2.2);
+ break;
+ case AVCOL_TRC_GAMMA28:
+ s->curves[trc] = cmsBuildGamma(s->ctx, 2.8);
+ break;
+ case AVCOL_TRC_BT709:
+ case AVCOL_TRC_SMPTE170M:
+ case AVCOL_TRC_BT2020_10:
+ case AVCOL_TRC_BT2020_12:
+ s->curves[trc] = cmsBuildParametricToneCurve(s->ctx, 4, (double[5]) {
+ /* γ = */ 1/0.45,
+ /* a = */ 1/1.099296826809442,
+ /* b = */ 1 - 1/1.099296826809442,
+ /* c = */ 1/4.5,
+ /* d = */ 4.5 * 0.018053968510807,
+ });
+ break;
+ case AVCOL_TRC_SMPTE240M:
+ s->curves[trc] = cmsBuildParametricToneCurve(s->ctx, 4, (double[5]) {
+ /* γ = */ 1/0.45,
+ /* a = */ 1/1.1115,
+ /* b = */ 1 - 1/1.1115,
+ /* c = */ 1/4.0,
+ /* d = */ 4.0 * 0.0228,
+ });
+ break;
+ case AVCOL_TRC_LOG:
+ s->curves[trc] = cmsBuildParametricToneCurve(s->ctx, 8, (double[5]) {
+ /* a = */ 1.0,
+ /* b = */ 10.0,
+ /* c = */ 2.0,
+ /* d = */ -1.0,
+ /* e = */ 0.0
+ });
+ break;
+ case AVCOL_TRC_LOG_SQRT:
+ s->curves[trc] = cmsBuildParametricToneCurve(s->ctx, 8, (double[5]) {
+ /* a = */ 1.0,
+ /* b = */ 10.0,
+ /* c = */ 2.5,
+ /* d = */ -1.0,
+ /* e = */ 0.0
+ });
+ break;
+ case AVCOL_TRC_IEC61966_2_1:
+ s->curves[trc] = cmsBuildParametricToneCurve(s->ctx, 4, (double[5]) {
+ /* γ = */ 2.4,
+ /* a = */ 1/1.055,
+ /* b = */ 1 - 1/1.055,
+ /* c = */ 1/12.92,
+ /* d = */ 12.92 * 0.0031308,
+ });
+ break;
+ case AVCOL_TRC_SMPTE428:
+ s->curves[trc] = cmsBuildParametricToneCurve(s->ctx, 2, (double[3]) {
+ /* γ = */ 2.6,
+ /* a = */ pow(52.37/48.0, 1/2.6),
+ /* b = */ 0.0
+ });
+ break;
+
+ /* Can't be represented using the existing parametric tone curves.
+ * FIXME: use cmsBuildTabulatedToneCurveFloat instead */
+ case AVCOL_TRC_IEC61966_2_4:
+ case AVCOL_TRC_BT1361_ECG:
+ case AVCOL_TRC_SMPTE2084:
+ case AVCOL_TRC_ARIB_STD_B67:
+ return AVERROR_PATCHWELCOME;
+
+ default:
+ return AVERROR_INVALIDDATA;
+ }
+
+ if (!s->curves[trc])
+ return AVERROR(ENOMEM);
+
+done:
+ *out_curve = s->curves[trc];
+ return 0;
+}
+
+int ff_icc_profile_generate(FFIccContext *s,
+ enum AVColorPrimaries color_prim,
+ enum AVColorTransferCharacteristic color_trc,
+ cmsHPROFILE *out_profile)
+{
+ cmsToneCurve *tonecurve;
+ const struct ColorPrimaries *prim;
+ int ret;
+
+ if (!(prim = ff_get_color_primaries(color_prim)))
+ return AVERROR_INVALIDDATA;
+ if ((ret = get_curve(s, color_trc, &tonecurve)) < 0)
+ return ret;
+
+ *out_profile = cmsCreateRGBProfileTHR(s->ctx,
+ &(cmsCIExyY) { prim->wp.xw, prim->wp.yw, 1.0 },
+ &(cmsCIExyYTRIPLE) {
+ .Red = { prim->prim.xr, prim->prim.yr, 1.0 },
+ .Green = { prim->prim.xg, prim->prim.yg, 1.0 },
+ .Blue = { prim->prim.xb, prim->prim.yb, 1.0 },
+ },
+ (cmsToneCurve *[3]) { tonecurve, tonecurve, tonecurve }
+ );
+
+ return *out_profile == NULL ? AVERROR_EXTERNAL : 0;
+}
+
+int ff_icc_profile_attach(FFIccContext *s, cmsHPROFILE profile, AVFrame *frame)
+{
+ cmsUInt32Number size;
+ AVBufferRef *buf;
+ int ret;
+
+ if (!cmsSaveProfileToMem(profile, NULL, &size))
+ return AVERROR_EXTERNAL;
+
+ buf = av_buffer_alloc(size);
+ if (!buf)
+ return AVERROR(ENOMEM);
+
+ if (!cmsSaveProfileToMem(profile, buf->data, &size) || size != buf->size) {
+ av_buffer_unref(&buf);
+ return AVERROR_EXTERNAL;
+ }
+
+ if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_ICC_PROFILE, buf)) {
+ av_buffer_unref(&buf);
+ return AVERROR(ENOMEM);
+ }
+
+ return ret;
+}
+
+static av_always_inline void XYZ_xy(cmsCIEXYZ XYZ, double *x, double *y)
+{
+ double k = 1.0 / (XYZ.X + XYZ.Y + XYZ.Z);
+ *x = k * XYZ.X;
+ *y = k * XYZ.Y;
+}
+
+int ff_icc_profile_read_primaries(FFIccContext *s, cmsHPROFILE profile,
+ struct ColorPrimaries *out_primaries)
+{
+ static const uint8_t testprimaries[4][3] = {
+ { 0xFF, 0, 0 }, /* red */
+ { 0, 0xFF, 0 }, /* green */
+ { 0, 0, 0xFF }, /* blue */
+ { 0xFF, 0xFF, 0xFF }, /* white */
+ };
+
+ struct WhitepointCoefficients *wp = &out_primaries->wp;
+ struct PrimaryCoefficients *prim = &out_primaries->prim;
+ cmsFloat64Number prev_adapt;
+ cmsHPROFILE xyz;
+ cmsHTRANSFORM tf;
+ cmsCIEXYZ dst[4];
+
+ xyz = cmsCreateXYZProfileTHR(s->ctx);
+ if (!xyz)
+ return AVERROR(ENOMEM);
+
+ /* We need to use an unadapted observer to get the raw values */
+ prev_adapt = cmsSetAdaptationStateTHR(s->ctx, 0.0);
+ tf = cmsCreateTransformTHR(s->ctx, profile, TYPE_RGB_8, xyz, TYPE_XYZ_DBL,
+ INTENT_ABSOLUTE_COLORIMETRIC,
+ /* Note: These flags mostly don't do anything
+ * anyway, but specify them regardless */
+ cmsFLAGS_NOCACHE |
+ cmsFLAGS_NOOPTIMIZE |
+ cmsFLAGS_LOWRESPRECALC |
+ cmsFLAGS_GRIDPOINTS(2));
+ cmsSetAdaptationStateTHR(s->ctx, prev_adapt);
+ cmsCloseProfile(xyz);
+ if (!tf) {
+ av_log(s->avctx, AV_LOG_ERROR, "Invalid ICC profile (e.g. CMYK)\n");
+ return AVERROR_INVALIDDATA;
+ }
+
+ cmsDoTransform(tf, testprimaries, dst, 4);
+ cmsDeleteTransform(tf);
+ XYZ_xy(dst[0], &prim->xr, &prim->yr);
+ XYZ_xy(dst[1], &prim->xg, &prim->yg);
+ XYZ_xy(dst[2], &prim->xb, &prim->yb);
+ XYZ_xy(dst[3], &wp->xw, &wp->yw);
+ return 0;
+}
+
+int ff_icc_profile_detect_transfer(FFIccContext *s, cmsHPROFILE profile,
+ enum AVColorTransferCharacteristic *out_trc)
+{
+ /* 8-bit linear grayscale ramp */
+ static const uint8_t testramp[16][3] = {
+ { 1, 1, 1}, /* avoid exact zero due to log100 etc. */
+ { 17, 17, 17},
+ { 34, 34, 34},
+ { 51, 51, 51},
+ { 68, 68, 68},
+ { 85, 85, 85},
+ { 02, 02, 02},
+ {119, 119, 119},
+ {136, 136, 136},
+ {153, 153, 153},
+ {170, 170, 170},
+ {187, 187, 187},
+ {204, 204, 204},
+ {221, 221, 221},
+ {238, 238, 238},
+ {255, 255, 255},
+ };
+
+ double dst[FF_ARRAY_ELEMS(testramp)];
+
+ for (enum AVColorTransferCharacteristic trc = 0; trc < AVCOL_TRC_NB; trc++) {
+ cmsToneCurve *tonecurve;
+ cmsHPROFILE ref;
+ cmsHTRANSFORM tf;
+ double delta = 0.0;
+ if (get_curve(s, trc, &tonecurve) < 0)
+ continue;
+
+ ref = cmsCreateGrayProfileTHR(s->ctx, cmsD50_xyY(), tonecurve);
+ if (!ref)
+ return AVERROR(ENOMEM);
+
+ tf = cmsCreateTransformTHR(s->ctx, profile, TYPE_RGB_8, ref, TYPE_GRAY_DBL,
+ INTENT_RELATIVE_COLORIMETRIC,
+ cmsFLAGS_NOCACHE | cmsFLAGS_NOOPTIMIZE);
+ cmsCloseProfile(ref);
+ if (!tf) {
+ av_log(s->avctx, AV_LOG_ERROR, "Invalid ICC profile (e.g. CMYK)\n");
+ return AVERROR_INVALIDDATA;
+ }
+
+ cmsDoTransform(tf, testramp, dst, FF_ARRAY_ELEMS(dst));
+ cmsDeleteTransform(tf);
+
+ for (int i = 0; i < FF_ARRAY_ELEMS(dst); i++)
+ delta += fabs(testramp[i][0] / 255.0 - dst[i]);
+ if (delta < 0.01) {
+ *out_trc = trc;
+ return 0;
+ }
+ }
+
+ *out_trc = AVCOL_TRC_UNSPECIFIED;
+ return 0;
+}
diff --git a/libavfilter/fflcms2.h b/libavfilter/fflcms2.h
new file mode 100644
index 0000000000..ad6c8c47cf
--- /dev/null
+++ b/libavfilter/fflcms2.h
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2022 Niklas Haas
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * Various functions for dealing with ICC profiles
+ */
+
+#ifndef AVFILTER_FFLCMS2_H
+#define AVFILTER_FFLCMS2_H
+
+#include "libavutil/frame.h"
+#include "libavutil/pixfmt.h"
+#include "colorspace.h"
+
+#include <lcms2.h>
+
+typedef struct FFIccContext {
+ void *avctx;
+ cmsContext ctx;
+ cmsToneCurve *curves[AVCOL_TRC_NB]; /* tone curve cache */
+} FFIccContext;
+
+/**
+ * Initializes an FFIccContext. This must be done prior to using it.
+ *
+ * Returns 0 on success, or a negative error code.
+ */
+int ff_icc_context_init(FFIccContext *s, void *avctx);
+void ff_icc_context_uninit(FFIccContext *s);
+
+/**
+ * Generate an ICC profile for a given combination of color primaries and
+ * transfer function. Both values must be set to valid entries (not
+ * "undefined") for this function to work.
+ *
+ * Returns 0 on success, or a negative error code.
+ */
+int ff_icc_profile_generate(FFIccContext *s,
+ enum AVColorPrimaries color_prim,
+ enum AVColorTransferCharacteristic color_trc,
+ cmsHPROFILE *out_profile);
+
+/**
+ * Attach an ICC profile to a frame. Helper wrapper around cmsSaveProfileToMem
+ * and av_frame_new_side_data_from_buf.
+ *
+ * Returns 0 on success, or a negative error code.
+ */
+int ff_icc_profile_attach(FFIccContext *s, cmsHPROFILE profile, AVFrame *frame);
+
+/**
+ * Read the color primaries and white point coefficients encoded by an ICC
+ * profile, and return the raw values in `out_primaries`.
+ *
+ * Returns 0 on success, or a negative error code.
+ */
+int ff_icc_profile_read_primaries(FFIccContext *s, cmsHPROFILE profile,
+ struct ColorPrimaries *out_primaries);
+
+/**
+ * Attempt detecting the transfer characteristic that best approximates the
+ * transfer function encoded by an ICC profile. Sets `out_trc` to
+ * AVCOL_TRC_UNSPECIFIED if no clear match can be identified.
+ *
+ * Returns 0 on success (including no match), or a negative error code.
+ */
+int ff_icc_profile_detect_transfer(FFIccContext *s, cmsHPROFILE profile,
+ enum AVColorTransferCharacteristic *out_trc);
+
+#endif /* AVFILTER_FFLCMS2_H */
--
2.35.1
More information about the ffmpeg-devel
mailing list