lcms: avoid fixed number LUT optimization on inf values

BUG=chromium:666705

Review-Url: https://codereview.chromium.org/2538703002

3 files changed, 121 insertions, 9 deletions

diff --git a/third_party/lcms2-2.6/0014-avoid-fixed-inf.patch b/third_party/lcms2-2.6/0014-avoid-fixed-inf.patch
new file mode 100644
index 0000000000..563787a941
--- /dev/null
+++ b/third_party/lcms2-2.6/0014-avoid-fixed-inf.patch
@@ -0,0 +1,95 @@
+diff --git a/third_party/lcms2-2.6/src/cmsopt.c b/third_party/lcms2-2.6/src/cmsopt.c
+index 684910d..76de015 100644
+--- a/third_party/lcms2-2.6/src/cmsopt.c
++++ b/third_party/lcms2-2.6/src/cmsopt.c
+@@ -1443,7 +1443,7 @@ void MatShaperEval16(register const cmsUInt16Number In[],
+ 
+ // This table converts from 8 bits to 1.14 after applying the curve
+ static
+-void FillFirstShaper(cmsS1Fixed14Number* Table, cmsToneCurve* Curve)
++cmsBool FillFirstShaper(cmsS1Fixed14Number* Table, cmsToneCurve* Curve)
+ {
+     int i;
+     cmsFloat32Number R, y;
+@@ -1452,14 +1452,17 @@ void FillFirstShaper(cmsS1Fixed14Number* Table, cmsToneCurve* Curve)
+ 
+         R   = (cmsFloat32Number) (i / 255.0);
+         y   = cmsEvalToneCurveFloat(Curve, R);
++        if (isinf(y))
++            return FALSE;
+ 
+         Table[i] = DOUBLE_TO_1FIXED14(y);
+     }
++    return TRUE;
+ }
+ 
+ // This table converts form 1.14 (being 0x4000 the last entry) to 8 bits after applying the curve
+ static
+-void FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8BitsOutput)
++cmsBool FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8BitsOutput)
+ {
+     int i;
+     cmsFloat32Number R, Val;
+@@ -1468,6 +1471,8 @@ void FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8Bi
+ 
+         R   = (cmsFloat32Number) (i / 16384.0);
+         Val = cmsEvalToneCurveFloat(Curve, R);    // Val comes 0..1.0
++        if (isinf(Val))
++            return FALSE;
+ 
+         if (Is8BitsOutput) {
+ 
+@@ -1482,6 +1487,7 @@ void FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8Bi
+         }
+         else Table[i]  = _cmsQuickSaturateWord(Val * 65535.0);
+     }
++    return TRUE;
+ }
+ 
+ // Compute the matrix-shaper structure
+@@ -1499,13 +1505,19 @@ cmsBool SetMatShaper(cmsPipeline* Dest, cmsToneCurve* Curve1[3], cmsMAT3* Mat, c
+     p -> ContextID = Dest -> ContextID;
+ 
+     // Precompute tables
+-    FillFirstShaper(p ->Shaper1R, Curve1[0]);
+-    FillFirstShaper(p ->Shaper1G, Curve1[1]);
+-    FillFirstShaper(p ->Shaper1B, Curve1[2]);
++    if (!FillFirstShaper(p ->Shaper1R, Curve1[0]))
++        goto Error;
++    if (!FillFirstShaper(p ->Shaper1G, Curve1[1]))
++        goto Error;
++    if (!FillFirstShaper(p ->Shaper1B, Curve1[2]))
++        goto Error;
+ 
+-    FillSecondShaper(p ->Shaper2R, Curve2[0], Is8Bits);
+-    FillSecondShaper(p ->Shaper2G, Curve2[1], Is8Bits);
+-    FillSecondShaper(p ->Shaper2B, Curve2[2], Is8Bits);
++    if (!FillSecondShaper(p ->Shaper2R, Curve2[0], Is8Bits))
++        goto Error;
++    if (!FillSecondShaper(p ->Shaper2G, Curve2[1], Is8Bits))
++        goto Error;
++    if (!FillSecondShaper(p ->Shaper2B, Curve2[2], Is8Bits))
++        goto Error;
+ 
+     // Convert matrix to nFixed14. Note that those values may take more than 16 bits as
+     for (i=0; i < 3; i++) {
+@@ -1531,6 +1543,9 @@ cmsBool SetMatShaper(cmsPipeline* Dest, cmsToneCurve* Curve1[3], cmsMAT3* Mat, c
+     // Fill function pointers
+     _cmsPipelineSetOptimizationParameters(Dest, MatShaperEval16, (void*) p, FreeMatShaper, DupMatShaper);
+     return TRUE;
++Error:
++    _cmsFree(Dest->ContextID, p);
++    return FALSE;
+ }
+ 
+ //  8 bits on input allows matrix-shaper boot up to 25 Mpixels per second on RGB. That's fast!
+@@ -1606,7 +1621,8 @@ cmsBool OptimizeMatrixShaper(cmsPipeline** Lut, cmsUInt32Number Intent, cmsUInt3
+         *dwFlags |= cmsFLAGS_NOCACHE;
+ 
+         // Setup the optimizarion routines
+-        SetMatShaper(Dest, mpeC1 ->TheCurves, &res, (cmsVEC3*) Data2 ->Offset, mpeC2->TheCurves, OutputFormat);
++        if (!SetMatShaper(Dest, mpeC1 ->TheCurves, &res, (cmsVEC3*) Data2 ->Offset, mpeC2->TheCurves, OutputFormat))
++            goto Error;
+     }
+ 
+     cmsPipelineFree(Src);
diff --git a/third_party/lcms2-2.6/README.pdfium b/third_party/lcms2-2.6/README.pdfium
index 38d03f6c9f..075d2e0727 100644
--- a/third_party/lcms2-2.6/README.pdfium
+++ b/third_party/lcms2-2.6/README.pdfium
@@ -24,4 +24,5 @@ Local Modifications:
 0012-backport-c0a98d86.patch: Fix several issues. Backport from upstream
     https://github.com/mm2/Little-CMS/commit/c0a98d86
 0013-utf8.patch: Encode source files as utf-8.
+0014-avoid-fixed-inf.patch: Avoid fixed number LUT optimization on inf values.
 TODO(ochang): List other patches.
diff --git a/third_party/lcms2-2.6/src/cmsopt.c b/third_party/lcms2-2.6/src/cmsopt.c
index 684910d799..76de01554c 100644
--- a/third_party/lcms2-2.6/src/cmsopt.c
+++ b/third_party/lcms2-2.6/src/cmsopt.c
@@ -1443,7 +1443,7 @@ void MatShaperEval16(register const cmsUInt16Number In[],
 
 // This table converts from 8 bits to 1.14 after applying the curve
 static
-void FillFirstShaper(cmsS1Fixed14Number* Table, cmsToneCurve* Curve)
+cmsBool FillFirstShaper(cmsS1Fixed14Number* Table, cmsToneCurve* Curve)
 {
     int i;
     cmsFloat32Number R, y;
@@ -1452,14 +1452,17 @@ void FillFirstShaper(cmsS1Fixed14Number* Table, cmsToneCurve* Curve)
 
         R   = (cmsFloat32Number) (i / 255.0);
         y   = cmsEvalToneCurveFloat(Curve, R);
+        if (isinf(y))
+            return FALSE;
 
         Table[i] = DOUBLE_TO_1FIXED14(y);
     }
+    return TRUE;
 }
 
 // This table converts form 1.14 (being 0x4000 the last entry) to 8 bits after applying the curve
 static
-void FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8BitsOutput)
+cmsBool FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8BitsOutput)
 {
     int i;
     cmsFloat32Number R, Val;
@@ -1468,6 +1471,8 @@ void FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8Bi
 
         R   = (cmsFloat32Number) (i / 16384.0);
         Val = cmsEvalToneCurveFloat(Curve, R);    // Val comes 0..1.0
+        if (isinf(Val))
+            return FALSE;
 
         if (Is8BitsOutput) {
 
@@ -1482,6 +1487,7 @@ void FillSecondShaper(cmsUInt16Number* Table, cmsToneCurve* Curve, cmsBool Is8Bi
         }
         else Table[i]  = _cmsQuickSaturateWord(Val * 65535.0);
     }
+    return TRUE;
 }
 
 // Compute the matrix-shaper structure
@@ -1499,13 +1505,19 @@ cmsBool SetMatShaper(cmsPipeline* Dest, cmsToneCurve* Curve1[3], cmsMAT3* Mat, c
     p -> ContextID = Dest -> ContextID;
 
     // Precompute tables
-    FillFirstShaper(p ->Shaper1R, Curve1[0]);
-    FillFirstShaper(p ->Shaper1G, Curve1[1]);
-    FillFirstShaper(p ->Shaper1B, Curve1[2]);
+    if (!FillFirstShaper(p ->Shaper1R, Curve1[0]))
+        goto Error;
+    if (!FillFirstShaper(p ->Shaper1G, Curve1[1]))
+        goto Error;
+    if (!FillFirstShaper(p ->Shaper1B, Curve1[2]))
+        goto Error;
 
-    FillSecondShaper(p ->Shaper2R, Curve2[0], Is8Bits);
-    FillSecondShaper(p ->Shaper2G, Curve2[1], Is8Bits);
-    FillSecondShaper(p ->Shaper2B, Curve2[2], Is8Bits);
+    if (!FillSecondShaper(p ->Shaper2R, Curve2[0], Is8Bits))
+        goto Error;
+    if (!FillSecondShaper(p ->Shaper2G, Curve2[1], Is8Bits))
+        goto Error;
+    if (!FillSecondShaper(p ->Shaper2B, Curve2[2], Is8Bits))
+        goto Error;
 
     // Convert matrix to nFixed14. Note that those values may take more than 16 bits as
     for (i=0; i < 3; i++) {
@@ -1531,6 +1543,9 @@ cmsBool SetMatShaper(cmsPipeline* Dest, cmsToneCurve* Curve1[3], cmsMAT3* Mat, c
     // Fill function pointers
     _cmsPipelineSetOptimizationParameters(Dest, MatShaperEval16, (void*) p, FreeMatShaper, DupMatShaper);
     return TRUE;
+Error:
+    _cmsFree(Dest->ContextID, p);
+    return FALSE;
 }
 
 //  8 bits on input allows matrix-shaper boot up to 25 Mpixels per second on RGB. That's fast!
@@ -1606,7 +1621,8 @@ cmsBool OptimizeMatrixShaper(cmsPipeline** Lut, cmsUInt32Number Intent, cmsUInt3
         *dwFlags |= cmsFLAGS_NOCACHE;
 
         // Setup the optimizarion routines
-        SetMatShaper(Dest, mpeC1 ->TheCurves, &res, (cmsVEC3*) Data2 ->Offset, mpeC2->TheCurves, OutputFormat);
+        if (!SetMatShaper(Dest, mpeC1 ->TheCurves, &res, (cmsVEC3*) Data2 ->Offset, mpeC2->TheCurves, OutputFormat))
+            goto Error;
     }
 
     cmsPipelineFree(Src);