Avoid (un)premultiplication for overlay image without alpha channel

Add 'premultiplied' boolean attribute to output info, helps test

docs/api-composite.md

@@ -11,6 +11,8 @@ Overlay (composite) an image over the processed (resized, extracted etc.) image.
 The overlay image must be the same size or smaller than the processed image.
 If both `top` and `left` options are provided, they take precedence over `gravity`.
 
+If the overlay image contains an alpha channel then composition with premultiplication will occur.
+
 **Parameters**
 
 -   `overlay` **([Buffer](https://nodejs.org/api/buffer.html) \| [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String))** Buffer containing image data or String containing the path to an image file.

docs/api-output.md

@@ -27,7 +27,8 @@ A Promises/A+ promise is returned when `callback` is not provided.
 
 -   `fileOut` **[String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String)** the path to write the image data to.
 -   `callback` **[Function](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/function)?** called on completion with two arguments `(err, info)`.
-    `info` contains the output image `format`, `size` (bytes), `width`, `height` and `channels`.
+    `info` contains the output image `format`, `size` (bytes), `width`, `height`,
+    `channels` and `premultiplied` (indicating if premultiplication was used).
 
 
 -   Throws **[Error](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error)** Invalid parameters
@@ -44,7 +45,8 @@ By default, the format will match the input image, except GIF and SVG input whic
 
 -   `err` is an error, if any.
 -   `data` is the output image data.
--   `info` contains the output image `format`, `size` (bytes), `width`, `height` and `channels`.
+-   `info` contains the output image `format`, `size` (bytes), `width`, `height`,
+    `channels` and `premultiplied` (indicating if premultiplication was used).
     A Promise is returned when `callback` is not provided.
 
 **Parameters**

docs/changelog.md

@@ -6,6 +6,10 @@ Requires libvips v8.5.2.
 
 #### v0.18.0 - TBD
 
+* Avoid costly (un)premultiply when using overlayWith without alpha channel.
+  [#573](https://github.com/lovell/sharp/issues/573)
+  [@strarsis](https://github.com/strarsis)
+
 * Expose warnings from libvips via NODE_DEBUG=sharp environment variable.
   [#607](https://github.com/lovell/sharp/issues/607)
   [@puzrin](https://github.com/puzrin)

lib/composite.js

@@ -8,6 +8,8 @@ const is = require('./is');
  * The overlay image must be the same size or smaller than the processed image.
  * If both `top` and `left` options are provided, they take precedence over `gravity`.
  *
+ * If the overlay image contains an alpha channel then composition with premultiplication will occur.
+ *
  * @example
  * sharp('input.png')
  *   .rotate(180)

lib/output.js

@@ -15,7 +15,8 @@ const sharp = require('../build/Release/sharp.node');
  *
  * @param {String} fileOut - the path to write the image data to.
  * @param {Function} [callback] - called on completion with two arguments `(err, info)`.
- * `info` contains the output image `format`, `size` (bytes), `width`, `height` and `channels`.
+ * `info` contains the output image `format`, `size` (bytes), `width`, `height`,
+ * `channels` and `premultiplied` (indicating if premultiplication was used).
  * @returns {Promise<Object>} - when no callback is provided
  * @throws {Error} Invalid parameters
  */
@@ -51,7 +52,8 @@ function toFile (fileOut, callback) {
  * `callback`, if present, gets three arguments `(err, data, info)` where:
  * - `err` is an error, if any.
  * - `data` is the output image data.
- * - `info` contains the output image `format`, `size` (bytes), `width`, `height` and `channels`.
+ * - `info` contains the output image `format`, `size` (bytes), `width`, `height`,
+ * `channels` and `premultiplied` (indicating if premultiplication was used).
  * A Promise is returned when `callback` is not provided.
  *
  * @param {Object} [options]

src/operations.cc

@@ -29,67 +29,32 @@ using vips::VError;
 namespace sharp {
 
   /*
-    Alpha composite src over dst with given gravity.
-    Assumes alpha channels are already premultiplied and will be unpremultiplied after.
+    Composite overlayImage over image at given position
+    Assumes alpha channels are already premultiplied and will be unpremultiplied after
    */
-  VImage Composite(VImage src, VImage dst, const int gravity) {
-    if (IsInputValidForComposition(src, dst)) {
-      // Enlarge overlay src, if required
-      if (src.width() < dst.width() || src.height() < dst.height()) {
-        // Calculate the (left, top) coordinates of the output image within the input image, applying the given gravity.
-        int left;
-        int top;
-        std::tie(left, top) = CalculateCrop(dst.width(), dst.height(), src.width(), src.height(), gravity);
-        // Embed onto transparent background
-        std::vector<double> background { 0.0, 0.0, 0.0, 0.0 };
-        src = src.embed(left, top, dst.width(), dst.height(), VImage::option()
+  VImage Composite(VImage image, VImage overlayImage, int const left, int const top) {
+    if (HasAlpha(overlayImage)) {
+      // Alpha composite
+      if (overlayImage.width() < image.width() || overlayImage.height() < image.height()) {
+        // Enlarge overlay
+        std::vector<double> const background { 0.0, 0.0, 0.0, 0.0 };
+        overlayImage = overlayImage.embed(left, top, image.width(), image.height(), VImage::option()
           ->set("extend", VIPS_EXTEND_BACKGROUND)
           ->set("background", background));
       }
-      return CompositeImage(src, dst);
+      return AlphaComposite(image, overlayImage);
+    } else {
+      if (HasAlpha(image)) {
+        // Add alpha channel to overlayImage so channels match
+        double const multiplier = sharp::Is16Bit(overlayImage.interpretation()) ? 256.0 : 1.0;
+        overlayImage = overlayImage.bandjoin(
+          VImage::new_matrix(overlayImage.width(), overlayImage.height()).new_from_image(255 * multiplier));
+      }
+      return image.insert(overlayImage, left, top);
     }
-    // If the input was not valid for composition the return the input image itself
-    return dst;
   }
 
-  VImage Composite(VImage src, VImage dst, const int x, const int y) {
-    if (IsInputValidForComposition(src, dst)) {
-      // Enlarge overlay src, if required
-      if (src.width() < dst.width() || src.height() < dst.height()) {
-        // Calculate the (left, top) coordinates of the output image within the input image, applying the given gravity.
-        int left;
-        int top;
-        std::tie(left, top) = CalculateCrop(dst.width(), dst.height(), src.width(), src.height(), x, y);
-        // Embed onto transparent background
-        std::vector<double> background { 0.0, 0.0, 0.0, 0.0 };
-        src = src.embed(left, top, dst.width(), dst.height(), VImage::option()
-          ->set("extend", VIPS_EXTEND_BACKGROUND)
-          ->set("background", background));
-      }
-      return CompositeImage(src, dst);
-    }
-    // If the input was not valid for composition the return the input image itself
-    return dst;
-  }
-
-  bool IsInputValidForComposition(VImage src, VImage dst) {
-    using sharp::CalculateCrop;
-    using sharp::HasAlpha;
-
-    if (!HasAlpha(src)) {
-      throw VError("Overlay image must have an alpha channel");
-    }
-    if (!HasAlpha(dst)) {
-      throw VError("Image to be overlaid must have an alpha channel");
-    }
-    if (src.width() > dst.width() || src.height() > dst.height()) {
-      throw VError("Overlay image must have same dimensions or smaller");
-    }
-
-    return true;
-  }
-
-  VImage CompositeImage(VImage src, VImage dst) {
+  VImage AlphaComposite(VImage dst, VImage src) {
     // Split src into non-alpha and alpha channels
     VImage srcWithoutAlpha = src.extract_band(0, VImage::option()->set("n", src.bands() - 1));
     VImage srcAlpha = src[src.bands() - 1] * (1.0 / 255.0);

src/operations.h

@@ -32,20 +32,14 @@ namespace sharp {
   VImage Composite(VImage src, VImage dst, const int gravity);
 
   /*
-    Alpha composite src over dst with given x and y offsets.
-    Assumes alpha channels are already premultiplied and will be unpremultiplied after.
+    Composite overlayImage over image at given position
    */
-  VImage Composite(VImage src, VImage dst, const int x, const int y);
+  VImage Composite(VImage image, VImage overlayImage, int const x, int const y);
 
   /*
-    Check if the src and dst Images for composition operation are valid
+    Alpha composite overlayImage over image, assumes matching dimensions
   */
-  bool IsInputValidForComposition(VImage src, VImage dst);
-
-  /*
-    Given a valid src and dst, returns the composite of the two images
-  */
-  VImage CompositeImage(VImage src, VImage dst);
+  VImage AlphaComposite(VImage image, VImage overlayImage);
 
   /*
     Cutout src over dst with given gravity.

src/pipeline.cc

@@ -333,22 +333,29 @@ class PipelineWorker : public Nan::AsyncWorker {
         image = image.colourspace(VIPS_INTERPRETATION_B_W);
       }
 
-      // Ensure image has an alpha channel when there is an overlay
-      bool hasOverlay = baton->overlay != nullptr;
-      if (hasOverlay && !HasAlpha(image)) {
+      // Ensure image has an alpha channel when there is an overlay with an alpha channel
+      VImage overlayImage;
+      ImageType overlayImageType = ImageType::UNKNOWN;
+      bool shouldOverlayWithAlpha = FALSE;
+      if (baton->overlay != nullptr) {
+        std::tie(overlayImage, overlayImageType) = OpenInput(baton->overlay, baton->accessMethod);
+        if (HasAlpha(overlayImage)) {
+          shouldOverlayWithAlpha = !baton->overlayCutout;
+          if (!HasAlpha(image)) {
             double const multiplier = sharp::Is16Bit(image.interpretation()) ? 256.0 : 1.0;
             image = image.bandjoin(
               VImage::new_matrix(image.width(), image.height()).new_from_image(255 * multiplier));
           }
+        }
+      }
 
       bool const shouldShrink = xshrink > 1 || yshrink > 1;
       bool const shouldReduce = xresidual != 1.0 || yresidual != 1.0;
       bool const shouldBlur = baton->blurSigma != 0.0;
       bool const shouldConv = baton->convKernelWidth * baton->convKernelHeight > 0;
       bool const shouldSharpen = baton->sharpenSigma != 0.0;
-      bool const shouldCutout = baton->overlayCutout;
       bool const shouldPremultiplyAlpha = HasAlpha(image) &&
-        (shouldShrink || shouldReduce || shouldBlur || shouldConv || shouldSharpen || (hasOverlay && !shouldCutout));
+        (shouldShrink || shouldReduce || shouldBlur || shouldConv || shouldSharpen || shouldOverlayWithAlpha);
 
       // Premultiply image alpha channel before all transformations to avoid
       // dark fringing around bright pixels
@@ -584,10 +591,11 @@ class PipelineWorker : public Nan::AsyncWorker {
       }
 
       // Composite with overlay, if present
-      if (hasOverlay) {
-        VImage overlayImage;
-        ImageType overlayImageType = ImageType::UNKNOWN;
-        std::tie(overlayImage, overlayImageType) = OpenInput(baton->overlay, baton->accessMethod);
+      if (baton->overlay != nullptr) {
+        // Verify overlay image is within current dimensions
+        if (overlayImage.width() > image.width() || overlayImage.height() > image.height()) {
+          throw vips::VError("Overlay image must have same dimensions or smaller");
+        }
         // Check if overlay is tiled
         if (baton->overlayTile) {
           int const overlayImageWidth = overlayImage.width();
@@ -620,31 +628,34 @@ class PipelineWorker : public Nan::AsyncWorker {
           // the overlayGravity was used for extract_area, therefore set it back to its default value of 0
           baton->overlayGravity = 0;
         }
-        if (shouldCutout) {
+        if (baton->overlayCutout) {
           // 'cut out' the image, premultiplication is not required
           image = sharp::Cutout(overlayImage, image, baton->overlayGravity);
         } else {
+          // Ensure overlay is sRGB
+          overlayImage = overlayImage.colourspace(VIPS_INTERPRETATION_sRGB);
+          // Ensure overlay matches premultiplication state
+          if (shouldPremultiplyAlpha) {
             // Ensure overlay has alpha channel
             if (!HasAlpha(overlayImage)) {
               double const multiplier = sharp::Is16Bit(overlayImage.interpretation()) ? 256.0 : 1.0;
               overlayImage = overlayImage.bandjoin(
                 VImage::new_matrix(overlayImage.width(), overlayImage.height()).new_from_image(255 * multiplier));
             }
-          // Ensure image has alpha channel
-          if (!HasAlpha(image)) {
-            double const multiplier = sharp::Is16Bit(image.interpretation()) ? 256.0 : 1.0;
-            image = image.bandjoin(
-              VImage::new_matrix(image.width(), image.height()).new_from_image(255 * multiplier));
+            overlayImage = overlayImage.premultiply();
           }
-          // Ensure overlay is premultiplied sRGB
-          overlayImage = overlayImage.colourspace(VIPS_INTERPRETATION_sRGB).premultiply();
+          int left;
+          int top;
           if (baton->overlayXOffset >= 0 && baton->overlayYOffset >= 0) {
-            // Composite images with given offsets
-            image = sharp::Composite(overlayImage, image, baton->overlayXOffset, baton->overlayYOffset);
+            // Composite images at given offsets
+            std::tie(left, top) = sharp::CalculateCrop(image.width(), image.height(),
+              overlayImage.width(), overlayImage.height(), baton->overlayXOffset, baton->overlayYOffset);
           } else {
             // Composite images with given gravity
-            image = sharp::Composite(overlayImage, image, baton->overlayGravity);
+            std::tie(left, top) = sharp::CalculateCrop(image.width(), image.height(),
+              overlayImage.width(), overlayImage.height(), baton->overlayGravity);
           }
+          image = sharp::Composite(image, overlayImage, left, top);
         }
       }
 
@@ -658,6 +669,7 @@ class PipelineWorker : public Nan::AsyncWorker {
           image = image.cast(VIPS_FORMAT_UCHAR);
         }
       }
+      baton->premultiplied = shouldPremultiplyAlpha;
 
       // Gamma decoding (brighten)
       if (baton->gamma >= 1 && baton->gamma <= 3) {
@@ -942,6 +954,7 @@ class PipelineWorker : public Nan::AsyncWorker {
       Set(info, New("width").ToLocalChecked(), New<v8::Uint32>(static_cast<uint32_t>(width)));
       Set(info, New("height").ToLocalChecked(), New<v8::Uint32>(static_cast<uint32_t>(height)));
       Set(info, New("channels").ToLocalChecked(), New<v8::Uint32>(static_cast<uint32_t>(baton->channels)));
+      Set(info, New("premultiplied").ToLocalChecked(), New<v8::Boolean>(baton->premultiplied));
       if (baton->cropCalcLeft != -1 && baton->cropCalcLeft != -1) {
         Set(info, New("cropCalcLeft").ToLocalChecked(), New<v8::Uint32>(static_cast<uint32_t>(baton->cropCalcLeft)));
         Set(info, New("cropCalcTop").ToLocalChecked(), New<v8::Uint32>(static_cast<uint32_t>(baton->cropCalcTop)));

src/pipeline.h

@@ -64,6 +64,7 @@ struct PipelineBaton {
   int crop;
   int cropCalcLeft;
   int cropCalcTop;
+  bool premultiplied;
   std::string kernel;
   std::string interpolator;
   bool centreSampling;
@@ -143,6 +144,7 @@ struct PipelineBaton {
     crop(0),
     cropCalcLeft(-1),
     cropCalcTop(-1),
+    premultiplied(false),
     centreSampling(false),
     flatten(false),
     negate(false),

test/unit/overlay.js

@@ -155,20 +155,22 @@ describe('Overlays', function () {
     });
   }
 
-  it('Composite JPEG onto PNG', function (done) {
+  it('Composite JPEG onto PNG, no premultiply', function (done) {
     sharp(fixtures.inputPngOverlayLayer1)
       .overlayWith(fixtures.inputJpgWithLandscapeExif1)
-      .toBuffer(function (error) {
-        if (error) return done(error);
+      .toBuffer(function (err, data, info) {
+        if (err) throw err;
+        assert.strictEqual(false, info.premultiplied);
         done();
       });
   });
 
-  it('Composite opaque JPEG onto JPEG', function (done) {
+  it('Composite opaque JPEG onto JPEG, no premultiply', function (done) {
     sharp(fixtures.inputJpg)
       .overlayWith(fixtures.inputJpgWithLandscapeExif1)
-      .toBuffer(function (error) {
-        if (error) return done(error);
+      .toBuffer(function (err, data, info) {
+        if (err) throw err;
+        assert.strictEqual(false, info.premultiplied);
         done();
       });
   });
@@ -561,14 +563,15 @@ describe('Overlays', function () {
         sharp(fixtures.inputJpg)
           .resize(2048, 1536)
           .overlayWith(data, { raw: info })
-          .toBuffer(function (err, data) {
+          .toBuffer(function (err, data, info) {
             if (err) throw err;
+            assert.strictEqual(true, info.premultiplied);
             fixtures.assertSimilar(fixtures.expected('overlay-jpeg-with-rgb.jpg'), data, done);
           });
       });
   });
 
-  it('Throws an error when called with an invalid file', function (done) {
+  it('Returns an error when called with an invalid file', function (done) {
     sharp(fixtures.inputJpg)
       .overlayWith('notfound.png')
       .toBuffer(function (err) {
@@ -576,4 +579,20 @@ describe('Overlays', function () {
         done();
       });
   });
+
+  it('Composite JPEG onto JPEG, no premultiply', function (done) {
+    sharp(fixtures.inputJpg)
+      .resize(480, 320)
+      .overlayWith(fixtures.inputJpgBooleanTest)
+      .png()
+      .toBuffer(function (err, data, info) {
+        if (err) throw err;
+        assert.strictEqual('png', info.format);
+        assert.strictEqual(480, info.width);
+        assert.strictEqual(320, info.height);
+        assert.strictEqual(3, info.channels);
+        assert.strictEqual(false, info.premultiplied);
+        fixtures.assertSimilar(fixtures.expected('overlay-jpeg-with-jpeg.jpg'), data, done);
+      });
+  });
 });