Premultiply alpha channel to avoid dark artifacts during tranformation

Add `Sharp.compare(file1, file2, callback)` function for comparing images
using mean squared error (MSE). This is useful for unit tests.

See:
- https://github.com/jcupitt/libvips/issues/291
- http://entropymine.com/imageworsener/resizealpha/
This commit is contained in:
Daniel Gasienica
2015-05-06 20:58:22 -07:00
committed by Lovell Fuller
parent c792a047b1
commit ef8db1eebf
33 changed files with 755 additions and 85 deletions

47
src/compare-internal.c Normal file
View File

@@ -0,0 +1,47 @@
#include <stdio.h>
#include <vips/vips.h>
#include "composite.h"
const int STATS_SUM2_COLUMN = 3;
/*
Compare images `actual` and `expected` and return mean squared error (MSE).
*/
int Compare(VipsObject *context, VipsImage *actual, VipsImage *expected, double *out) {
if (actual->Bands != expected->Bands)
return -1;
if (actual->Type != expected->Type)
return -1;
VipsImage *actualPremultiplied;
if (Premultiply(context, actual, &actualPremultiplied))
return -1;
vips_object_local(context, actualPremultiplied);
VipsImage *expectedPremultiplied;
if (Premultiply(context, expected, &expectedPremultiplied))
return -1;
vips_object_local(context, expectedPremultiplied);
VipsImage *difference;
if (vips_subtract(expectedPremultiplied, actualPremultiplied, &difference, NULL))
return -1;
vips_object_local(context, difference);
VipsImage *stats;
if (vips_stats(difference, &stats, NULL))
return -1;
vips_object_local(context, stats);
double *statsData = (double*) stats->data;
int numValues = actual->Xsize * actual->Ysize * actual->Bands;
double sumOfSquares = statsData[STATS_SUM2_COLUMN];
double meanSquaredError = sumOfSquares / numValues;
// Return a reference to the mean squared error:
*out = meanSquaredError;
return 0;
}

15
src/compare-internal.h Normal file
View File

@@ -0,0 +1,15 @@
#ifndef SRC_COMPARE_INTERNAL_H_
#define SRC_COMPARE_INTERNAL_H_
#ifdef __cplusplus
extern "C" {
#endif
int Compare(VipsObject *context, VipsImage *actual, VipsImage *expected, double *out);;
#ifdef __cplusplus
}
#endif
#endif // SRC_COMPARE_INTERNAL_H_

186
src/compare.cc Normal file
View File

@@ -0,0 +1,186 @@
#include <node.h>
#include <vips/vips.h>
#include "nan.h"
#include "common.h"
#include "compare-internal.h"
using v8::Boolean;
using v8::Exception;
using v8::Function;
using v8::Handle;
using v8::Local;
using v8::Number;
using v8::Object;
using v8::String;
using v8::Value;
using sharp::ImageType;
using sharp::DetermineImageType;
using sharp::InitImage;
using sharp::counterQueue;
struct CompareBaton {
// Input
std::string fileIn1;
std::string fileIn2;
// Output
bool isEqual;
float meanSquaredError;
std::string err;
std::string status;
CompareBaton():
isEqual(false),
meanSquaredError(0.0) {}
};
class CompareWorker : public NanAsyncWorker {
public:
CompareWorker(NanCallback *callback, CompareBaton *baton) : NanAsyncWorker(callback), baton(baton) {}
~CompareWorker() {}
void Execute() {
// Decrement queued task counter
g_atomic_int_dec_and_test(&counterQueue);
ImageType imageType1 = ImageType::UNKNOWN;
ImageType imageType2 = ImageType::UNKNOWN;
VipsImage *image1 = NULL;
VipsImage *image2 = NULL;
// Create "hook" VipsObject to hang image references from
hook = reinterpret_cast<VipsObject*>(vips_image_new());
// From files
imageType1 = DetermineImageType(baton->fileIn1.c_str());
imageType2 = DetermineImageType(baton->fileIn2.c_str());
if (imageType1 != ImageType::UNKNOWN) {
image1 = InitImage(baton->fileIn1.c_str(), VIPS_ACCESS_RANDOM);
if (image1 == NULL) {
(baton->err).append("Input file 1 has corrupt header");
imageType1 = ImageType::UNKNOWN;
} else {
vips_object_local(hook, image1);
}
} else {
(baton->err).append("Input file 1 is of an unsupported image format");
}
if (imageType2 != ImageType::UNKNOWN) {
image2 = InitImage(baton->fileIn2.c_str(), VIPS_ACCESS_RANDOM);
if (image2 == NULL) {
(baton->err).append("Input file 2 has corrupt header");
imageType2 = ImageType::UNKNOWN;
} else {
vips_object_local(hook, image2);
}
} else {
(baton->err).append("Input file 2 is of an unsupported image format");
}
if (image1 != NULL && imageType1 != ImageType::UNKNOWN && image2 != NULL && imageType2 != ImageType::UNKNOWN) {
double meanSquaredError;
baton->meanSquaredError = -1.0;
baton->isEqual = FALSE;
if (image1->Type != image2->Type) {
baton->status = "mismatchedTypes";
} else if (image1->Bands != image2->Bands) {
baton->status = "mismatchedBands";
} else if (image1->Xsize != image2->Xsize || image1->Ysize != image2->Ysize) {
baton->status = "mismatchedDimensions";
} else {
if (Compare(hook, image1, image2, &meanSquaredError)) {
(baton->err).append("Failed to compare two images");
return Error();
} else {
baton->status = "success";
baton->meanSquaredError = meanSquaredError;
baton->isEqual = meanSquaredError == 0.0;
}
}
} else {
return Error();
}
CleanUp();
}
void CleanUp() {
// Clean up any dangling image references
g_object_unref(hook);
// Clean up
vips_error_clear();
vips_thread_shutdown();
}
void HandleOKCallback () {
NanScope();
Handle<Value> argv[2] = { NanNull(), NanNull() };
if (!baton->err.empty()) {
// Error
argv[0] = Exception::Error(NanNew<String>(baton->err.data(), baton->err.size()));
} else {
// Compare Object
Local<Object> info = NanNew<Object>();
info->Set(NanNew<String>("isEqual"), NanNew<Boolean>(baton->isEqual));
info->Set(NanNew<String>("status"), NanNew<String>(baton->status));
if (baton->meanSquaredError >= 0.0) {
info->Set(NanNew<String>("meanSquaredError"), NanNew<Number>(baton->meanSquaredError));
}
argv[1] = info;
}
delete baton;
// Return to JavaScript
callback->Call(2, argv);
}
/*
Copy then clear the error message.
Unref all transitional images on the hook.
Clear all thread-local data.
*/
void Error() {
// Get libvips' error message
(baton->err).append(vips_error_buffer());
CleanUp();
}
private:
CompareBaton* baton;
VipsObject *hook;
};
/*
compare(options, callback)
*/
NAN_METHOD(compare) {
NanScope();
// V8 objects are converted to non-V8 types held in the baton struct
CompareBaton *baton = new CompareBaton;
Local<Object> options = args[0]->ToObject();
// Input filename
baton->fileIn1 = *String::Utf8Value(options->Get(NanNew<String>("fileIn1"))->ToString());
baton->fileIn2 = *String::Utf8Value(options->Get(NanNew<String>("fileIn2"))->ToString());
// Join queue for worker thread
NanCallback *callback = new NanCallback(args[1].As<v8::Function>());
NanAsyncQueueWorker(new CompareWorker(callback, baton));
// Increment queued task counter
g_atomic_int_inc(&counterQueue);
NanReturnUndefined();
}

8
src/compare.h Normal file
View File

@@ -0,0 +1,8 @@
#ifndef SRC_COMPARE_H_
#define SRC_COMPARE_H_
#include "nan.h"
NAN_METHOD(compare);
#endif // SRC_COMPARE_H_

View File

@@ -2,45 +2,58 @@
#include <vips/vips.h>
// Constants
const int ALPHA_BAND_INDEX = 3;
const int NUM_COLOR_BANDS = 3;
// TODO: Copied from `common.cc`. Deduplicate once this becomes a C++ module.
int HasAlpha(VipsImage *image) {
return (
(image->Bands == 2 && image->Type == VIPS_INTERPRETATION_B_W) ||
(image->Bands == 4 && image->Type != VIPS_INTERPRETATION_CMYK) ||
(image->Bands == 5 && image->Type == VIPS_INTERPRETATION_CMYK)
);
}
/*
Composite images `src` and `dst`
Composite images `src` and `dst` with premultiplied alpha channel and output
image with premultiplied alpha.
*/
int Composite(VipsObject *context, VipsImage *src, VipsImage *dst, VipsImage **out) {
if (src->Bands != 4 || dst->Bands != 4)
int Composite(VipsObject *context, VipsImage *srcPremultiplied, VipsImage *dstPremultiplied, VipsImage **outPremultiplied) {
if (srcPremultiplied->Bands != 4 || dstPremultiplied->Bands != 4)
return -1;
// Extract RGB bands:
VipsImage *srcRGB;
VipsImage *dstRGB;
if (vips_extract_band(src, &srcRGB, 0, "n", NUM_COLOR_BANDS, NULL) ||
vips_extract_band(dst, &dstRGB, 0, "n", NUM_COLOR_BANDS, NULL))
VipsImage *srcRGBPremultiplied;
if (vips_extract_band(srcPremultiplied, &srcRGBPremultiplied, 0, "n", NUM_COLOR_BANDS, NULL))
return -1;
vips_object_local(context, srcRGBPremultiplied);
vips_object_local(context, srcRGB);
vips_object_local(context, dstRGB);
VipsImage *dstRGBPremultiplied;
if (vips_extract_band(dstPremultiplied, &dstRGBPremultiplied, 0, "n", NUM_COLOR_BANDS, NULL))
return -1;
vips_object_local(context, dstRGBPremultiplied);
// Extract alpha bands:
VipsImage *srcAlpha;
VipsImage *dstAlpha;
if (vips_extract_band(src, &srcAlpha, ALPHA_BAND_INDEX, NULL) ||
vips_extract_band(dst, &dstAlpha, ALPHA_BAND_INDEX, NULL))
if (vips_extract_band(srcPremultiplied, &srcAlpha, ALPHA_BAND_INDEX, "n", 1, NULL))
return -1;
vips_object_local(context, srcAlpha);
VipsImage *dstAlpha;
if (vips_extract_band(dstPremultiplied, &dstAlpha, ALPHA_BAND_INDEX, "n", 1, NULL))
return -1;
vips_object_local(context, dstAlpha);
// Compute normalized input alpha channels:
VipsImage *srcAlphaNormalized;
VipsImage *dstAlphaNormalized;
if (vips_linear1(srcAlpha, &srcAlphaNormalized, 1.0 / 255.0, 0.0, NULL) ||
vips_linear1(dstAlpha, &dstAlphaNormalized, 1.0 / 255.0, 0.0, NULL))
if (vips_linear1(srcAlpha, &srcAlphaNormalized, 1.0 / 255.0, 0.0, NULL))
return -1;
vips_object_local(context, srcAlphaNormalized);
VipsImage *dstAlphaNormalized;
if (vips_linear1(dstAlpha, &dstAlphaNormalized, 1.0 / 255.0, 0.0, NULL))
return -1;
vips_object_local(context, dstAlphaNormalized);
//
@@ -56,15 +69,18 @@ int Composite(VipsObject *context, VipsImage *src, VipsImage *dst, VipsImage **o
// ^^^^^^^^^^^^^^^^^^^
// t1
VipsImage *t0;
VipsImage *t1;
VipsImage *outAlphaNormalized;
if (vips_linear1(srcAlphaNormalized, &t0, -1.0, 1.0, NULL) ||
vips_multiply(dstAlphaNormalized, t0, &t1, NULL) ||
vips_add(srcAlphaNormalized, t1, &outAlphaNormalized, NULL))
if (vips_linear1(srcAlphaNormalized, &t0, -1.0, 1.0, NULL))
return -1;
vips_object_local(context, t0);
VipsImage *t1;
if (vips_multiply(dstAlphaNormalized, t0, &t1, NULL))
return -1;
vips_object_local(context, t1);
VipsImage *outAlphaNormalized;
if (vips_add(srcAlphaNormalized, t1, &outAlphaNormalized, NULL))
return -1;
vips_object_local(context, outAlphaNormalized);
//
@@ -72,54 +88,142 @@ int Composite(VipsObject *context, VipsImage *src, VipsImage *dst, VipsImage **o
//
// Wikipedia:
// out_rgb = (src_rgb * src_a + dst_rgb * dst_a * (1 - src_a)) / out_a
// ^^^^^^^^^^^
// t0
//
// `vips_ifthenelse` with `blend=TRUE`: http://bit.ly/1KoSsga
// out = (cond / 255) * in1 + (1 - cond / 255) * in2
// Omit division by `out_a` since `Compose` is supposed to output a
// premultiplied RGBA image as reversal of premultiplication is handled
// externally.
//
// Substitutions:
//
// cond --> src_a
// in1 --> src_rgb
// in2 --> dst_rgb * dst_a (premultiplied destination RGB)
//
// Finally, manually divide by `out_a` to unpremultiply the RGB channels.
// Failing to do so results in darker than expected output with low
// opacity images.
//
VipsImage *dstRGBPremultiplied;
if (vips_multiply(dstRGB, dstAlphaNormalized, &dstRGBPremultiplied, NULL))
VipsImage *t2;
if (vips_multiply(dstRGBPremultiplied, t0, &t2, NULL))
return -1;
vips_object_local(context, dstRGBPremultiplied);
vips_object_local(context, t2);
VipsImage *outRGBPremultiplied;
if (vips_ifthenelse(srcAlpha, srcRGB, dstRGBPremultiplied,
&outRGBPremultiplied, "blend", TRUE, NULL))
if (vips_add(srcRGBPremultiplied, t2, &outRGBPremultiplied, NULL))
return -1;
vips_object_local(context, outRGBPremultiplied);
// Unpremultiply RGB channels:
VipsImage *outRGB;
if (vips_divide(outRGBPremultiplied, outAlphaNormalized, &outRGB, NULL))
return -1;
vips_object_local(context, outRGB);
// Denormalize output alpha channel:
VipsImage *outAlpha;
if (vips_linear1(outAlphaNormalized, &outAlpha, 255.0, 0.0, NULL))
return -1;
vips_object_local(context, outAlpha);
// Combine RGB and alpha channel into output image:
VipsImage *joined;
if (vips_bandjoin2(outRGB, outAlpha, &joined, NULL))
if (vips_bandjoin2(outRGBPremultiplied, outAlpha, &joined, NULL))
return -1;
// Return a reference to the output image:
*out = joined;
// Return a reference to the composited output image:
*outPremultiplied = joined;
return 0;
}
/*
* Premultiply alpha channel of `image`.
*/
int Premultiply(VipsObject *context, VipsImage *image, VipsImage **out) {
VipsImage *imagePremultiplied;
// Trivial case: Copy images without alpha channel:
if (!HasAlpha(image)) {
return vips_image_write(image, *out);
}
#if (VIPS_MAJOR_VERSION >= 9 || (VIPS_MAJOR_VERSION >= 8 && VIPS_MINOR_VERSION >= 1))
if (vips_premultiply(image, &imagePremultiplied, NULL))
return -1;
#else
if (image->Bands != 4)
return -1;
VipsImage *imageRGB;
if (vips_extract_band(image, &imageRGB, 0, "n", NUM_COLOR_BANDS, NULL))
return -1;
vips_object_local(context, imageRGB);
VipsImage *imageAlpha;
if (vips_extract_band(image, &imageAlpha, ALPHA_BAND_INDEX, "n", 1, NULL))
return -1;
vips_object_local(context, imageAlpha);
VipsImage *imageAlphaNormalized;
if (vips_linear1(imageAlpha, &imageAlphaNormalized, 1.0 / 255.0, 0.0, NULL))
return -1;
vips_object_local(context, imageAlphaNormalized);
VipsImage *imageRGBPremultiplied;
if (vips_multiply(imageRGB, imageAlphaNormalized, &imageRGBPremultiplied, NULL))
return -1;
vips_object_local(context, imageRGBPremultiplied);
if (vips_bandjoin2(imageRGBPremultiplied, imageAlpha, &imagePremultiplied, NULL))
return -1;
#endif
// Return a reference to the premultiplied output image:
*out = imagePremultiplied;
return 0;
}
/*
* Unpremultiply alpha channel of `image`.
*/
int Unpremultiply(VipsObject *context, VipsImage *image, VipsImage **out) {
VipsImage *imageUnpremultiplied;
// Trivial case: Copy images without alpha channel:
if (!HasAlpha(image)) {
return vips_image_write(image, *out);
}
#if (VIPS_MAJOR_VERSION >= 9 || (VIPS_MAJOR_VERSION >= 8 && VIPS_MINOR_VERSION >= 1))
if (vips_unpremultiply(image, &imageUnpremultiplied, NULL))
return -1;
#else
if (image->Bands != 4)
return -1;
VipsImage *imageRGBPremultipliedTransformed;
if (vips_extract_band(image, &imageRGBPremultipliedTransformed, 0, "n", NUM_COLOR_BANDS, NULL))
return -1;
vips_object_local(context, imageRGBPremultipliedTransformed);
VipsImage *imageAlphaTransformed;
if (vips_extract_band(image, &imageAlphaTransformed, ALPHA_BAND_INDEX, "n", 1, NULL))
return -1;
vips_object_local(context, imageAlphaTransformed);
VipsImage *imageAlphaNormalizedTransformed;
if (vips_linear1(imageAlphaTransformed, &imageAlphaNormalizedTransformed, 1.0 / 255.0, 0.0, NULL))
return -1;
vips_object_local(context, imageAlphaNormalizedTransformed);
VipsImage *imageRGBUnpremultipliedTransformed;
if (vips_divide(imageRGBPremultipliedTransformed, imageAlphaNormalizedTransformed, &imageRGBUnpremultipliedTransformed, NULL))
return -1;
vips_object_local(context, imageRGBUnpremultipliedTransformed);
if (vips_bandjoin2(imageRGBUnpremultipliedTransformed, imageAlphaTransformed, &imageUnpremultiplied, NULL))
return -1;
#endif
// Return a reference to the unpremultiplied output image:
*out = imageUnpremultiplied;
return 0;
}

View File

@@ -5,10 +5,10 @@
#ifdef __cplusplus
extern "C" {
#endif
/*
Composite images `src` and `dst`.
*/
int Composite(VipsObject *context, VipsImage *src, VipsImage *dst, VipsImage **out);
int Premultiply(VipsObject *context, VipsImage *image, VipsImage **out);
int Unpremultiply(VipsObject *context, VipsImage *image, VipsImage **out);
#ifdef __cplusplus
}

View File

@@ -480,8 +480,29 @@ class ResizeWorker : public NanAsyncWorker {
}
}
// Premultiply image alpha channel before all transformations to avoid
// dark fringing around bright pixels
// See: http://entropymine.com/imageworsener/resizealpha/
bool shouldAffineTransform = xresidual != 0.0 || yresidual != 0.0;
bool shouldBlur = baton->blurSigma != 0.0;
bool shouldSharpen = baton->sharpenRadius != 0;
bool shouldTransform = shouldAffineTransform || shouldBlur || shouldSharpen;
bool hasOverlay = !baton->overlayPath.empty();
bool shouldPremultiplyAlpha = HasAlpha(image) && image->Bands == 4 && (shouldTransform || hasOverlay);
if (shouldPremultiplyAlpha) {
VipsImage *imagePremultiplied;
if (Premultiply(hook, image, &imagePremultiplied)) {
(baton->err).append("Failed to premultiply alpha channel.");
return Error();
}
vips_object_local(hook, imagePremultiplied);
image = imagePremultiplied;
}
// Use vips_affine with the remaining float part
if (xresidual != 0.0 || yresidual != 0.0) {
if (shouldAffineTransform) {
// Use average of x and y residuals to compute sigma for Gaussian blur
double residual = (xresidual + yresidual) / 2.0;
// Apply Gaussian blur before large affine reductions
@@ -646,7 +667,7 @@ class ResizeWorker : public NanAsyncWorker {
}
// Blur
if (baton->blurSigma != 0.0) {
if (shouldBlur) {
VipsImage *blurred;
if (baton->blurSigma < 0.0) {
// Fast, mild blur - averages neighbouring pixels
@@ -677,7 +698,7 @@ class ResizeWorker : public NanAsyncWorker {
}
// Sharpen
if (baton->sharpenRadius != 0) {
if (shouldSharpen) {
VipsImage *sharpened;
if (baton->sharpenRadius == -1) {
// Fast, mild sharpen
@@ -793,7 +814,7 @@ class ResizeWorker : public NanAsyncWorker {
#endif
// Composite with overlay, if present
if (!baton->overlayPath.empty()) {
if (hasOverlay) {
VipsImage *overlayImage = NULL;
ImageType overlayImageType = ImageType::UNKNOWN;
overlayImageType = DetermineImageType(baton->overlayPath.c_str());
@@ -802,6 +823,8 @@ class ResizeWorker : public NanAsyncWorker {
if (overlayImage == NULL) {
(baton->err).append("Overlay input file has corrupt header");
overlayImageType = ImageType::UNKNOWN;
} else {
vips_object_local(hook, overlayImage);
}
} else {
(baton->err).append("Overlay input file is of an unsupported image format");
@@ -831,8 +854,15 @@ class ResizeWorker : public NanAsyncWorker {
return Error();
}
VipsImage *overlayImagePremultiplied;
if (Premultiply(hook, overlayImage, &overlayImagePremultiplied)) {
(baton->err).append("Failed to premultiply alpha channel of overlay image.");
return Error();
}
vips_object_local(hook, overlayImagePremultiplied);
VipsImage *composited;
if (Composite(hook, overlayImage, image, &composited)) {
if (Composite(hook, overlayImagePremultiplied, image, &composited)) {
(baton->err).append("Failed to composite images");
return Error();
}
@@ -840,6 +870,18 @@ class ResizeWorker : public NanAsyncWorker {
image = composited;
}
// Reverse premultiplication after all transformations:
if (shouldPremultiplyAlpha) {
VipsImage *imageUnpremultiplied;
if (Unpremultiply(hook, image, &imageUnpremultiplied)) {
(baton->err).append("Failed to unpremultiply alpha channel.");
return Error();
}
vips_object_local(hook, imageUnpremultiplied);
image = imageUnpremultiplied;
}
// Convert image to sRGB, if not already
if (image->Type != VIPS_INTERPRETATION_sRGB) {
// Switch intrepretation to sRGB

View File

@@ -4,6 +4,7 @@
#include "nan.h"
#include "common.h"
#include "compare.h"
#include "metadata.h"
#include "resize.h"
#include "utilities.h"
@@ -19,6 +20,7 @@ extern "C" void init(v8::Handle<v8::Object> target) {
// Methods available to JavaScript
NODE_SET_METHOD(target, "metadata", metadata);
NODE_SET_METHOD(target, "resize", resize);
NODE_SET_METHOD(target, "compare", compare);
NODE_SET_METHOD(target, "cache", cache);
NODE_SET_METHOD(target, "concurrency", concurrency);
NODE_SET_METHOD(target, "counters", counters);