Add private maxColourDistance for functional tests

Switch MSE-based tests to use it

Remove experimental MSE-based compare API
This commit is contained in:
Lovell Fuller 2015-05-19 17:57:03 +01:00
parent ef8db1eebf
commit 8832ae0bf9
16 changed files with 172 additions and 446 deletions

View File

@ -666,17 +666,6 @@ Provides access to internal task counters.
var counters = sharp.counters(); // { queue: 2, process: 4 }
```
#### Experimental: sharp.compare(filename1, filename2, callback)
`filename1` and `filename2` are strings containing the filenames of the images to compare.
`callback` is called with two arguments `(err, info)` where:
* `err` contains an error message, if any.
* `info` contains the info about the difference between the two images such as `isEqual` (Boolean), `meanSquaredError` (Number; present iff `status='success'`, otherwise `undefined`), and `status` (String; one of `success`, `mismatchedDimensions`, `mismatchedBands`, `mismatchedType`).
This function can be useful for comparing test output images against reference images.
## Contributing
A [guide for contributors](https://github.com/lovell/sharp/blob/master/CONTRIBUTING.md) covers reporting bugs, requesting features and submitting code changes.

View File

@ -3,8 +3,6 @@
'target_name': 'sharp',
'sources': [
'src/common.cc',
'src/compare-internal.c',
'src/compare.cc',
'src/composite.c',
'src/metadata.cc',
'src/resize.cc',

View File

@ -758,30 +758,6 @@ Sharp.prototype.metadata = function(callback) {
}
};
/*
Reads two input images and returns stats about their differences.
Supports only callbacks.
*/
Sharp.compare = function(input1, input2, callback) {
if (typeof input1 !== 'string' || input1 === '') {
throw new TypeError('`input1` must be a non-empty string');
}
if (typeof input2 !== 'string' || input2 === '') {
throw new TypeError('`input2` must be a non-empty string');
}
if (typeof callback !== 'function') {
throw new TypeError('`callback` must be a function');
}
var options = {
fileIn1: input1,
fileIn2: input2
};
sharp.compare(options, callback);
};
/*
Get and set cache memory and item limits
*/

View File

@ -1,47 +0,0 @@
#include <stdio.h>
#include <vips/vips.h>
#include "composite.h"
const int STATS_SUM2_COLUMN = 3;
/*
Compare images `actual` and `expected` and return mean squared error (MSE).
*/
int Compare(VipsObject *context, VipsImage *actual, VipsImage *expected, double *out) {
if (actual->Bands != expected->Bands)
return -1;
if (actual->Type != expected->Type)
return -1;
VipsImage *actualPremultiplied;
if (Premultiply(context, actual, &actualPremultiplied))
return -1;
vips_object_local(context, actualPremultiplied);
VipsImage *expectedPremultiplied;
if (Premultiply(context, expected, &expectedPremultiplied))
return -1;
vips_object_local(context, expectedPremultiplied);
VipsImage *difference;
if (vips_subtract(expectedPremultiplied, actualPremultiplied, &difference, NULL))
return -1;
vips_object_local(context, difference);
VipsImage *stats;
if (vips_stats(difference, &stats, NULL))
return -1;
vips_object_local(context, stats);
double *statsData = (double*) stats->data;
int numValues = actual->Xsize * actual->Ysize * actual->Bands;
double sumOfSquares = statsData[STATS_SUM2_COLUMN];
double meanSquaredError = sumOfSquares / numValues;
// Return a reference to the mean squared error:
*out = meanSquaredError;
return 0;
}

View File

@ -1,15 +0,0 @@
#ifndef SRC_COMPARE_INTERNAL_H_
#define SRC_COMPARE_INTERNAL_H_
#ifdef __cplusplus
extern "C" {
#endif
int Compare(VipsObject *context, VipsImage *actual, VipsImage *expected, double *out);;
#ifdef __cplusplus
}
#endif
#endif // SRC_COMPARE_INTERNAL_H_

View File

@ -1,186 +0,0 @@
#include <node.h>
#include <vips/vips.h>
#include "nan.h"
#include "common.h"
#include "compare-internal.h"
using v8::Boolean;
using v8::Exception;
using v8::Function;
using v8::Handle;
using v8::Local;
using v8::Number;
using v8::Object;
using v8::String;
using v8::Value;
using sharp::ImageType;
using sharp::DetermineImageType;
using sharp::InitImage;
using sharp::counterQueue;
struct CompareBaton {
// Input
std::string fileIn1;
std::string fileIn2;
// Output
bool isEqual;
float meanSquaredError;
std::string err;
std::string status;
CompareBaton():
isEqual(false),
meanSquaredError(0.0) {}
};
class CompareWorker : public NanAsyncWorker {
public:
CompareWorker(NanCallback *callback, CompareBaton *baton) : NanAsyncWorker(callback), baton(baton) {}
~CompareWorker() {}
void Execute() {
// Decrement queued task counter
g_atomic_int_dec_and_test(&counterQueue);
ImageType imageType1 = ImageType::UNKNOWN;
ImageType imageType2 = ImageType::UNKNOWN;
VipsImage *image1 = NULL;
VipsImage *image2 = NULL;
// Create "hook" VipsObject to hang image references from
hook = reinterpret_cast<VipsObject*>(vips_image_new());
// From files
imageType1 = DetermineImageType(baton->fileIn1.c_str());
imageType2 = DetermineImageType(baton->fileIn2.c_str());
if (imageType1 != ImageType::UNKNOWN) {
image1 = InitImage(baton->fileIn1.c_str(), VIPS_ACCESS_RANDOM);
if (image1 == NULL) {
(baton->err).append("Input file 1 has corrupt header");
imageType1 = ImageType::UNKNOWN;
} else {
vips_object_local(hook, image1);
}
} else {
(baton->err).append("Input file 1 is of an unsupported image format");
}
if (imageType2 != ImageType::UNKNOWN) {
image2 = InitImage(baton->fileIn2.c_str(), VIPS_ACCESS_RANDOM);
if (image2 == NULL) {
(baton->err).append("Input file 2 has corrupt header");
imageType2 = ImageType::UNKNOWN;
} else {
vips_object_local(hook, image2);
}
} else {
(baton->err).append("Input file 2 is of an unsupported image format");
}
if (image1 != NULL && imageType1 != ImageType::UNKNOWN && image2 != NULL && imageType2 != ImageType::UNKNOWN) {
double meanSquaredError;
baton->meanSquaredError = -1.0;
baton->isEqual = FALSE;
if (image1->Type != image2->Type) {
baton->status = "mismatchedTypes";
} else if (image1->Bands != image2->Bands) {
baton->status = "mismatchedBands";
} else if (image1->Xsize != image2->Xsize || image1->Ysize != image2->Ysize) {
baton->status = "mismatchedDimensions";
} else {
if (Compare(hook, image1, image2, &meanSquaredError)) {
(baton->err).append("Failed to compare two images");
return Error();
} else {
baton->status = "success";
baton->meanSquaredError = meanSquaredError;
baton->isEqual = meanSquaredError == 0.0;
}
}
} else {
return Error();
}
CleanUp();
}
void CleanUp() {
// Clean up any dangling image references
g_object_unref(hook);
// Clean up
vips_error_clear();
vips_thread_shutdown();
}
void HandleOKCallback () {
NanScope();
Handle<Value> argv[2] = { NanNull(), NanNull() };
if (!baton->err.empty()) {
// Error
argv[0] = Exception::Error(NanNew<String>(baton->err.data(), baton->err.size()));
} else {
// Compare Object
Local<Object> info = NanNew<Object>();
info->Set(NanNew<String>("isEqual"), NanNew<Boolean>(baton->isEqual));
info->Set(NanNew<String>("status"), NanNew<String>(baton->status));
if (baton->meanSquaredError >= 0.0) {
info->Set(NanNew<String>("meanSquaredError"), NanNew<Number>(baton->meanSquaredError));
}
argv[1] = info;
}
delete baton;
// Return to JavaScript
callback->Call(2, argv);
}
/*
Copy then clear the error message.
Unref all transitional images on the hook.
Clear all thread-local data.
*/
void Error() {
// Get libvips' error message
(baton->err).append(vips_error_buffer());
CleanUp();
}
private:
CompareBaton* baton;
VipsObject *hook;
};
/*
compare(options, callback)
*/
NAN_METHOD(compare) {
NanScope();
// V8 objects are converted to non-V8 types held in the baton struct
CompareBaton *baton = new CompareBaton;
Local<Object> options = args[0]->ToObject();
// Input filename
baton->fileIn1 = *String::Utf8Value(options->Get(NanNew<String>("fileIn1"))->ToString());
baton->fileIn2 = *String::Utf8Value(options->Get(NanNew<String>("fileIn2"))->ToString());
// Join queue for worker thread
NanCallback *callback = new NanCallback(args[1].As<v8::Function>());
NanAsyncQueueWorker(new CompareWorker(callback, baton));
// Increment queued task counter
g_atomic_int_inc(&counterQueue);
NanReturnUndefined();
}

View File

@ -1,8 +0,0 @@
#ifndef SRC_COMPARE_H_
#define SRC_COMPARE_H_
#include "nan.h"
NAN_METHOD(compare);
#endif // SRC_COMPARE_H_

View File

@ -4,7 +4,6 @@
#include "nan.h"
#include "common.h"
#include "compare.h"
#include "metadata.h"
#include "resize.h"
#include "utilities.h"
@ -20,12 +19,12 @@ extern "C" void init(v8::Handle<v8::Object> target) {
// Methods available to JavaScript
NODE_SET_METHOD(target, "metadata", metadata);
NODE_SET_METHOD(target, "resize", resize);
NODE_SET_METHOD(target, "compare", compare);
NODE_SET_METHOD(target, "cache", cache);
NODE_SET_METHOD(target, "concurrency", concurrency);
NODE_SET_METHOD(target, "counters", counters);
NODE_SET_METHOD(target, "libvipsVersion", libvipsVersion);
NODE_SET_METHOD(target, "format", format);
NODE_SET_METHOD(target, "_maxColourDistance", _maxColourDistance);
}
NODE_MODULE(sharp, init)

View File

@ -5,6 +5,7 @@
#include "common.h"
#include "utilities.h"
#include "composite.h"
using v8::Local;
using v8::Object;
@ -12,6 +13,10 @@ using v8::Number;
using v8::String;
using v8::Boolean;
using sharp::DetermineImageType;
using sharp::ImageType;
using sharp::InitImage;
using sharp::HasAlpha;
using sharp::counterQueue;
using sharp::counterProcess;
@ -142,3 +147,103 @@ NAN_METHOD(format) {
NanReturnValue(format);
}
/*
Synchronous, internal-only method used by some of the functional tests.
Calculates the maximum colour distance using the DE2000 algorithm
between two images of the same dimensions and number of channels.
*/
NAN_METHOD(_maxColourDistance) {
NanScope();
// Create "hook" VipsObject to hang image references from
VipsObject *hook = reinterpret_cast<VipsObject*>(vips_image_new());
// Open input files
VipsImage *image1 = NULL;
ImageType imageType1 = DetermineImageType(*String::Utf8Value(args[0]));
if (imageType1 != ImageType::UNKNOWN) {
image1 = InitImage(*String::Utf8Value(args[0]), VIPS_ACCESS_SEQUENTIAL);
if (image1 == NULL) {
g_object_unref(hook);
return NanThrowError("Input file 1 has corrupt header");
} else {
vips_object_local(hook, image1);
}
} else {
g_object_unref(hook);
return NanThrowError("Input file 1 is of an unsupported image format");
}
VipsImage *image2 = NULL;
ImageType imageType2 = DetermineImageType(*String::Utf8Value(args[1]));
if (imageType2 != ImageType::UNKNOWN) {
image2 = InitImage(*String::Utf8Value(args[1]), VIPS_ACCESS_SEQUENTIAL);
if (image2 == NULL) {
g_object_unref(hook);
return NanThrowError("Input file 2 has corrupt header");
} else {
vips_object_local(hook, image2);
}
} else {
g_object_unref(hook);
return NanThrowError("Input file 2 is of an unsupported image format");
}
// Ensure same number of channels
if (image1->Bands != image2->Bands) {
g_object_unref(hook);
return NanThrowError("mismatchedBands");
}
// Ensure same dimensions
if (image1->Xsize != image2->Xsize || image1->Ysize != image2->Ysize) {
g_object_unref(hook);
return NanThrowError("mismatchedDimensions");
}
// Premultiply and remove alpha
if (HasAlpha(image1)) {
VipsImage *imagePremultiplied1;
if (Premultiply(hook, image1, &imagePremultiplied1)) {
g_object_unref(hook);
return NanThrowError(vips_error_buffer());
}
vips_object_local(hook, imagePremultiplied1);
VipsImage *imagePremultipliedNoAlpha1;
if (vips_extract_band(image1, &imagePremultipliedNoAlpha1, 1, "n", image1->Bands - 1, NULL)) {
g_object_unref(hook);
return NanThrowError(vips_error_buffer());
}
vips_object_local(hook, imagePremultipliedNoAlpha1);
image1 = imagePremultipliedNoAlpha1;
}
if (HasAlpha(image2)) {
VipsImage *imagePremultiplied2;
if (Premultiply(hook, image2, &imagePremultiplied2)) {
g_object_unref(hook);
return NanThrowError(vips_error_buffer());
}
vips_object_local(hook, imagePremultiplied2);
VipsImage *imagePremultipliedNoAlpha2;
if (vips_extract_band(image2, &imagePremultipliedNoAlpha2, 1, "n", image2->Bands - 1, NULL)) {
g_object_unref(hook);
return NanThrowError(vips_error_buffer());
}
vips_object_local(hook, imagePremultipliedNoAlpha2);
image2 = imagePremultipliedNoAlpha2;
}
// Calculate colour distance
VipsImage *difference;
if (vips_dE00(image1, image2, &difference, NULL)) {
g_object_unref(hook);
return NanThrowError(vips_error_buffer());
}
vips_object_local(hook, difference);
// Extract maximum distance
double maxColourDistance;
if (vips_max(difference, &maxColourDistance, NULL)) {
g_object_unref(hook);
return NanThrowError(vips_error_buffer());
}
g_object_unref(hook);
NanReturnValue(maxColourDistance);
}

View File

@ -8,5 +8,6 @@ NAN_METHOD(concurrency);
NAN_METHOD(counters);
NAN_METHOD(libvipsVersion);
NAN_METHOD(format);
NAN_METHOD(_maxColourDistance);
#endif // SRC_UTILITIES_H_

View File

@ -3,10 +3,7 @@
var path = require('path');
var assert = require('assert');
var sharp = require('../../index');
// Constants
var MAX_ALLOWED_MEAN_SQUARED_ERROR = 0.0005;
var maxColourDistance = require('../../build/Release/sharp')._maxColourDistance;
// Helpers
var getPath = function(filename) {
@ -130,29 +127,21 @@ module.exports = {
});
},
assertEqual: function(actualImagePath, expectedImagePath, callback) {
assertMaxColourDistance: function(actualImagePath, expectedImagePath, acceptedDistance) {
if (typeof actualImagePath !== 'string') {
throw new TypeError('`actualImagePath` must be a string; got ' + actualImagePath);
}
if (typeof expectedImagePath !== 'string') {
throw new TypeError('`expectedImagePath` must be a string; got ' + expectedImagePath);
}
if (typeof callback !== 'function') {
throw new TypeError('`callback` must be a function');
if (typeof acceptedDistance !== 'number') {
// Default threshold
acceptedDistance = 1;
}
sharp.compare(actualImagePath, expectedImagePath, function (error, info) {
if (error) return callback(error);
var meanSquaredError = info.meanSquaredError;
if (typeof meanSquaredError !== 'undefined' && meanSquaredError > MAX_ALLOWED_MEAN_SQUARED_ERROR) {
return callback(new Error('Expected images be equal. Mean squared error: ' + meanSquaredError + '.'));
var distance = maxColourDistance(actualImagePath, expectedImagePath);
if (distance > acceptedDistance) {
throw new Error('Expected maximum absolute distance of ' + acceptedDistance + ', actual ' + distance);
}
return callback(null, info);
});
}
};

View File

@ -83,8 +83,12 @@ describe('Alpha transparency', function() {
.resize(2048, 1536)
.interpolateWith('bicubic')
.toFile(actual, function(err) {
if (err) throw err;
fixtures.assertEqual(actual, expected, done);
if (err) {
done(err);
} else {
fixtures.assertMaxColourDistance(actual, expected, 102);
done();
}
});
});
@ -96,8 +100,12 @@ describe('Alpha transparency', function() {
.resize(1024, 768)
.interpolateWith('bicubic')
.toFile(actual, function(err) {
if (err) throw err;
fixtures.assertEqual(actual, expected, done);
if (err) {
done(err);
} else {
fixtures.assertMaxColourDistance(actual, expected, 102);
done();
}
});
});

View File

@ -1,77 +0,0 @@
'use strict';
var assert = require('assert');
var fixtures = require('../fixtures');
var fs = require('fs');
var sharp = require('../../index');
sharp.cache(0);
// Constants
var MAX_ALLOWED_MEAN_SQUARED_ERROR = 0.0005;
// Tests
describe('sharp.compare', function() {
it('should report equality when comparing an image to itself', function(done) {
var image = fixtures.inputPngOverlayLayer0;
sharp.compare(image, image, function (error, info) {
if (error) return done(error);
assert.strictEqual(info.isEqual, true, 'image is equal to itself');
assert.strictEqual(info.status, 'success', 'status is correct');
assert(0 <= info.meanSquaredError &&
info.meanSquaredError <= MAX_ALLOWED_MEAN_SQUARED_ERROR,
'MSE is within tolerance');
done();
});
});
it('should report that two images have a mismatched number of bands (channels)', function(done) {
var actual = fixtures.inputPngOverlayLayer1;
var expected = fixtures.inputJpg;
sharp.compare(actual, expected, function (error, info) {
if (error) return done(error);
assert.strictEqual(info.isEqual, false);
assert.strictEqual(info.status, 'mismatchedBands');
assert(typeof info.meanSquaredError === 'undefined', 'MSE is undefined');
done();
});
});
it('should report that two images have a mismatched dimensions', function(done) {
var actual = fixtures.inputJpg;
var expected = fixtures.inputJpgWithExif;
sharp.compare(actual, expected, function (error, info) {
if (error) return done(error);
assert.strictEqual(info.isEqual, false);
assert.strictEqual(info.status, 'mismatchedDimensions');
assert(typeof info.meanSquaredError === 'undefined', 'MSE is undefined');
done();
});
});
it('should report the correct mean squared error for two different images', function(done) {
var actual = fixtures.inputPngOverlayLayer0;
var expected = fixtures.inputPngOverlayLayer1;
sharp.compare(actual, expected, function (error, info) {
if (error) return done(error);
var meanSquaredError = info.meanSquaredError;
assert.strictEqual(info.isEqual, false);
assert.strictEqual(info.status, 'success');
// ImageMagick reports: 42242.5
// `compare -metric mse 'actual' 'expected' comparison.png`
assert(41900 <= meanSquaredError && meanSquaredError <= 41950,
'Expected: 41900 <= meanSquaredError <= 41950. Actual: ' + meanSquaredError);
done();
});
});
});

26
test/unit/fixtures.js Normal file
View File

@ -0,0 +1,26 @@
'use strict';
var assert = require('assert');
var fixtures = require('../fixtures');
describe('Test fixtures', function() {
describe('assertMaxColourDistance', function() {
it('should throw an Error when images have a different number of channels', function() {
assert.throws(function() {
fixtures.assertMaxColourDistance(fixtures.inputPngOverlayLayer1, fixtures.inputJpg);
});
});
it('should throw an Error when images have different dimensions', function() {
assert.throws(function() {
fixtures.assertMaxColourDistance(fixtures.inputJpg, fixtures.inputJpgWithExif);
});
});
it('should accept a zero threshold when comparing an image to itself', function() {
var image = fixtures.inputPngOverlayLayer0;
fixtures.assertMaxColourDistance(image, image, 0);
});
it('should accept a numeric threshold for two different images', function() {
fixtures.assertMaxColourDistance(fixtures.inputPngOverlayLayer0, fixtures.inputPngOverlayLayer1, 100);
});
});
});

View File

@ -1,54 +1,22 @@
'use strict';
var assert = require('assert');
var fixtures = require('../fixtures');
var fs = require('fs');
var sharp = require('../../index');
sharp.cache(0);
// Constants
var MAX_ALLOWED_IMAGE_MAGICK_MEAN_SQUARED_ERROR = 0.3;
// Helpers
var getPaths = function(baseName, extension) {
if (typeof extension === 'undefined') {
extension = 'png';
}
var actual = fixtures.path('output.' + baseName + '.' + extension);
var expected = fixtures.expected(baseName + '.' + extension);
var expectedMagick = fixtures.expected(baseName + '-imagemagick.' + extension);
return {
actual: actual,
expected: expected,
expectedMagick: expectedMagick
actual: fixtures.path('output.' + baseName + '.' + extension),
expected: fixtures.expected(baseName + '.' + extension),
};
};
var assertEqual = function (paths, callback) {
if (typeof callback !== 'function') {
throw new TypeError('`callback` must be a function');
}
fixtures.assertEqual(paths.actual, paths.expected, function (error) {
if (error) return callback(error);
sharp.compare(paths.actual, paths.expectedMagick, function (error, info) {
if (error) return callback(error);
if (info.meanSquaredError > MAX_ALLOWED_IMAGE_MAGICK_MEAN_SQUARED_ERROR) {
return callback(new Error('Expected MSE against ImageMagick to be <= ' +
MAX_ALLOWED_IMAGE_MAGICK_MEAN_SQUARED_ERROR + '. Actual: ' +
info.meanSquaredError));
}
callback();
});
});
};
// Test
describe('Overlays', function() {
it('Overlay transparent PNG on solid background', function(done) {
@ -58,8 +26,8 @@ describe('Overlays', function() {
.overlayWith(fixtures.inputPngOverlayLayer1)
.toFile(paths.actual, function (error) {
if (error) return done(error);
assertEqual(paths, done);
fixtures.assertMaxColourDistance(paths.actual, paths.expected);
done();
});
});
@ -70,8 +38,8 @@ describe('Overlays', function() {
.overlayWith(fixtures.inputPngOverlayLayer1LowAlpha)
.toFile(paths.actual, function (error) {
if (error) return done(error);
assertEqual(paths, done);
fixtures.assertMaxColourDistance(paths.actual, paths.expected);
done();
});
});
@ -82,13 +50,12 @@ describe('Overlays', function() {
.overlayWith(fixtures.inputPngOverlayLayer1)
.toBuffer(function (error, data, info) {
if (error) return done(error);
sharp(data)
.overlayWith(fixtures.inputPngOverlayLayer2)
.toFile(paths.actual, function (error) {
if (error) return done(error);
assertEqual(paths, done);
fixtures.assertMaxColourDistance(paths.actual, paths.expected);
done();
});
});
});
@ -100,8 +67,8 @@ describe('Overlays', function() {
.overlayWith(fixtures.inputPngOverlayLayer2)
.toFile(paths.actual, function (error, data, info) {
if (error) return done(error);
assertEqual(paths, done);
fixtures.assertMaxColourDistance(paths.actual, paths.expected);
done();
});
});
@ -112,8 +79,8 @@ describe('Overlays', function() {
.overlayWith(fixtures.inputPngOverlayLayer2LowAlpha)
.toFile(paths.actual, function (error, data, info) {
if (error) return done(error);
assertEqual(paths, done);
fixtures.assertMaxColourDistance(paths.actual, paths.expected, 2);
done();
});
});
@ -129,8 +96,8 @@ describe('Overlays', function() {
.overlayWith(fixtures.inputPngOverlayLayer2LowAlpha)
.toFile(paths.actual, function (error, data, info) {
if (error) return done(error);
assertEqual(paths, done);
fixtures.assertMaxColourDistance(paths.actual, paths.expected);
done();
});
});
});
@ -138,11 +105,11 @@ describe('Overlays', function() {
it('Composite transparent PNG onto JPEG', function(done) {
sharp(fixtures.inputJpg)
.overlayWith(fixtures.inputPngOverlayLayer1)
.toBuffer(function (error, data, info) {
.toBuffer(function (error) {
assert.strictEqual(true, error instanceof Error);
if (error.message !== 'Input image must have an alpha channel') {
return done(new Error('Unexpected error: ' + error.message));
}
done();
});
});

View File

@ -1,6 +1,7 @@
'use strict';
var assert = require('assert');
var fixtures = require('../fixtures');
var sharp = require('../../index');
var defaultConcurrency = sharp.concurrency();