diff --git a/docs/changelog.md b/docs/changelog.md index 48446e03..7644b1a9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -22,6 +22,9 @@ Requires libvips v8.5.2. [#607](https://github.com/lovell/sharp/issues/607) [@puzrin](https://github.com/puzrin) +* Switch to the libvips implementation of "attention" and "entropy" crop strategies. + [#727](https://github.com/lovell/sharp/issues/727) + * Improve performance and accuracy of nearest neighbour integral upsampling. [#752](https://github.com/lovell/sharp/issues/752) [@MrIbby](https://github.com/MrIbby) diff --git a/src/operations.cc b/src/operations.cc index 1cfdb02b..274ec911 100644 --- a/src/operations.cc +++ b/src/operations.cc @@ -267,118 +267,6 @@ namespace sharp { } } - /* - Calculate the Shannon entropy - */ - double EntropyStrategy::operator()(VImage image) { - return image.hist_find().hist_entropy(); - } - - /* - Calculate the intensity of edges, skin tone and saturation - */ - double AttentionStrategy::operator()(VImage image) { - // Flatten RGBA onto a mid-grey background - if (image.bands() == 4 && HasAlpha(image)) { - double const midgrey = sharp::Is16Bit(image.interpretation()) ? 32768.0 : 128.0; - std::vector background { midgrey, midgrey, midgrey }; - image = image.flatten(VImage::option()->set("background", background)); - } - // Convert to LAB colourspace - VImage lab = image.colourspace(VIPS_INTERPRETATION_LAB); - VImage l = lab[0]; - VImage a = lab[1]; - VImage b = lab[2]; - // Edge detect luminosity with the Sobel operator - VImage sobel = vips::VImage::new_matrixv(3, 3, - -1.0, 0.0, 1.0, - -2.0, 0.0, 2.0, - -1.0, 0.0, 1.0); - VImage edges = l.conv(sobel).abs() + l.conv(sobel.rot90()).abs(); - // Skin tone chroma thresholds trained with http://humanae.tumblr.com/ - VImage skin = (a >= 3) & (a <= 22) & (b >= 4) & (b <= 31); - // Chroma >~50% saturation - VImage lch = lab.colourspace(VIPS_INTERPRETATION_LCH); - VImage c = lch[1]; - VImage saturation = c > 60; - // Find maximum in combined saliency mask - VImage mask = edges + skin + saturation; - return mask.max(); - } - - /* - Calculate crop area based on image entropy - */ - std::tuple Crop( - VImage image, int const outWidth, int const outHeight, std::function strategy - ) { - int left = 0; - int top = 0; - int const inWidth = image.width(); - int const inHeight = image.height(); - if (inWidth > outWidth) { - // Reduce width by repeated removing slices from edge with lowest score - int width = inWidth; - double leftScore = 0.0; - double rightScore = 0.0; - // Max width of each slice - int const maxSliceWidth = static_cast(ceil((inWidth - outWidth) / 8.0)); - while (width > outWidth) { - // Width of current slice - int const slice = std::min(width - outWidth, maxSliceWidth); - if (leftScore == 0.0) { - // Update score of left slice - leftScore = strategy(image.extract_area(left, 0, slice, inHeight)); - } - if (rightScore == 0.0) { - // Update score of right slice - rightScore = strategy(image.extract_area(width - slice - 1, 0, slice, inHeight)); - } - // Keep slice with highest score - if (leftScore >= rightScore) { - // Discard right slice - rightScore = 0.0; - } else { - // Discard left slice - leftScore = 0.0; - left = left + slice; - } - width = width - slice; - } - } - if (inHeight > outHeight) { - // Reduce height by repeated removing slices from edge with lowest score - int height = inHeight; - double topScore = 0.0; - double bottomScore = 0.0; - // Max height of each slice - int const maxSliceHeight = static_cast(ceil((inHeight - outHeight) / 8.0)); - while (height > outHeight) { - // Height of current slice - int const slice = std::min(height - outHeight, maxSliceHeight); - if (topScore == 0.0) { - // Update score of top slice - topScore = strategy(image.extract_area(0, top, inWidth, slice)); - } - if (bottomScore == 0.0) { - // Update score of bottom slice - bottomScore = strategy(image.extract_area(0, height - slice - 1, inWidth, slice)); - } - // Keep slice with highest score - if (topScore >= bottomScore) { - // Discard bottom slice - bottomScore = 0.0; - } else { - // Discard top slice - topScore = 0.0; - top = top + slice; - } - height = height - slice; - } - } - return std::make_tuple(left, top); - } - /* Insert a tile cache to prevent over-computation of any previous operations in the pipeline */ diff --git a/src/operations.h b/src/operations.h index 66836e77..529bca44 100644 --- a/src/operations.h +++ b/src/operations.h @@ -72,22 +72,6 @@ namespace sharp { */ VImage Sharpen(VImage image, double const sigma, double const flat, double const jagged); - /* - Crop strategy functors - */ - struct EntropyStrategy { - double operator()(VImage image); - }; - struct AttentionStrategy { - double operator()(VImage image); - }; - - /* - Calculate crop area based on given strategy (Entropy, Attention) - */ - std::tuple Crop( - VImage image, int const outWidth, int const outHeight, std::function strategy); - /* Insert a tile cache to prevent over-computation of any previous operations in the pipeline */ diff --git a/src/pipeline.cc b/src/pipeline.cc index f31540cb..47c50772 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -509,24 +509,20 @@ class PipelineWorker : public Nan::AsyncWorker { ->set("background", background)); } else if (baton->canvas != Canvas::IGNORE_ASPECT) { // Crop/max/min - int left; - int top; if (baton->crop < 9) { // Gravity-based crop + int left; + int top; std::tie(left, top) = sharp::CalculateCrop( image.width(), image.height(), baton->width, baton->height, baton->crop); - } else if (baton->crop == 16) { - // Entropy-based crop - std::tie(left, top) = sharp::Crop(image, baton->width, baton->height, sharp::EntropyStrategy()); + int width = std::min(image.width(), baton->width); + int height = std::min(image.height(), baton->height); + image = image.extract_area(left, top, width, height); } else { - // Attention-based crop - std::tie(left, top) = sharp::Crop(image, baton->width, baton->height, sharp::AttentionStrategy()); + // Attention-based or Entropy-based crop + image = image.smartcrop(baton->width, baton->height, VImage::option() + ->set("interesting", baton->crop == 16 ? VIPS_INTERESTING_ENTROPY : VIPS_INTERESTING_ATTENTION)); } - int width = std::min(image.width(), baton->width); - int height = std::min(image.height(), baton->height); - image = image.extract_area(left, top, width, height); - baton->cropCalcLeft = left; - baton->cropCalcTop = top; } } diff --git a/test/fixtures/expected/crop-strategy-attention.jpg b/test/fixtures/expected/crop-strategy-attention.jpg new file mode 100644 index 00000000..16ddaa16 Binary files /dev/null and b/test/fixtures/expected/crop-strategy-attention.jpg differ diff --git a/test/fixtures/expected/crop-strategy-entropy.jpg b/test/fixtures/expected/crop-strategy-entropy.jpg new file mode 100644 index 00000000..8f236071 Binary files /dev/null and b/test/fixtures/expected/crop-strategy-entropy.jpg differ diff --git a/test/fixtures/expected/crop-strategy.jpg b/test/fixtures/expected/crop-strategy.jpg deleted file mode 100644 index af12e8a3..00000000 Binary files a/test/fixtures/expected/crop-strategy.jpg and /dev/null differ diff --git a/test/unit/crop.js b/test/unit/crop.js index b8087a3a..9a5a2759 100644 --- a/test/unit/crop.js +++ b/test/unit/crop.js @@ -161,7 +161,7 @@ describe('Crop', function () { describe('Entropy-based strategy', function () { it('JPEG', function (done) { - sharp(fixtures.inputJpgWithCmykProfile) + sharp(fixtures.inputJpg) .resize(80, 320) .crop(sharp.strategy.entropy) .toBuffer(function (err, data, info) { @@ -170,9 +170,7 @@ describe('Crop', function () { assert.strictEqual(3, info.channels); assert.strictEqual(80, info.width); assert.strictEqual(320, info.height); - assert.strictEqual(250, info.cropCalcLeft); - assert.strictEqual(0, info.cropCalcTop); - fixtures.assertSimilar(fixtures.expected('crop-strategy.jpg'), data, done); + fixtures.assertSimilar(fixtures.expected('crop-strategy-entropy.jpg'), data, done); }); }); @@ -186,8 +184,6 @@ describe('Crop', function () { assert.strictEqual(4, info.channels); assert.strictEqual(320, info.width); assert.strictEqual(80, info.height); - assert.strictEqual(0, info.cropCalcLeft); - assert.strictEqual(80, info.cropCalcTop); fixtures.assertSimilar(fixtures.expected('crop-strategy.png'), data, done); }); }); @@ -202,8 +198,6 @@ describe('Crop', function () { assert.strictEqual(4, info.channels); assert.strictEqual(320, info.width); assert.strictEqual(80, info.height); - assert.strictEqual(0, info.cropCalcLeft); - assert.strictEqual(80, info.cropCalcTop); fixtures.assertSimilar(fixtures.expected('crop-strategy.png'), data, done); }); }); @@ -211,7 +205,7 @@ describe('Crop', function () { describe('Attention strategy', function () { it('JPEG', function (done) { - sharp(fixtures.inputJpgWithCmykProfile) + sharp(fixtures.inputJpg) .resize(80, 320) .crop(sharp.strategy.attention) .toBuffer(function (err, data, info) { @@ -220,9 +214,7 @@ describe('Crop', function () { assert.strictEqual(3, info.channels); assert.strictEqual(80, info.width); assert.strictEqual(320, info.height); - assert.strictEqual(250, info.cropCalcLeft); - assert.strictEqual(0, info.cropCalcTop); - fixtures.assertSimilar(fixtures.expected('crop-strategy.jpg'), data, done); + fixtures.assertSimilar(fixtures.expected('crop-strategy-attention.jpg'), data, done); }); }); @@ -236,8 +228,6 @@ describe('Crop', function () { assert.strictEqual(4, info.channels); assert.strictEqual(320, info.width); assert.strictEqual(80, info.height); - assert.strictEqual(0, info.cropCalcLeft); - assert.strictEqual(80, info.cropCalcTop); fixtures.assertSimilar(fixtures.expected('crop-strategy.png'), data, done); }); }); @@ -252,8 +242,6 @@ describe('Crop', function () { assert.strictEqual(4, info.channels); assert.strictEqual(320, info.width); assert.strictEqual(80, info.height); - assert.strictEqual(0, info.cropCalcLeft); - assert.strictEqual(80, info.cropCalcTop); fixtures.assertSimilar(fixtures.expected('crop-strategy.png'), data, done); }); });