Use tile cache for interlace output

Doubles performance of Adam7 interlaced PNG output

.travis.yml

@@ -5,12 +5,12 @@ node_js:
 before_install:
   - sudo add-apt-repository ppa:lyrasis/precise-backports -y
   - sudo apt-get update -qq
-  - sudo apt-get install -qq automake gobject-introspection gtk-doc-tools libfftw3-dev libglib2.0-dev libjpeg-turbo8-dev libpng12-dev libwebp-dev libtiff4-dev libexif-dev libxml2-dev swig graphicsmagick libmagick++-dev
+  - sudo apt-get install -qq automake gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng12-dev libwebp-dev libtiff4-dev libexif-dev libxml2-dev swig graphicsmagick libmagick++-dev
   - git clone https://github.com/jcupitt/libvips.git
   - cd libvips
   - git checkout 7.38
   - ./bootstrap.sh
-  - ./configure --enable-debug=no --enable-cxx=yes --without-orc --without-python
+  - ./configure --enable-debug=no --enable-cxx=yes --without-orc --without-python --without-fftw
   - make
   - sudo make install
   - sudo ldconfig

README.md

@@ -48,12 +48,12 @@ The _gettext_ dependency of _libvips_ [can lead](https://github.com/lovell/sharp
 
 Compiling from source is recommended:
 
-	sudo apt-get install automake build-essential git gobject-introspection gtk-doc-tools libfftw3-dev libglib2.0-dev libjpeg-turbo8-dev libpng12-dev libwebp-dev libtiff5-dev libexif-dev libxml2-dev swig
+	sudo apt-get install automake build-essential git gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng12-dev libwebp-dev libtiff5-dev libexif-dev libxml2-dev swig
 	git clone https://github.com/jcupitt/libvips.git
 	cd libvips
 	git checkout 7.38
 	./bootstrap.sh
-	./configure --enable-debug=no --enable-cxx=yes --without-python --without-orc
+	./configure --enable-debug=no --enable-cxx=yes --without-python --without-orc --without-fftw
 	make
 	sudo make install
 	sudo ldconfig
@@ -68,6 +68,10 @@ Requires `libtiff4-dev` instead of `libtiff5-dev` and has [a bug](https://bugs.l
 
 Then follow Ubuntu 13.x instructions.
 
+### Install libvips on Heroku
+
+[Alessandro Tagliapietra](https://github.com/alex88) maintains an [Heroku buildpack for libvips](https://github.com/alex88/heroku-buildpack-vips) and its dependencies.
+
 ## Usage examples
 
 ```javascript
@@ -160,11 +164,11 @@ Embed the resized image on a black background of the exact size specified.
 
 ### rotate([angle])
 
-Rotate the output image by either an explicit angle or auto-orient based on the EXIF `Orientation` tag. Mirroring is not supported.
+Rotate the output image by either an explicit angle or auto-orient based on the EXIF `Orientation` tag.
 
 `angle`, if present, is a Number with a value of `0`, `90`, `180` or `270`.
 
-Use this method without `angle` to determine the angle from EXIF data.
+Use this method without `angle` to determine the angle from EXIF data. Mirroring is currently unsupported.
 
 ### withoutEnlargement()
 
@@ -257,9 +261,9 @@ This method always returns cache statistics, useful for determining how much wor
 Warnings such as _Application transferred too many scanlines_ are a good indicator you've set this value too low.
 
 ```javascript
-var stats = sharp.cache(); // { current: 98, high: 115, limit: 100 }
-sharp.cache(200); // { current: 98, high: 115, limit: 200 }
-sharp.cache(50); // { current: 49, high: 115, limit: 50 }
+var stats = sharp.cache(); // { current: 98, high: 115, limit: 100, queue: 0 }
+sharp.cache(200); // { current: 98, high: 115, limit: 200, queue: 0 }
+sharp.cache(50); // { current: 49, high: 115, limit: 50, queue: 0 }
 ```
 
 ## Testing

binding.gyp

@@ -12,7 +12,10 @@
       '<!(PKG_CONFIG_PATH="<(PKG_CONFIG_PATH)" pkg-config --cflags vips glib-2.0)',
       '<!(node -e "require(\'nan\')")'
     ],
-    'cflags': ['-fexceptions', '-pedantic', '-Wall', '-O3'],
-    'cflags_cc': ['-fexceptions', '-pedantic', '-Wall', '-O3']
+    'cflags': ['-fexceptions', '-Wall', '-O3'],
+    'cflags_cc': ['-fexceptions', '-Wall', '-O3'],
+    'xcode_settings': {
+      'OTHER_CFLAGS': ['-fexceptions', '-Wall', '-O3']
+    }
   }]
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sharp",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "author": "Lovell Fuller <npm@lovell.info>",
   "contributors": [
     "Pierre Inglebert <pierre.inglebert@gmail.com>",
@@ -33,11 +33,11 @@
   ],
   "dependencies": {
     "nan": "^1.2.0",
-    "bluebird": "^2.1.2"
+    "bluebird": "^2.2.1"
   },
   "devDependencies": {
     "imagemagick": "^0.1.3",
-    "imagemagick-native": "^1.1.1",
+    "imagemagick-native": "^1.2.2",
     "gm": "^1.16.0",
     "async": "^0.9.0",
     "benchmark": "^1.0.0"

src/sharp.cc

@@ -54,6 +54,9 @@ unsigned char const MARKER_JPEG[] = {0xff, 0xd8};
 unsigned char const MARKER_PNG[] = {0x89, 0x50};
 unsigned char const MARKER_WEBP[] = {0x52, 0x49};
 
+// How many tasks are in the queue?
+volatile int queue_length = 0;
+
 static bool ends_with(std::string const &str, std::string const &end) {
   return str.length() >= end.length() && 0 == str.compare(str.length() - end.length(), end.length(), end);
 }
@@ -370,46 +373,58 @@ class ResizeWorker : public NanAsyncWorker {
     vips_colourspace(sharpened, &colourspaced, VIPS_INTERPRETATION_sRGB, NULL);
     g_object_unref(sharpened);
 
+    // Generate image tile cache when interlace output is required
+    VipsImage *cached = vips_image_new();
+    if (baton->progressive) {
+      if (vips_tilecache(colourspaced, &cached, "threaded", TRUE, "persistent", TRUE, "max_tiles", -1, NULL)) {
+        return resize_error(baton, colourspaced);
+      }
+    } else {
+      vips_copy(colourspaced, &cached, NULL);
+    }
+    g_object_unref(colourspaced);
+
     // Output
+    VipsImage *output = cached;
     if (baton->file_out == "__jpeg" || (baton->file_out == "__input" && inputImageType == JPEG)) {
       // Write JPEG to buffer
-      if (vips_jpegsave_buffer(colourspaced, &baton->buffer_out, &baton->buffer_out_len, "strip", TRUE, "Q", baton->quality, "optimize_coding", TRUE, "interlace", baton->progressive, NULL)) {
-        return resize_error(baton, colourspaced);
+      if (vips_jpegsave_buffer(output, &baton->buffer_out, &baton->buffer_out_len, "strip", TRUE, "Q", baton->quality, "optimize_coding", TRUE, "interlace", baton->progressive, NULL)) {
+        return resize_error(baton, output);
       }
     } else if (baton->file_out == "__png" || (baton->file_out == "__input" && inputImageType == PNG)) {
       // Write PNG to buffer
-      if (vips_pngsave_buffer(colourspaced, &baton->buffer_out, &baton->buffer_out_len, "strip", TRUE, "compression", baton->compressionLevel, "interlace", baton->progressive, NULL)) {
-        return resize_error(baton, colourspaced);
+      if (vips_pngsave_buffer(output, &baton->buffer_out, &baton->buffer_out_len, "strip", TRUE, "compression", baton->compressionLevel, "interlace", baton->progressive, NULL)) {
+        return resize_error(baton, output);
       }
     } else if (baton->file_out == "__webp" || (baton->file_out == "__input" && inputImageType == WEBP)) {
       // Write WEBP to buffer
-      if (vips_webpsave_buffer(colourspaced, &baton->buffer_out, &baton->buffer_out_len, "strip", TRUE, "Q", baton->quality, NULL)) {
-        return resize_error(baton, colourspaced);
+      if (vips_webpsave_buffer(output, &baton->buffer_out, &baton->buffer_out_len, "strip", TRUE, "Q", baton->quality, NULL)) {
+        return resize_error(baton, output);
       }
     } else if (is_jpeg(baton->file_out))  {
       // Write JPEG to file
-      if (vips_jpegsave(colourspaced, baton->file_out.c_str(), "strip", TRUE, "Q", baton->quality, "optimize_coding", TRUE, "interlace", baton->progressive, NULL)) {
-        return resize_error(baton, colourspaced);
+      if (vips_jpegsave(output, baton->file_out.c_str(), "strip", TRUE, "Q", baton->quality, "optimize_coding", TRUE, "interlace", baton->progressive, NULL)) {
+        return resize_error(baton, output);
       }
     } else if (is_png(baton->file_out)) {
       // Write PNG to file
-      if (vips_pngsave(colourspaced, baton->file_out.c_str(), "strip", TRUE, "compression", baton->compressionLevel, "interlace", baton->progressive, NULL)) {
-        return resize_error(baton, colourspaced);
+      if (vips_pngsave(output, baton->file_out.c_str(), "strip", TRUE, "compression", baton->compressionLevel, "interlace", baton->progressive, NULL)) {
+        return resize_error(baton, output);
       }
     } else if (is_webp(baton->file_out)) {
       // Write WEBP to file
-      if (vips_webpsave(colourspaced, baton->file_out.c_str(), "strip", TRUE, "Q", baton->quality, NULL)) {
-        return resize_error(baton, colourspaced);
+      if (vips_webpsave(output, baton->file_out.c_str(), "strip", TRUE, "Q", baton->quality, NULL)) {
+        return resize_error(baton, output);
       }
     } else if (is_tiff(baton->file_out)) {
       // Write TIFF to file
-      if (vips_tiffsave(colourspaced, baton->file_out.c_str(), "strip", TRUE, "compression", VIPS_FOREIGN_TIFF_COMPRESSION_JPEG, "Q", baton->quality, NULL)) {
-        return resize_error(baton, colourspaced);
+      if (vips_tiffsave(output, baton->file_out.c_str(), "strip", TRUE, "compression", VIPS_FOREIGN_TIFF_COMPRESSION_JPEG, "Q", baton->quality, NULL)) {
+        return resize_error(baton, output);
       }
     } else {
       (baton->err).append("Unsupported output " + baton->file_out);
     }
-    g_object_unref(colourspaced);
+    g_object_unref(output);
     vips_thread_shutdown();
   }
 
@@ -427,6 +442,8 @@ class ResizeWorker : public NanAsyncWorker {
     }
     delete baton;
     callback->Call(2, argv);
+    // Decrement queue length
+    g_atomic_int_dec_and_test(&queue_length);
   }
 
  private:
@@ -480,6 +497,10 @@ NAN_METHOD(resize) {
   // Join queue for worker thread
   NanCallback *callback = new NanCallback(args[2].As<v8::Function>());
   NanAsyncQueueWorker(new ResizeWorker(callback, baton));
+
+  // Increment queue length
+  g_atomic_int_inc(&queue_length);
+
   NanReturnUndefined();
 }
 
@@ -496,6 +517,7 @@ NAN_METHOD(cache) {
   cache->Set(NanNew<String>("current"), NanNew<Number>(vips_tracked_get_mem() / 1048576));
   cache->Set(NanNew<String>("high"), NanNew<Number>(vips_tracked_get_mem_highwater() / 1048576));
   cache->Set(NanNew<String>("limit"), NanNew<Number>(vips_cache_get_max_mem() / 1048576));
+  cache->Set(NanNew<String>("queue"), NanNew<Number>(queue_length));
   NanReturnValue(cache);
 }
 

tests/parallel.js

@@ -8,6 +8,10 @@ var inputJpg = path.join(__dirname, "fixtures/2569067123_aca715a2ee_o.jpg"); //
 var width = 720;
 var height = 480;
 
+var timer = setInterval(function() {
+  console.dir(sharp.cache());
+}, 100);
+
 async.mapSeries([1, 1, 2, 4, 8, 16, 32, 64, 128], function(parallelism, next) {
   var start = new Date().getTime();
   async.times(parallelism,
@@ -28,5 +32,6 @@ async.mapSeries([1, 1, 2, 4, 8, 16, 32, 64, 128], function(parallelism, next) {
     }
   );
 }, function() {
+  clearInterval(timer);
   console.dir(sharp.cache());
 });