Expose libvips' op-cache max items #76

This commit is contained in:
Lovell Fuller 2014-08-19 20:30:21 +01:00
parent e45956db6c
commit 8ef0851a49
4 changed files with 88 additions and 18 deletions

View File

@ -288,18 +288,30 @@ A Promises/A+ promise is returned when `callback` is not provided.
### Utility methods ### Utility methods
#### sharp.cache([limit]) #### sharp.cache([memory], [items])
If `limit` is provided, set the (soft) limit of _libvips_ working/cache memory to this value in MB. The default value is 100. If `memory` or `items` are provided, set the limits of _libvips'_ operation cache.
* `memory` is the maximum memory in MB to use for this cache, with a default value of 100
* `items` is the maximum number of operations to cache, with a default value of 500
This method always returns cache statistics, useful for determining how much working memory is required for a particular task. This method always returns cache statistics, useful for determining how much working memory is required for a particular task.
Warnings such as _Application transferred too many scanlines_ are a good indicator you've set this value too low. ```javascript
var stats = sharp.cache(); // { current: 75, high: 99, memory: 100, items: 500 }
sharp.cache(200); // { current: 75, high: 99, memory: 200, items: 500 }
sharp.cache(50, 200); // { current: 49, high: 99, memory: 50, items: 200}
```
#### sharp.counters()
Provides access to internal task counters.
* `queue` is the number of tasks queuing for _libuv_ to provide a thread from its pool
* `process` is the number of tasks being processed
```javascript ```javascript
var stats = sharp.cache(); // { current: 98, high: 115, limit: 100, queue: 0 } var counters = sharp.counters(); // { queue: 2, process: 4 }
sharp.cache(200); // { current: 98, high: 115, limit: 200, queue: 0 }
sharp.cache(50); // { current: 49, high: 115, limit: 50, queue: 0 }
``` ```
## Testing ## Testing

View File

@ -335,9 +335,22 @@ Sharp.prototype._sharp = function(callback) {
} }
}; };
module.exports.cache = function(limit) { /*
if (Number.isNaN(limit)) { Get and set cache memory and item limits
limit = null; */
module.exports.cache = function(memory, items) {
if (Number.isNaN(memory)) {
memory = null;
} }
return sharp.cache(limit); if (Number.isNaN(items)) {
items = null;
}
return sharp.cache(memory, items);
};
/*
Get internal counters
*/
module.exports.counters = function() {
return sharp.counters();
}; };

View File

@ -55,7 +55,9 @@ unsigned char const MARKER_PNG[] = {0x89, 0x50};
unsigned char const MARKER_WEBP[] = {0x52, 0x49}; unsigned char const MARKER_WEBP[] = {0x52, 0x49};
// How many tasks are in the queue? // How many tasks are in the queue?
volatile int queue_length = 0; volatile int counter_queue = 0;
// How many tasks are being processed?
volatile int counter_process = 0;
static bool ends_with(std::string const &str, std::string const &end) { static bool ends_with(std::string const &str, std::string const &end) {
return str.length() >= end.length() && 0 == str.compare(str.length() - end.length(), end.length(), end); return str.length() >= end.length() && 0 == str.compare(str.length() - end.length(), end.length(), end);
@ -124,6 +126,11 @@ class ResizeWorker : public NanAsyncWorker {
~ResizeWorker() {} ~ResizeWorker() {}
void Execute () { void Execute () {
// Decrement queued task counter
g_atomic_int_dec_and_test(&counter_queue);
// Increment processing task counter
g_atomic_int_inc(&counter_process);
// Input // Input
ImageType inputImageType = JPEG; ImageType inputImageType = JPEG;
VipsImage *in = vips_image_new(); VipsImage *in = vips_image_new();
@ -455,9 +462,12 @@ class ResizeWorker : public NanAsyncWorker {
} }
} }
delete baton; delete baton;
// Decrement processing task counter
g_atomic_int_dec_and_test(&counter_process);
// Return to JavaScript
callback->Call(3, argv); callback->Call(3, argv);
// Decrement queue length
g_atomic_int_dec_and_test(&queue_length);
} }
private: private:
@ -512,29 +522,48 @@ NAN_METHOD(resize) {
NanCallback *callback = new NanCallback(args[1].As<v8::Function>()); NanCallback *callback = new NanCallback(args[1].As<v8::Function>());
NanAsyncQueueWorker(new ResizeWorker(callback, baton)); NanAsyncQueueWorker(new ResizeWorker(callback, baton));
// Increment queue length // Increment queued task counter
g_atomic_int_inc(&queue_length); g_atomic_int_inc(&counter_queue);
NanReturnUndefined(); NanReturnUndefined();
} }
/*
Get and set cache memory and item limits
*/
NAN_METHOD(cache) { NAN_METHOD(cache) {
NanScope(); NanScope();
// Set cache limit // Set cache memory limit
if (args[0]->IsInt32()) { if (args[0]->IsInt32()) {
vips_cache_set_max_mem(args[0]->Int32Value() * 1048576); vips_cache_set_max_mem(args[0]->Int32Value() * 1048576);
} }
// Set cache items limit
if (args[1]->IsInt32()) {
vips_cache_set_max(args[1]->Int32Value());
}
// Get cache statistics // Get cache statistics
Local<Object> cache = NanNew<Object>(); Local<Object> cache = NanNew<Object>();
cache->Set(NanNew<String>("current"), NanNew<Number>(vips_tracked_get_mem() / 1048576)); cache->Set(NanNew<String>("current"), NanNew<Number>(vips_tracked_get_mem() / 1048576));
cache->Set(NanNew<String>("high"), NanNew<Number>(vips_tracked_get_mem_highwater() / 1048576)); cache->Set(NanNew<String>("high"), NanNew<Number>(vips_tracked_get_mem_highwater() / 1048576));
cache->Set(NanNew<String>("limit"), NanNew<Number>(vips_cache_get_max_mem() / 1048576)); cache->Set(NanNew<String>("memory"), NanNew<Number>(vips_cache_get_max_mem() / 1048576));
cache->Set(NanNew<String>("queue"), NanNew<Number>(queue_length)); cache->Set(NanNew<String>("items"), NanNew<Number>(vips_cache_get_max()));
NanReturnValue(cache); NanReturnValue(cache);
} }
/*
Get internal counters (queued tasks, processing tasks)
*/
NAN_METHOD(counters) {
NanScope();
Local<Object> counters = NanNew<Object>();
counters->Set(NanNew<String>("queue"), NanNew<Number>(counter_queue));
counters->Set(NanNew<String>("process"), NanNew<Number>(counter_process));
NanReturnValue(counters);
}
static void at_exit(void* arg) { static void at_exit(void* arg) {
NanScope(); NanScope();
vips_shutdown(); vips_shutdown();
@ -544,8 +573,13 @@ extern "C" void init(Handle<Object> target) {
NanScope(); NanScope();
vips_init(""); vips_init("");
AtExit(at_exit); AtExit(at_exit);
// Set libvips operation cache limits
vips_cache_set_max_mem(100 * 1048576); // 100 MB
vips_cache_set_max(500); // 500 operations
// Methods available to JavaScript
NODE_SET_METHOD(target, "resize", resize); NODE_SET_METHOD(target, "resize", resize);
NODE_SET_METHOD(target, "cache", cache); NODE_SET_METHOD(target, "cache", cache);
NODE_SET_METHOD(target, "counters", counters);
} }
NODE_MODULE(sharp, init) NODE_MODULE(sharp, init)

View File

@ -18,6 +18,10 @@ var outputTiff = path.join(fixturesPath, "output.tiff");
var inputJpgWithExif = path.join(fixturesPath, "Landscape_8.jpg"); // https://github.com/recurser/exif-orientation-examples/blob/master/Landscape_8.jpg var inputJpgWithExif = path.join(fixturesPath, "Landscape_8.jpg"); // https://github.com/recurser/exif-orientation-examples/blob/master/Landscape_8.jpg
// Ensure cache limits can be set
sharp.cache(0); // Disable
sharp.cache(50, 500); // 50MB, 500 items
async.series([ async.series([
// Resize with exact crop // Resize with exact crop
function(done) { function(done) {
@ -339,5 +343,12 @@ async.series([
}); });
var pipeline = sharp().resize(320, 240); var pipeline = sharp().resize(320, 240);
readable.pipe(pipeline).pipe(writable) readable.pipe(pipeline).pipe(writable)
},
// Verify internal counters
function(done) {
var counters = sharp.counters();
assert.strictEqual(0, counters.queue);
assert.strictEqual(0, counters.process);
done();
} }
]); ]);