Reduce concurrency when using glibc-based Linux

to help prevent memory fragmentation

docs/changelog.md

@@ -14,6 +14,8 @@ Requires libvips v8.10.6
 
 * Reduce the default PNG `compressionLevel` to the more commonly used 6.
 
+* Reduce concurrency on glibc-based Linux when using the default memory allocator to help prevent fragmentation.
+
 ## v0.27 - *avif*
 
 Requires libvips v8.10.5

lib/utility.js

@@ -1,6 +1,8 @@
 'use strict';
 
 const events = require('events');
+const detectLibc = require('detect-libc');
+
 const is = require('./is');
 const sharp = require('../build/Release/sharp.node');
 
@@ -84,8 +86,12 @@ cache(true);
 /**
  * Gets or, when a concurrency is provided, sets
  * the number of threads _libvips'_ should create to process each image.
- * The default value is the number of CPU cores.
- * A value of `0` will reset to this default.
+ *
+ * The default value is the number of CPU cores,
+ * except when using glibc-based Linux without jemalloc,
+ * where the default is `1` to help reduce memory fragmentation.
+ *
+ * A value of `0` will reset this to the number of CPU cores.
  *
  * The maximum number of images that can be processed in parallel
  * is limited by libuv's `UV_THREADPOOL_SIZE` environment variable.
@@ -103,6 +109,11 @@ cache(true);
 function concurrency (concurrency) {
   return sharp.concurrency(is.integer(concurrency) ? concurrency : null);
 }
+/* istanbul ignore next */
+if (detectLibc.family === detectLibc.GLIBC && !sharp._isUsingJemalloc()) {
+  // Reduce default concurrency to 1 when using glibc memory allocator
+  sharp.concurrency(1);
+}
 
 /**
  * An EventEmitter that emits a `change` event when a task is either:

src/sharp.cc

@@ -44,6 +44,7 @@ Napi::Object init(Napi::Env env, Napi::Object exports) {
   exports.Set("libvipsVersion", Napi::Function::New(env, libvipsVersion));
   exports.Set("format", Napi::Function::New(env, format));
   exports.Set("_maxColourDistance", Napi::Function::New(env, _maxColourDistance));
+  exports.Set("_isUsingJemalloc", Napi::Function::New(env, _isUsingJemalloc));
   exports.Set("stats", Napi::Function::New(env, stats));
   return exports;
 }

src/utilities.cc

@@ -225,3 +225,19 @@ Napi::Value _maxColourDistance(const Napi::CallbackInfo& info) {
 
   return Napi::Number::New(env, maxColourDistance);
 }
+
+#if defined(__GNUC__)
+// mallctl will be resolved by the runtime linker when jemalloc is being used
+extern "C" {
+  int mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen) __attribute__((weak));
+}
+Napi::Value _isUsingJemalloc(const Napi::CallbackInfo& info) {
+  Napi::Env env = info.Env();
+  return Napi::Boolean::New(env, mallctl != nullptr);
+}
+#else
+Napi::Value _isUsingJemalloc(const Napi::CallbackInfo& info) {
+  Napi::Env env = info.Env();
+  return Napi::Boolean::New(env, false);
+}
+#endif

src/utilities.h

@@ -24,5 +24,6 @@ Napi::Value simd(const Napi::CallbackInfo& info);
 Napi::Value libvipsVersion(const Napi::CallbackInfo& info);
 Napi::Value format(const Napi::CallbackInfo& info);
 Napi::Value _maxColourDistance(const Napi::CallbackInfo& info);
+Napi::Value _isUsingJemalloc(const Napi::CallbackInfo& info);
 
 #endif  // SRC_UTILITIES_H_

test/unit/util.js

@@ -3,8 +3,6 @@
 const assert = require('assert');
 const sharp = require('../../');
 
-const defaultConcurrency = sharp.concurrency();
-
 describe('Utilities', function () {
   describe('Cache', function () {
     it('Can be disabled', function () {
@@ -60,10 +58,10 @@ describe('Utilities', function () {
     });
     it('Can be reset to default', function () {
       sharp.concurrency(0);
-      assert.strictEqual(defaultConcurrency, sharp.concurrency());
+      assert.strictEqual(true, sharp.concurrency() > 0);
     });
     it('Ignores invalid values', function () {
-      sharp.concurrency(0);
+      const defaultConcurrency = sharp.concurrency();
       sharp.concurrency('spoons');
       assert.strictEqual(defaultConcurrency, sharp.concurrency());
     });