Distribute prebuilt binaries via the npm registry #3750

- Remove all custom download logic for prebuilt binaries
- Add scripts to populate package contents
- Specify minimum versions of common package managers
- Remove sharp.vendor runtime API as no-longer relevant
- Update installation docs and issue templates

lib/agent.js

@@ -1,44 +0,0 @@
-// Copyright 2013 Lovell Fuller and others.
-// SPDX-License-Identifier: Apache-2.0
-
-'use strict';
-
-const url = require('url');
-const tunnelAgent = require('tunnel-agent');
-
-const is = require('./is');
-
-const proxies = [
-  'HTTPS_PROXY',
-  'https_proxy',
-  'HTTP_PROXY',
-  'http_proxy',
-  'npm_config_https_proxy',
-  'npm_config_proxy'
-];
-
-function env (key) {
-  return process.env[key];
-}
-
-module.exports = function (log) {
-  try {
-    const proxy = new url.URL(proxies.map(env).find(is.string));
-    const tunnel = proxy.protocol === 'https:'
-      ? tunnelAgent.httpsOverHttps
-      : tunnelAgent.httpsOverHttp;
-    const proxyAuth = proxy.username && proxy.password
-      ? `${decodeURIComponent(proxy.username)}:${decodeURIComponent(proxy.password)}`
-      : null;
-    log(`Via proxy ${proxy.protocol}//${proxy.hostname}:${proxy.port} ${proxyAuth ? 'with' : 'no'} credentials`);
-    return tunnel({
-      proxy: {
-        port: Number(proxy.port),
-        host: proxy.hostname,
-        proxyAuth
-      }
-    });
-  } catch (err) {
-    return null;
-  }
-};

lib/constructor.js

@@ -7,7 +7,6 @@ const util = require('util');
 const stream = require('stream');
 const is = require('./is');
 
-require('./libvips').hasVendoredLibvips();
 require('./sharp');
 
 // Use NODE_DEBUG=sharp to enable libvips warnings

lib/index.d.ts

@@ -91,12 +91,6 @@ declare namespace sharp {
         zlib?: string | undefined;
     };
 
-    /** An Object containing the platform and architecture of the current and installed vendored binaries. */
-    const vendor: {
-        current: string;
-        installed: string[];
-    };
-
     /** An Object containing the available interpolators and their proper values */
     const interpolators: Interpolators;
 

lib/libvips.js

@@ -3,53 +3,30 @@
 
 'use strict';
 
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
 const spawnSync = require('child_process').spawnSync;
 const semverCoerce = require('semver/functions/coerce');
 const semverGreaterThanOrEqualTo = require('semver/functions/gte');
+const detectLibc = require('detect-libc');
 
-const platform = require('./platform');
-const { config } = require('../package.json');
+const { engines } = require('../package.json');
 
-const env = process.env;
-const minimumLibvipsVersionLabelled = env.npm_package_config_libvips || /* istanbul ignore next */
-  config.libvips;
+const minimumLibvipsVersionLabelled = process.env.npm_package_config_libvips || /* istanbul ignore next */
+  engines.libvips;
 const minimumLibvipsVersion = semverCoerce(minimumLibvipsVersionLabelled).version;
 
+const prebuiltPlatforms = [
+  'darwin-arm64', 'darwin-x64',
+  'linux-arm', 'linux-arm64', 'linux-x64',
+  'linuxmusl-arm64', 'linuxmusl-x64',
+  'win32-ia32', 'win32-x64'
+];
+
 const spawnSyncOptions = {
   encoding: 'utf8',
   shell: true
 };
 
-const vendorPath = path.join(__dirname, '..', 'vendor', minimumLibvipsVersion, platform());
-
-const mkdirSync = function (dirPath) {
-  try {
-    fs.mkdirSync(dirPath, { recursive: true });
-  } catch (err) {
-    /* istanbul ignore next */
-    if (err.code !== 'EEXIST') {
-      throw err;
-    }
-  }
-};
-
-const cachePath = function () {
-  const npmCachePath = env.npm_config_cache || /* istanbul ignore next */
-    (env.APPDATA ? path.join(env.APPDATA, 'npm-cache') : path.join(os.homedir(), '.npm'));
-  mkdirSync(npmCachePath);
-  const libvipsCachePath = path.join(npmCachePath, '_libvips');
-  mkdirSync(libvipsCachePath);
-  return libvipsCachePath;
-};
-
-const integrity = function (platformAndArch) {
-  return env[`npm_package_config_integrity_${platformAndArch.replace('-', '_')}`] || config.integrity[platformAndArch];
-};
-
-const log = function (item) {
+const log = (item) => {
   if (item instanceof Error) {
     console.error(`sharp: Installation error: ${item.message}`);
   } else {
@@ -57,7 +34,43 @@ const log = function (item) {
   }
 };
 
-const isRosetta = function () {
+/* istanbul ignore next */
+const runtimeLibc = () => detectLibc.isNonGlibcLinuxSync() ? detectLibc.familySync() : '';
+
+const runtimePlatformArch = () => `${process.platform}${runtimeLibc()}-${process.arch}`;
+
+/* istanbul ignore next */
+const buildPlatformArch = () => {
+  /* eslint camelcase: ["error", { allow: ["^npm_config_"] }] */
+  const { npm_config_arch, npm_config_platform, npm_config_libc } = process.env;
+  return `${npm_config_platform || process.platform}${npm_config_libc || runtimeLibc()}-${npm_config_arch || process.arch}`;
+};
+
+const buildSharpLibvipsIncludeDir = () => {
+  try {
+    return require('@sharpen/sharp-libvips-dev/include');
+  } catch {}
+  /* istanbul ignore next */
+  return '';
+};
+
+const buildSharpLibvipsCPlusPlusDir = () => {
+  try {
+    return require('@sharpen/sharp-libvips-dev/cplusplus');
+  } catch {}
+  /* istanbul ignore next */
+  return '';
+};
+
+const buildSharpLibvipsLibDir = () => {
+  try {
+    return require(`@sharpen/sharp-libvips-${buildPlatformArch()}/lib`);
+  } catch {}
+  /* istanbul ignore next */
+  return '';
+};
+
+const isRosetta = () => {
   /* istanbul ignore next */
   if (process.platform === 'darwin' && process.arch === 'x64') {
     const translated = spawnSync('sysctl sysctl.proc_translated', spawnSyncOptions).stdout;
@@ -66,12 +79,19 @@ const isRosetta = function () {
   return false;
 };
 
-const globalLibvipsVersion = function () {
+/* istanbul ignore next */
+const gypRebuild = () =>
+  spawnSync('node-gyp rebuild', {
+    ...spawnSyncOptions,
+    stdio: 'inherit'
+  }).status;
+
+const globalLibvipsVersion = () => {
   if (process.platform !== 'win32') {
     const globalLibvipsVersion = spawnSync('pkg-config --modversion vips-cpp', {
       ...spawnSyncOptions,
       env: {
-        ...env,
+        ...process.env,
         PKG_CONFIG_PATH: pkgConfigPath()
       }
     }).stdout;
@@ -82,17 +102,8 @@ const globalLibvipsVersion = function () {
   }
 };
 
-const hasVendoredLibvips = function () {
-  return fs.existsSync(vendorPath);
-};
-
 /* istanbul ignore next */
-const removeVendoredLibvips = function () {
-  fs.rmSync(vendorPath, { recursive: true, maxRetries: 3, force: true });
-};
-
-/* istanbul ignore next */
-const pkgConfigPath = function () {
+const pkgConfigPath = () => {
   if (process.platform !== 'win32') {
     const brewPkgConfigPath = spawnSync(
       'which brew >/dev/null 2>&1 && brew environment --plain | grep PKG_CONFIG_LIBDIR | cut -d" " -f2',
@@ -100,7 +111,7 @@ const pkgConfigPath = function () {
     ).stdout || '';
     return [
       brewPkgConfigPath.trim(),
-      env.PKG_CONFIG_PATH,
+      process.env.PKG_CONFIG_PATH,
       '/usr/local/lib/pkgconfig',
       '/usr/lib/pkgconfig',
       '/usr/local/libdata/pkgconfig',
@@ -111,8 +122,9 @@ const pkgConfigPath = function () {
   }
 };
 
-const useGlobalLibvips = function () {
-  if (Boolean(env.SHARP_IGNORE_GLOBAL_LIBVIPS) === true) {
+const useGlobalLibvips = () => {
+  if (Boolean(process.env.SHARP_IGNORE_GLOBAL_LIBVIPS) === true) {
+    log('Detected SHARP_IGNORE_GLOBAL_LIBVIPS, skipping search for globally-installed libvips');
     return false;
   }
   /* istanbul ignore next */
@@ -127,14 +139,15 @@ const useGlobalLibvips = function () {
 
 module.exports = {
   minimumLibvipsVersion,
-  minimumLibvipsVersionLabelled,
-  cachePath,
-  integrity,
+  prebuiltPlatforms,
+  buildPlatformArch,
+  buildSharpLibvipsIncludeDir,
+  buildSharpLibvipsCPlusPlusDir,
+  buildSharpLibvipsLibDir,
+  runtimePlatformArch,
   log,
+  gypRebuild,
   globalLibvipsVersion,
-  hasVendoredLibvips,
-  removeVendoredLibvips,
   pkgConfigPath,
-  useGlobalLibvips,
-  mkdirSync
+  useGlobalLibvips
 };

lib/platform.js

@@ -1,30 +0,0 @@
-// Copyright 2013 Lovell Fuller and others.
-// SPDX-License-Identifier: Apache-2.0
-
-'use strict';
-
-const detectLibc = require('detect-libc');
-
-const env = process.env;
-
-module.exports = function () {
-  const arch = env.npm_config_arch || process.arch;
-  const platform = env.npm_config_platform || process.platform;
-  const libc = process.env.npm_config_libc ||
-    /* istanbul ignore next */
-    (detectLibc.isNonGlibcLinuxSync() ? detectLibc.familySync() : '');
-  const libcId = platform !== 'linux' || libc === detectLibc.GLIBC ? '' : libc;
-
-  const platformId = [`${platform}${libcId}`];
-
-  if (arch === 'arm') {
-    const fallback = process.versions.electron ? '7' : '6';
-    platformId.push(`armv${env.npm_config_arm_version || process.config.variables.arm_version || fallback}`);
-  } else if (arch === 'arm64') {
-    platformId.push(`arm64v${env.npm_config_arm_version || '8'}`);
-  } else {
-    platformId.push(arch);
-  }
-
-  return platformId.join('-');
-};

lib/sharp.js

@@ -3,36 +3,58 @@
 
 'use strict';
 
-const platformAndArch = require('./platform')();
+// Inspects the runtime environment and exports the relevant sharp.node binary
+
+const { familySync, versionSync } = require('detect-libc');
+
+const { runtimePlatformArch, prebuiltPlatforms, minimumLibvipsVersion } = require('./libvips');
+const runtimePlatform = runtimePlatformArch();
 
 /* istanbul ignore next */
 try {
-  module.exports = require(`../build/Release/sharp-${platformAndArch}.node`);
-} catch (err) {
-  // Bail early if bindings aren't available
-  const help = ['', 'Something went wrong installing the "sharp" module', '', err.message, '', 'Possible solutions:'];
-  if (/dylib/.test(err.message) && /Incompatible library version/.test(err.message)) {
-    help.push('- Update Homebrew: "brew update && brew upgrade vips"');
-  } else {
-    const [platform, arch] = platformAndArch.split('-');
-    if (platform === 'linux' && /Module did not self-register/.test(err.message)) {
-      help.push('- Using worker threads? See https://sharp.pixelplumbing.com/install#worker-threads');
+  // Check for local build
+  module.exports = require(`../build/Release/sharp-${runtimePlatform}.node`);
+} catch (errLocal) {
+  try {
+    // Check for runtime package
+    module.exports = require(`@sharpen/sharp-${runtimePlatform}/sharp.node`);
+  } catch (errPackage) {
+    const help = ['Could not load the "sharp" module at runtime'];
+    if (errLocal.code !== 'MODULE_NOT_FOUND') {
+      help.push(`${errLocal.code}: ${errLocal.message}`);
     }
-    help.push(
-      '- Install with verbose logging and look for errors: "npm install --ignore-scripts=false --foreground-scripts --verbose sharp"',
-      `- Install for the current ${platformAndArch} runtime: "npm install --platform=${platform} --arch=${arch} sharp"`
-    );
-  }
-  help.push(
-    '- Consult the installation documentation: https://sharp.pixelplumbing.com/install'
-  );
-  // Check loaded
-  if (process.platform === 'win32' || /symbol/.test(err.message)) {
-    const loadedModule = Object.keys(require.cache).find((i) => /[\\/]build[\\/]Release[\\/]sharp(.*)\.node$/.test(i));
-    if (loadedModule) {
-      const [, loadedPackage] = loadedModule.match(/node_modules[\\/]([^\\/]+)[\\/]/);
-      help.push(`- Ensure the version of sharp aligns with the ${loadedPackage} package: "npm ls sharp"`);
+    if (errPackage.code !== 'MODULE_NOT_FOUND') {
+      help.push(`${errPackage.code}: ${errPackage.message}`);
     }
+    help.push('Possible solutions:');
+    // Common error messages
+    if (prebuiltPlatforms.includes(runtimePlatform)) {
+      help.push(`- Add an explicit dependency for the runtime platform: "npm install --force @sharpen/sharp-${runtimePlatform}"`);
+    } else {
+      help.push(`- The ${runtimePlatform} platform requires manual installation of libvips >= ${minimumLibvipsVersion}`);
+    }
+    if (runtimePlatform.startsWith('linux') && /symbol not found/i.test(errPackage)) {
+      try {
+        const { engines } = require(`@sharpen/sharp-libvips-${runtimePlatform}/package`);
+        const libcFound = `${familySync()} ${versionSync()}`;
+        const libcRequires = `${engines.musl ? 'musl' : 'glibc'} ${engines.musl || engines.glibc}`;
+        help.push(`- Update your OS: found ${libcFound}, requires ${libcRequires}`);
+      } catch (errEngines) {}
+    }
+    if (runtimePlatform.startsWith('darwin') && /Incompatible library version/.test(errLocal.message)) {
+      help.push('- Update Homebrew: "brew update && brew upgrade vips"');
+    }
+    if (errPackage.code === 'ERR_DLOPEN_DISABLED') {
+      help.push('- Run Node.js without using the --no-addons flag');
+    }
+    // Link to installation docs
+    if (runtimePlatform.startsWith('linux') && /Module did not self-register/.test(errLocal.message + errPackage.message)) {
+      help.push('- Using worker threads on Linux? See https://sharp.pixelplumbing.com/install#worker-threads');
+    } else if (runtimePlatform.startsWith('win32') && /The specified procedure could not be found/.test(errPackage.message)) {
+      help.push('- Using the canvas package on Windows? See https://sharp.pixelplumbing.com/install#canvas-and-windows');
+    } else {
+      help.push('- Consult the installation documentation: https://sharp.pixelplumbing.com/install');
+    }
+    throw new Error(help.join('\n'));
   }
-  throw new Error(help.join('\n'));
 }

lib/utility.js

@@ -3,13 +3,11 @@
 
 'use strict';
 
-const fs = require('fs');
-const path = require('path');
 const events = require('events');
 const detectLibc = require('detect-libc');
 
 const is = require('./is');
-const platformAndArch = require('./platform')();
+const { runtimePlatformArch } = require('./libvips');
 const sharp = require('./sharp');
 
 /**
@@ -55,25 +53,14 @@ let versions = {
   vips: sharp.libvipsVersion()
 };
 try {
-  versions = require(`../vendor/${versions.vips}/${platformAndArch}/versions.json`);
-} catch (_err) { /* ignore */ }
+  versions = require(`@sharpen/sharp-${runtimePlatformArch()}/versions`);
+} catch (_) {
+  try {
+    versions = require(`@sharpen/sharp-libvips-${runtimePlatformArch()}/versions`);
+  } catch (_) {}
+}
 versions.sharp = require('../package.json').version;
 
-/**
- * An Object containing the platform and architecture
- * of the current and installed vendored binaries.
- * @member
- * @example
- * console.log(sharp.vendor);
- */
-const vendor = {
-  current: platformAndArch,
-  installed: []
-};
-try {
-  vendor.installed = fs.readdirSync(path.join(__dirname, `../vendor/${versions.vips}`));
-} catch (_err) { /* ignore */ }
-
 /**
  * Gets or, when options are provided, sets the limits of _libvips'_ operation cache.
  * Existing entries in the cache will be trimmed after any change in limits.
@@ -280,7 +267,6 @@ module.exports = function (Sharp) {
   Sharp.format = format;
   Sharp.interpolators = interpolators;
   Sharp.versions = versions;
-  Sharp.vendor = vendor;
   Sharp.queue = queue;
   Sharp.block = block;
   Sharp.unblock = unblock;