Add infrastructure to build and publish as wasm32 (#3840)

Co-authored-by: Ingvar Stepanyan <me@rreverser.com>

.github/workflows/ci.yml

@@ -161,6 +161,38 @@ jobs:
             npm install --ignore-scripts
             npx mocha --no-config --spec=test/unit/io.js --timeout=30000
             [[ -n $prebuild_upload ]] && cd src && ln -s ../package.json && npx prebuild || true
+  github-runner-emscripten:
+    permissions:
+      contents: write
+    name: wasm32 - prebuild
+    runs-on: ubuntu-22.04
+    container: "emscripten/emsdk:3.1.48"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Dependencies
+        run: apt-get update && apt-get install -y pkg-config
+      - name: Dependencies (Node.js)
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+      - name: Install
+        run: emmake npm install --build-from-source
+      - name: Test
+        run: emmake npm test
+      - name: Test packaging
+        run: |
+          emmake npm run package-from-local-build
+          npm pkg set "optionalDependencies.@img/sharp-wasm32=file:./npm/wasm32"
+          npm run clean
+          npm install --cpu=wasm32
+          npm test
+      - name: Prebuild
+        if: startsWith(github.ref, 'refs/tags/')
+        env:
+          npm_config_nodedir: emscripten
+          prebuild_upload: ${{ secrets.GITHUB_TOKEN }}
+        run: cd src && ln -s ../package.json && emmake npx prebuild --platform=emscripten --arch=wasm32 --strip=0
   macstadium-runner:
     permissions:
       contents: write

docs/changelog.md

@@ -14,6 +14,9 @@ Requires libvips v8.15.0
 
 * Remove `sharp.vendor`.
 
+* Add experimental support for WebAssembly-based runtimes.
+  [@RReverser](https://github.com/RReverser)
+
 * Options for `trim` operation must be an Object, add new `lineArt` option.
   [#2363](https://github.com/lovell/sharp/issues/2363)
 

docs/humans.txt

@@ -278,3 +278,6 @@ GitHub: https://github.com/bianjunjie1981
 
 Name: Dennis Beatty
 GitHub: https://github.com/dnsbty
+
+Name: Ingvar Stepanyan
+GitHub: https://github.com/RReverser

docs/install.md

@@ -78,6 +78,15 @@ For cross-compiling, the `--platform`, `--arch` and `--libc` npm flags
 (or the `npm_config_platform`, `npm_config_arch` and `npm_config_libc` environment variables)
 can be used to configure the target environment.
 
+## WebAssembly
+
+Experimental support is provided for runtime environments that provide
+multi-threaded Wasm via Workers.
+
+```sh
+npm install --cpu=wasm32 sharp
+```
+
 ## FreeBSD
 
 The `vips` package must be installed before `npm install` is run.

lib/libvips.js

@@ -41,15 +41,23 @@ const runtimePlatformArch = () => `${process.platform}${runtimeLibc()}-${process
 
 /* istanbul ignore next */
 const buildPlatformArch = () => {
+  if (isEmscripten()) {
+    return 'wasm32';
+  }
   /* eslint camelcase: ["error", { allow: ["^npm_config_"] }] */
   const { npm_config_arch, npm_config_platform, npm_config_libc } = process.env;
-  return `${npm_config_platform || process.platform}${npm_config_libc || runtimeLibc()}-${npm_config_arch || process.arch}`;
+  const libc = typeof npm_config_libc === 'string' ? npm_config_libc : runtimeLibc();
+  return `${npm_config_platform || process.platform}${libc}-${npm_config_arch || process.arch}`;
 };
 
 const buildSharpLibvipsIncludeDir = () => {
   try {
-    return require('@img/sharp-libvips-dev/include');
+    return require(`@img/sharp-libvips-dev-${buildPlatformArch()}/include`);
-  } catch {}
+  } catch {
+    try {
+      return require('@img/sharp-libvips-dev/include');
+    } catch {}
+  }
   /* istanbul ignore next */
   return '';
 };
@@ -64,12 +72,22 @@ const buildSharpLibvipsCPlusPlusDir = () => {
 
 const buildSharpLibvipsLibDir = () => {
   try {
-    return require(`@img/sharp-libvips-${buildPlatformArch()}/lib`);
+    return require(`@img/sharp-libvips-dev-${buildPlatformArch()}/lib`);
-  } catch {}
+  } catch {
+    try {
+      return require(`@img/sharp-libvips-${buildPlatformArch()}/lib`);
+    } catch {}
+  }
   /* istanbul ignore next */
   return '';
 };
 
+/* istanbul ignore next */
+const isEmscripten = () => {
+  const { CC } = process.env;
+  return Boolean(CC && CC.endsWith('/emcc'));
+};
+
 const isRosetta = () => {
   /* istanbul ignore next */
   if (process.platform === 'darwin' && process.arch === 'x64') {
@@ -81,7 +99,7 @@ const isRosetta = () => {
 
 /* istanbul ignore next */
 const spawnRebuild = () =>
-  spawnSync('node-gyp rebuild --directory=src', {
+  spawnSync(`node-gyp rebuild --directory=src ${isEmscripten() ? '--nodedir=emscripten' : ''}`, {
     ...spawnSyncOptions,
     stdio: 'inherit'
   }).status;

lib/sharp.js

@@ -12,7 +12,9 @@ const runtimePlatform = runtimePlatformArch();
 
 const paths = [
   `../src/build/Release/sharp-${runtimePlatform}.node`,
-  `@img/sharp-${runtimePlatform}/sharp.node`
+  '../src/build/Release/sharp-wasm32.node',
+  `@img/sharp-${runtimePlatform}/sharp.node`,
+  '@img/sharp-wasm32/sharp.node'
 ];
 
 const errors = [];
@@ -47,6 +49,8 @@ if (!module.exports) {
     help.push(`    npm install --force @img/sharp-${runtimePlatform}`);
   } else {
     help.push(`- Manually install libvips >= ${minimumLibvipsVersion}`);
+    help.push('- Add experimental WebAssembly-based dependencies:');
+    help.push('    npm install --cpu=wasm32 sharp');
   }
   if (isLinux && /symbol not found/i.test(messages)) {
     try {

lib/utility.js

@@ -59,11 +59,17 @@ let versions = {
 };
 /* istanbul ignore next */
 if (!libvipsVersion.isGlobal) {
-  try {
+  if (!libvipsVersion.isWasm) {
-    versions = require(`@img/sharp-${runtimePlatform}/versions`);
-  } catch (_) {
     try {
-      versions = require(`@img/sharp-libvips-${runtimePlatform}/versions`);
+      versions = require(`@img/sharp-${runtimePlatform}/versions`);
+    } catch (_) {
+      try {
+        versions = require(`@img/sharp-libvips-${runtimePlatform}/versions`);
+      } catch (_) {}
+    }
+  } else {
+    try {
+      versions = require('@img/sharp-wasm32/versions');
     } catch (_) {}
   }
 }

npm/from-github-release.js

@@ -38,7 +38,8 @@ limitations under the License.
 `;
 
 workspaces.map(async platform => {
-  const url = `https://github.com/lovell/sharp/releases/download/v${version}/sharp-v${version}-napi-v9-${platform}.tar.gz`;
+  const prebuildPlatform = platform === 'wasm32' ? 'emscripten-wasm32' : platform;
+  const url = `https://github.com/lovell/sharp/releases/download/v${version}/sharp-v${version}-napi-v9-${prebuildPlatform}.tar.gz`;
   const dir = path.join(__dirname, platform);
   const response = await fetch(url);
   if (!response.ok) {
@@ -58,8 +59,8 @@ workspaces.map(async platform => {
   await writeFile(path.join(dir, 'README.md'), `# \`${name}\`\n\n${description}.\n${licensing}`);
   // Copy Apache-2.0 LICENSE
   await copyFile(path.join(__dirname, '..', 'LICENSE'), path.join(dir, 'LICENSE'));
-  // Copy Windows-specific files
+  // Copy files for packages without an explicit sharp-libvips dependency (Windows, wasm)
-  if (platform.startsWith('win32-')) {
+  if (platform.startsWith('win') || platform.startsWith('wasm')) {
     const sharpLibvipsDir = path.join(require(`@img/sharp-libvips-${platform}/lib`), '..');
     // Copy versions.json
     await copyFile(path.join(sharpLibvipsDir, 'versions.json'), path.join(dir, 'versions.json'));

npm/package.json

@@ -11,6 +11,7 @@
     "linux-x64",
     "linuxmusl-arm64",
     "linuxmusl-x64",
+    "wasm32",
     "win32-ia32",
     "win32-x64"
   ]

npm/wasm32/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@img/sharp-wasm32",
+  "version": "0.33.0-alpha.10",
+  "description": "Prebuilt sharp for use with wasm32",
+  "author": "Lovell Fuller <npm@lovell.info>",
+  "homepage": "https://sharp.pixelplumbing.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/lovell/sharp.git",
+    "directory": "npm/wasm32"
+  },
+  "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT",
+  "funding": {
+    "url": "https://opencollective.com/libvips"
+  },
+  "preferUnplugged": true,
+  "files": [
+    "lib",
+    "versions.json"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "commonjs",
+  "exports": {
+    "./sharp.node": "./lib/sharp-wasm32.node.js",
+    "./package": "./package.json",
+    "./versions": "./versions.json"
+  },
+  "engines": {
+    "node": "^18.17.0 || ^20.3.0 || >=21.0.0",
+    "npm": ">=9.6.5",
+    "yarn": ">=3.2.0",
+    "pnpm": ">=7.1.0"
+  },
+  "dependencies": {
+    "@emnapi/runtime": "^0.43.1"
+  },
+  "cpu": [
+    "wasm32"
+  ]
+}

package.json

@@ -87,7 +87,8 @@
     "Brahim Ait elhaj <brahima@gmail.com>",
     "Mart Jansink <m.jansink@gmail.com>",
     "Lachlan Newman <lachnewman007@gmail.com>",
-    "Dennis Beatty <dennis@dcbeatty.com>"
+    "Dennis Beatty <dennis@dcbeatty.com>",
+    "Ingvar Stepanyan <me@rreverser.com>"
   ],
   "scripts": {
     "install": "node install/check",
@@ -156,16 +157,20 @@
     "@img/sharp-linux-x64": "0.33.0-alpha.10",
     "@img/sharp-linuxmusl-arm64": "0.33.0-alpha.10",
     "@img/sharp-linuxmusl-x64": "0.33.0-alpha.10",
+    "@img/sharp-wasm32": "0.33.0-alpha.10",
     "@img/sharp-win32-ia32": "0.33.0-alpha.10",
     "@img/sharp-win32-x64": "0.33.0-alpha.10"
   },
   "devDependencies": {
+    "@emnapi/runtime": "^0.43.1",
     "@img/sharp-libvips-dev": "0.0.3",
+    "@img/sharp-libvips-dev-wasm32": "0.0.3",
     "@img/sharp-libvips-win32-ia32": "0.0.3",
     "@img/sharp-libvips-win32-x64": "0.0.3",
     "@types/node": "*",
     "async": "^3.2.5",
     "cc": "^3.0.1",
+    "emnapi": "^0.43.1",
     "exif-reader": "^2.0.0",
     "extract-zip": "^2.0.1",
     "icc": "^3.0.0",
@@ -203,6 +208,11 @@
       "build/include"
     ]
   },
+  "nyc": {
+    "include": [
+      "lib"
+    ]
+  },
   "tsd": {
     "directory": "test/types/"
   }

src/binding.gyp

@@ -179,6 +179,26 @@
                 '-Wl,-rpath=\'$$ORIGIN/../../../node_modules/@img/sharp-libvips-<(platform_and_arch)/lib\''
               ]
             }
+          }],
+          ['OS == "emscripten"', {
+            'product_extension': 'node.js',
+            'link_settings': {
+              'ldflags': [
+                '-fexceptions',
+                '--pre-js=<!(node -p "require.resolve(\'./emscripten/pre.js\')")',
+                '-Oz',
+                '-sALLOW_MEMORY_GROWTH',
+                '-sENVIRONMENT=node',
+                '-sEXPORTED_FUNCTIONS=["_vips_shutdown", "_uv_library_shutdown"]',
+                '-sNODERAWFS',
+                '-sTEXTDECODER=0',
+                '-sWASM_ASYNC_COMPILATION=0',
+                '-sWASM_BIGINT'
+              ],
+              'libraries': [
+                '<!@(PKG_CONFIG_PATH="<!(node -p "require(\'@img/sharp-libvips-dev-wasm32/lib\')")/pkgconfig" pkg-config --static --libs vips-cpp)'
+              ],
+            }
           }]
         ]
       }]

src/emscripten/common.gypi

@@ -0,0 +1,40 @@
+# Copyright 2013 Lovell Fuller and others.
+# SPDX-License-Identifier: Apache-2.0
+
+{
+  'variables': {
+    'OS': 'emscripten'
+  },
+  'target_defaults': {
+    'default_configuration': 'Release',
+    'type': 'executable',
+    'cflags': [
+      '-pthread',
+      '-sDEFAULT_TO_CXX=0'
+    ],
+    'cflags_cc': [
+      '-pthread'
+    ],
+    'ldflags': [
+      '--js-library=<!(node -p "require(\'emnapi\').js_library")',
+      '-sAUTO_JS_LIBRARIES=0',
+      '-sAUTO_NATIVE_LIBRARIES=0',
+      '-sNODEJS_CATCH_EXIT=0',
+      '-sNODEJS_CATCH_REJECTION=0'
+    ],
+    'defines': [
+      '__STDC_FORMAT_MACROS',
+      'BUILDING_NODE_EXTENSION',
+      'EMNAPI_WORKER_POOL_SIZE=1'
+    ],
+    'include_dirs': [
+      '<!(node -p "require(\'emnapi\').include")'
+    ],
+    'sources': [
+      '<!@(node -p "require(\'emnapi\').sources.map(x => JSON.stringify(path.relative(process.cwd(), x))).join(\' \')")'
+    ],
+    'configurations': {
+      'Release': {}
+    }
+  }
+}

src/emscripten/pre.js

@@ -0,0 +1,19 @@
+// Copyright 2013 Lovell Fuller and others.
+// SPDX-License-Identifier: Apache-2.0
+
+/* global Module, ENV, _vips_shutdown, _uv_library_shutdown */
+
+Module.preRun = () => {
+  ENV.VIPS_CONCURRENCY = Number(process.env.VIPS_CONCURRENCY) || 1;
+};
+
+Module.onRuntimeInitialized = () => {
+  module.exports = Module.emnapiInit({
+    context: require('@emnapi/runtime').getDefaultContext()
+  });
+
+  process.once('exit', () => {
+    _vips_shutdown();
+    _uv_library_shutdown();
+  });
+};

src/utilities.cc

@@ -102,6 +102,11 @@ Napi::Value libvipsVersion(const Napi::CallbackInfo& info) {
   version.Set("isGlobal", Napi::Boolean::New(env, true));
 #else
   version.Set("isGlobal", Napi::Boolean::New(env, false));
+#endif
+#ifdef __EMSCRIPTEN__
+  version.Set("isWasm", Napi::Boolean::New(env, true));
+#else
+  version.Set("isWasm", Napi::Boolean::New(env, false));
 #endif
   return version;
 }

test/unit/libvips.js

@@ -69,17 +69,25 @@ describe('libvips binaries', function () {
   });
 
   describe('Build time platform detection', () => {
-    it('Can override platform with npm_config_platform and npm_config_libc', () => {
+    it('Can override platform with npm_config_platform and npm_config_libc', function () {
       process.env.npm_config_platform = 'testplatform';
       process.env.npm_config_libc = 'testlibc';
-      const [platform] = libvips.buildPlatformArch().split('-');
+      const platformArch = libvips.buildPlatformArch();
+      if (platformArch === 'wasm32') {
+        return this.skip();
+      }
+      const [platform] = platformArch.split('-');
       assert.strictEqual(platform, 'testplatformtestlibc');
       delete process.env.npm_config_platform;
       delete process.env.npm_config_libc;
     });
-    it('Can override arch with npm_config_arch', () => {
+    it('Can override arch with npm_config_arch', function () {
       process.env.npm_config_arch = 'test';
-      const [, arch] = libvips.buildPlatformArch().split('-');
+      const platformArch = libvips.buildPlatformArch();
+      if (platformArch === 'wasm32') {
+        return this.skip();
+      }
+      const [, arch] = platformArch.split('-');
       assert.strictEqual(arch, 'test');
       delete process.env.npm_config_arch;
     });