From a2b1ff3929d9aca66201108bc684ac5bc394d99a Mon Sep 17 00:00:00 2001 From: Lovell Fuller Date: Sat, 3 Feb 2018 16:06:36 +0000 Subject: [PATCH] Add support for prebuilt sharp binaries on common platforms --- .npmignore | 1 + .prebuildrc | 4 ++ CONTRIBUTING.md | 2 +- README.md | 9 +++- binding.gyp | 103 +++-------------------------------------- binding.js | 108 ------------------------------------------- docs/changelog.md | 7 +++ docs/index.md | 7 +-- docs/install.md | 31 ++++++++++--- install/dll-copy.js | 34 ++++++++++++++ install/libvips.js | 78 +++++++++++++++++++++++++++++++ lib/libvips.js | 59 +++++++++++++++++++++++ package.json | 10 +++- test/unit/libvips.js | 62 +++++++++++++++++++++++++ 14 files changed, 295 insertions(+), 220 deletions(-) create mode 100644 .prebuildrc delete mode 100644 binding.js create mode 100644 install/dll-copy.js create mode 100644 install/libvips.js create mode 100644 lib/libvips.js create mode 100644 test/unit/libvips.js diff --git a/.npmignore b/.npmignore index 0bcda8798..692924808 100644 --- a/.npmignore +++ b/.npmignore @@ -9,5 +9,6 @@ test appveyor.yml mkdocs.yml vendor +.prebuildrc .nyc_output CONTRIBUTING.md diff --git a/.prebuildrc b/.prebuildrc new file mode 100644 index 000000000..e9978aa33 --- /dev/null +++ b/.prebuildrc @@ -0,0 +1,4 @@ +{ + "include-regex": "(sharp\\.node|libvips-cpp\\.dll)", + "strip": true +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 032022217..28db9d0cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,8 +41,8 @@ Any change that modifies the existing public API should be added to the relevant | Release | WIP branch | | ------: | :--------- | -| v0.19.0 | suit | | v0.20.0 | teeth | +| v0.21.0 | uptake | Please squash your changes into a single commit using a command like `git rebase -i upstream/`. diff --git a/README.md b/README.md index 158e4e8ab..f0a7cbacd 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ npm install sharp ``` +```sh +yarn add sharp +``` + The typical use case for this high speed Node.js module is to convert large images in common formats to smaller, web-friendly JPEG, PNG and WebP images of varying dimensions. @@ -17,8 +21,9 @@ Lanczos resampling ensures quality is not sacrificed for speed. As well as image resizing, operations such as rotation, extraction, compositing and gamma correction are available. -OS X, Windows (x64), Linux (x64, ARM) systems do not require -the installation of any external runtime dependencies. +Most modern 64-bit OS X, Windows and Linux (glibc) systems running +Node versions 4, 6, 8 and 9 +do not require any additional install or runtime dependencies. ## Examples diff --git a/binding.gyp b/binding.gyp index 8826c49d3..1115b1cd8 100644 --- a/binding.gyp +++ b/binding.gyp @@ -5,9 +5,6 @@ ['OS == "win"', { # Build libvips C++ binding for Windows due to MSVC std library ABI changes 'type': 'shared_library', - 'variables': { - 'download_vips': '/dev/null 2>&1 && eval $(brew --env) && echo $PKG_CONFIG_LIBDIR || true):$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig:/usr/lib/pkgconfig' - }, { - 'pkg_config_path': '' - }] - ], - }, - 'conditions': [ - ['OS != "win"', { - # Which version, if any, of libvips is available globally via pkg-config? - 'global_vips_version': '/dev/null || true)' - }, { - 'global_vips_version': '' - }] - ], - 'pkg_config_path%': '<(pkg_config_path)' - }, - 'pkg_config_path%': '<(pkg_config_path)', 'runtime_link%': 'shared', 'conditions': [ ['OS != "win"', { - # Does the globally available version of libvips, if any, meet the minimum version requirement? - 'use_global_vips': '= ${minimumLibvipsVersion} - please see http://sharp.pixelplumbing.com/page/install`); - } - // Ensure glibc Linux - if (detectLibc.isNonGlibcLinux) { - error(`Use with ${detectLibc.family} libc requires manual installation of libvips >= ${minimumLibvipsVersion} - please see http://sharp.pixelplumbing.com/page/install`); - } - // Ensure glibc >= 2.13 - if (detectLibc.family === detectLibc.GLIBC && detectLibc.version && semver.lt(`${detectLibc.version}.0`, '2.13.0')) { - error(`Use with glibc version ${detectLibc.version} requires manual installation of libvips >= ${minimumLibvipsVersion} - please see http://sharp.pixelplumbing.com/page/install`); - } - // Arch/platform-specific .tar.gz - const tarFilename = ['libvips', minimumLibvipsVersion, currentPlatformId].join('-') + '.tar.gz'; - // Download to per-process temporary file - const tarPathTemp = path.join(os.tmpdir(), `${process.pid}-${tarFilename}`); - const tmpFile = fs.createWriteStream(tarPathTemp).on('close', function () { - unpack(tarPathTemp, function () { - // Attempt to remove temporary file - try { - fs.unlinkSync(tarPathTemp); - } catch (err) {} - }); - }); - const url = distBaseUrl + tarFilename; - const simpleGetOpt = { - url: url, - agent: agent() - }; - simpleGet(simpleGetOpt, function (err, response) { - if (err) { - error(`${url} download failed: ${err.message}`); - } - if (response.statusCode !== 200) { - error(`${url} download failed: status ${response.statusCode}`); - } - response.pipe(tmpFile); - }); -}; - -module.exports.use_global_vips = function () { - const globalVipsVersion = process.env.GLOBAL_VIPS_VERSION; - if (globalVipsVersion) { - const useGlobalVips = semver.gte( - globalVipsVersion, - minimumLibvipsVersion - ); - process.stdout.write(useGlobalVips ? 'true' : 'false'); - } else { - process.stdout.write('false'); - } -}; diff --git a/docs/changelog.md b/docs/changelog.md index 5d719c63e..26bca9e46 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,12 @@ # Changelog +### v0.20 - "*teeth*" + +#### v0.20.0 - TBD + +* Add support for prebuilt sharp binaries on common platforms. + [#186](https://github.com/lovell/sharp/issues/186) + ### v0.19 - "*suit*" Requires libvips v8.6.1. diff --git a/docs/index.md b/docs/index.md index c72d4abb6..4692bf480 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,8 +13,9 @@ Lanczos resampling ensures quality is not sacrificed for speed. As well as image resizing, operations such as rotation, extraction, compositing and gamma correction are available. -OS X, Windows (x64), Linux (x64, ARM) systems do not require -the installation of any external runtime dependencies. +Most 64-bit OS X, Windows and Linux (glibc) systems running +Node versions 4, 6, 8 and 9 +do not require any additional install or runtime dependencies. [![Test Coverage](https://coveralls.io/repos/lovell/sharp/badge.png?branch=master)](https://coveralls.io/r/lovell/sharp?branch=master) @@ -46,7 +47,7 @@ are held in memory and processed at a time, taking full advantage of multiple CPU cores and L1/L2/L3 cache. Everything remains non-blocking thanks to _libuv_, -no child processes are spawned and Promises/A+ are supported. +no child processes are spawned and Promises/async/await are supported. ### Optimal diff --git a/docs/install.md b/docs/install.md index dc58d4de2..78ef3a486 100644 --- a/docs/install.md +++ b/docs/install.md @@ -8,12 +8,29 @@ npm install sharp yarn add sharp ``` -### Prerequisites +## Prerequisites * Node v4.5.0+ + +### Building from source + +Pre-compiled binaries for sharp are provided for use with +Node versions 4, 6, 8 and 9 on +64-bit Windows, OS X and Linux platforms. + +Sharp will be built from source at install time when: + +* a globally-installed libvips is detected, +* pre-compiled binaries do not exist for the current platform and Node version, or +* when the `npm install --build-from-source` flag is used. + +Building from source requires: + * C++11 compatible compiler such as gcc 4.8+, clang 3.0+ or MSVC 2013+ * [node-gyp](https://github.com/TooTallNate/node-gyp#installation) and its dependencies (includes Python) +## libvips + ### Linux [![Ubuntu 16.04 Build Status](https://travis-ci.org/lovell/sharp.png?branch=master)](https://travis-ci.org/lovell/sharp) @@ -23,14 +40,14 @@ This involves an automated HTTPS download of approximately 7MB. Most recent Linux-based operating systems with glibc running on x64 and ARMv6+ CPUs should "just work", e.g.: -* Debian 7, 8 -* Ubuntu 14.04, 16.04 -* Centos 7 +* Debian 7+ +* Ubuntu 14.04+ +* Centos 7+ * Fedora -* openSUSE 13.2 +* openSUSE 13.2+ * Archlinux * Raspbian Jessie -* Amazon Linux 2017.03.1 +* Amazon Linux * Solus To use a globally-installed version of libvips instead of the provided binaries, @@ -63,7 +80,7 @@ via `sharp.cache(false)` to avoid a stack overflow. ### Mac OS -[![OS X 10.9.5 Build Status](https://travis-ci.org/lovell/sharp.png?branch=master)](https://travis-ci.org/lovell/sharp) +[![OS X 10.12 Build Status](https://travis-ci.org/lovell/sharp.png?branch=master)](https://travis-ci.org/lovell/sharp) libvips and its dependencies are fetched and stored within `node_modules/sharp/vendor` during `npm install`. This involves an automated HTTPS download of approximately 7MB. diff --git a/install/dll-copy.js b/install/dll-copy.js new file mode 100644 index 000000000..a99a8d891 --- /dev/null +++ b/install/dll-copy.js @@ -0,0 +1,34 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const copyFileSync = require('fs-copy-file-sync'); +const npmLog = require('npmlog'); + +if (process.platform === 'win32') { + const buildDir = path.join(__dirname, '..', 'build'); + const buildReleaseDir = path.join(buildDir, 'Release'); + npmLog.info('sharp', `Creating ${buildReleaseDir}`); + try { + fs.mkdirSync(buildDir); + fs.mkdirSync(buildReleaseDir); + } catch (err) {} + const vendorLibDir = path.join(__dirname, '..', 'vendor', 'lib'); + npmLog.info('sharp', `Copying DLLs from ${vendorLibDir} to ${buildReleaseDir}`); + try { + fs + .readdirSync(vendorLibDir) + .filter(function (filename) { + return /\.dll$/.test(filename); + }) + .forEach(function (filename) { + copyFileSync( + path.join(vendorLibDir, filename), + path.join(buildReleaseDir, filename) + ); + }); + } catch (err) { + npmLog.error('sharp', err.message); + } +} diff --git a/install/libvips.js b/install/libvips.js new file mode 100644 index 000000000..231c02db6 --- /dev/null +++ b/install/libvips.js @@ -0,0 +1,78 @@ +'use strict'; + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const detectLibc = require('detect-libc'); +const npmLog = require('npmlog'); +const semver = require('semver'); +const simpleGet = require('simple-get'); +const tar = require('tar'); + +const agent = require('../lib/agent'); +const libvips = require('../lib/libvips'); +const platform = require('../lib/platform'); + +const minimumLibvipsVersion = libvips.minimumLibvipsVersion; +const distBaseUrl = process.env.SHARP_DIST_BASE_URL || `https://github.com/lovell/sharp-libvips/releases/download/v${minimumLibvipsVersion}/`; + +try { + const globalLibvipsVersion = libvips.globalLibvipsVersion(); + if (globalLibvipsVersion) { + npmLog.info('sharp', `Detected globally-installed libvips v${globalLibvipsVersion}`); + npmLog.info('sharp', 'Building from source via node-gyp'); + process.exit(1); + } else if (libvips.hasVendoredLibvips()) { + npmLog.info('sharp', `Using existing vendored libvips v${minimumLibvipsVersion}`); + } else { + // Is this arch/platform supported? + const arch = process.env.npm_config_arch || process.arch; + if (arch === 'ia32') { + throw new Error(`Intel Architecture 32-bit systems require manual installation of libvips >= ${minimumLibvipsVersion}\n`); + } + if (detectLibc.isNonGlibcLinux) { + throw new Error(`Use with ${detectLibc.family} libc requires manual installation of libvips >= ${minimumLibvipsVersion}`); + } + if (detectLibc.family === detectLibc.GLIBC && detectLibc.version && semver.lt(`${detectLibc.version}.0`, '2.13.0')) { + throw new Error(`Use with glibc version ${detectLibc.version} requires manual installation of libvips >= ${minimumLibvipsVersion}`); + } + // Download to per-process temporary file + const tarFilename = ['libvips', minimumLibvipsVersion, platform()].join('-') + '.tar.gz'; + const tarPathTemp = path.join(os.tmpdir(), `${process.pid}-${tarFilename}`); + const tmpFile = fs.createWriteStream(tarPathTemp); + const url = distBaseUrl + tarFilename; + npmLog.info('sharp', `Downloading ${url}`); + simpleGet({ url: url, agent: agent() }, function (err, response) { + if (err) { + throw err; + } + if (response.statusCode !== 200) { + throw new Error(`Status ${response.statusCode}`); + } + response.pipe(tmpFile); + }); + tmpFile.on('close', function () { + const vendorPath = path.join(__dirname, '..', 'vendor'); + fs.mkdirSync(vendorPath); + tar + .extract({ + file: tarPathTemp, + cwd: vendorPath, + strict: true + }) + .then(function () { + try { + fs.unlinkSync(tarPathTemp); + } catch (err) {} + }) + .catch(function (err) { + throw err; + }); + }); + } +} catch (err) { + npmLog.error('sharp', err.message); + npmLog.error('sharp', 'Please see http://sharp.pixelplumbing.com/page/install'); + process.exit(1); +} diff --git a/lib/libvips.js b/lib/libvips.js new file mode 100644 index 000000000..f91a45725 --- /dev/null +++ b/lib/libvips.js @@ -0,0 +1,59 @@ +'use strict'; + +const path = require('path'); +const spawnSync = require('child_process').spawnSync; +const semver = require('semver'); +const platform = require('./platform'); + +const minimumLibvipsVersion = process.env.npm_package_config_libvips || require('../package.json').config.libvips; + +const spawnSyncOptions = { + encoding: 'utf8', + shell: true +}; + +const globalLibvipsVersion = function () { + if (process.platform !== 'win32') { + const globalLibvipsVersion = spawnSync(`PKG_CONFIG_PATH="${pkgConfigPath()}" pkg-config --modversion vips-cpp`, spawnSyncOptions).stdout || ''; + return globalLibvipsVersion.trim(); + } else { + return ''; + } +}; + +const hasVendoredLibvips = function () { + const currentPlatformId = platform(); + try { + const vendorPlatformId = require(path.join(__dirname, '..', 'vendor', 'platform.json')); + if (currentPlatformId === vendorPlatformId) { + return true; + } else { + throw new Error(`'${vendorPlatformId}' binaries cannot be used on the '${currentPlatformId}' platform. Please remove the 'node_modules/sharp/vendor' directory and run 'npm install'.`); + } + } catch (err) {} + return false; +}; + +const pkgConfigPath = function () { + if (process.platform !== 'win32') { + const brewPkgConfigPath = spawnSync('which brew >/dev/null 2>&1 && eval $(brew --env) && echo $PKG_CONFIG_LIBDIR', spawnSyncOptions).stdout || ''; + return [brewPkgConfigPath.trim(), process.env.PKG_CONFIG_PATH, '/usr/local/lib/pkgconfig', '/usr/lib/pkgconfig'] + .filter(function (p) { return !!p; }) + .join(':'); + } else { + return ''; + } +}; + +const useGlobalLibvips = function () { + const globalVipsVersion = globalLibvipsVersion(); + return !!globalVipsVersion && semver.gte(globalVipsVersion, minimumLibvipsVersion); +}; + +module.exports = { + minimumLibvipsVersion: minimumLibvipsVersion, + globalLibvipsVersion: globalLibvipsVersion, + hasVendoredLibvips: hasVendoredLibvips, + pkgConfigPath: pkgConfigPath, + useGlobalLibvips: useGlobalLibvips +}; diff --git a/package.json b/package.json index 7647630c5..19db9ae8b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "sharp", "description": "High performance Node.js image processing, the fastest module to resize JPEG, PNG, WebP and TIFF images", - "version": "0.19.0", + "version": "0.20.0-prebuild.test.1", "author": "Lovell Fuller ", "homepage": "https://github.com/lovell/sharp", "contributors": [ @@ -45,8 +45,9 @@ "Oleh Aleinyk " ], "scripts": { + "install": "(node install/libvips && node install/dll-copy && prebuild-install) || (node-gyp rebuild && node install/dll-copy)", "clean": "rm -rf node_modules/ build/ vendor/ coverage/ test/fixtures/output.*", - "test": "semistandard && cc && nyc --reporter=lcov --branches=99 mocha --slow=5000 --timeout=60000 ./test/unit/*.js", + "test": "semistandard && cc && nyc --reporter=lcov --branches=99 mocha --slow=5000 --timeout=60000 ./test/unit/*.js && prebuild-ci", "coverage": "./test/coverage/report.sh", "test-leak": "./test/leak/leak.sh", "docs": "for m in constructor input resize composite operation colour channel output utility; do documentation build --shallow --format=md lib/$m.js >docs/api-$m.md; done" @@ -75,7 +76,10 @@ "dependencies": { "color": "^2.0.1", "detect-libc": "^1.0.3", + "fs-copy-file-sync": "^1.0.1", "nan": "^2.8.0", + "npmlog": "^4.1.2", + "prebuild-install": "^2.5.0", "semver": "^5.4.1", "simple-get": "^2.7.0", "tar": "^4.2.0", @@ -89,6 +93,8 @@ "icc": "^1.0.0", "mocha": "^4.1.0", "nyc": "^11.4.1", + "prebuild": "^7.4.0", + "prebuild-ci": "^2.2.3", "rimraf": "^2.6.2", "semistandard": "^12.0.0", "unzip": "^0.1.11" diff --git a/test/unit/libvips.js b/test/unit/libvips.js new file mode 100644 index 000000000..4f1a11448 --- /dev/null +++ b/test/unit/libvips.js @@ -0,0 +1,62 @@ +'use strict'; + +const assert = require('assert'); +const semver = require('semver'); +const libvips = require('../../lib/libvips'); + +const originalPlatform = process.platform; + +const setPlatform = function (platform) { + Object.defineProperty(process, 'platform', { value: platform }); +}; + +const restorePlatform = function () { + setPlatform(originalPlatform); +}; + +describe('libvips binaries', function () { + describe('Windows platform', function () { + before(function () { setPlatform('win32'); }); + after(restorePlatform); + + it('pkgConfigPath returns empty string', function () { + assert.strictEqual('', libvips.pkgConfigPath()); + }); + it('globalLibvipsVersion returns empty string', function () { + assert.strictEqual('', libvips.globalLibvipsVersion()); + }); + it('globalLibvipsVersion is always false', function () { + assert.strictEqual(false, libvips.useGlobalLibvips()); + }); + }); + + describe('non-Windows platforms', function () { + before(function () { setPlatform('linux'); }); + after(restorePlatform); + + it('pkgConfigPath returns a string', function () { + const pkgConfigPath = libvips.pkgConfigPath(); + assert.strictEqual('string', typeof pkgConfigPath); + }); + it('globalLibvipsVersion returns a string', function () { + const globalLibvipsVersion = libvips.globalLibvipsVersion(); + assert.strictEqual('string', typeof globalLibvipsVersion); + }); + it('globalLibvipsVersion returns a boolean', function () { + const useGlobalLibvips = libvips.useGlobalLibvips(); + assert.strictEqual('boolean', typeof useGlobalLibvips); + }); + }); + + describe('platform agnostic', function () { + it('minimumLibvipsVersion returns a valid semver', function () { + const minimumLibvipsVersion = libvips.minimumLibvipsVersion; + assert.strictEqual('string', typeof minimumLibvipsVersion); + assert.notStrictEqual(null, semver.valid(minimumLibvipsVersion)); + }); + it('hasVendoredLibvips returns a boolean', function () { + const hasVendoredLibvips = libvips.hasVendoredLibvips(); + assert.strictEqual('boolean', typeof hasVendoredLibvips); + }); + }); +});