diff --git a/README.md b/README.md index a4e40d1d..8c9bf04f 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,28 @@ try: except ImportError: pass ``` - +### Slim Package +_Works on non 'win32' environments: Docker, WSL are included_ +To remove the tests, information and caches from the installed packages, +enable the `slim` option. This will: `strip` the `.so` files, remove `__pycache__` +directories and `dist-info` directories. +```yaml +custom: + pythonRequirements: + slim: true +``` +#### Custom Removal Patterns +To specify additional directories to remove from the installed packages, +define the patterns using regex as a `slimPatterns` option in serverless config: +```yaml +custom: + pythonRequirements: + slim: true + slimPatterns: + - "*.egg-info*" +``` +This will remove all folders within the installed requirements that match +the names in `slimPatterns` ## Omitting Packages You can omit a package from deployment with the `noDeploy` option. Note that dependencies of omitted packages must explicitly be omitted too. @@ -266,3 +287,5 @@ For usage of `dockerizePip` on Windows do Step 1 only if running serverless on w * [@kichik](https://github.com/kichik) - Imposed windows & `noDeploy` support, switched to adding files straight to zip instead of creating symlinks, and improved pip chache support when using docker. + * [@dee-me-tree-or-love](https://github.com/dee-me-tree-or-love) - the `slim` package option + diff --git a/index.js b/index.js index 33e6a3da..bff0994f 100644 --- a/index.js +++ b/index.js @@ -26,6 +26,7 @@ class ServerlessPythonRequirements { get options() { const options = Object.assign( { + slim: false, zip: false, cleanupZipHelper: true, invalidateCaches: false, diff --git a/lib/pip.js b/lib/pip.js index d9cbee34..2b061371 100644 --- a/lib/pip.js +++ b/lib/pip.js @@ -6,6 +6,7 @@ const set = require('lodash.set'); const { spawnSync } = require('child_process'); const values = require('lodash.values'); const { buildImage, getBindPath, getDockerUid } = require('./docker'); +const { getSlimPackageCommands } = require('./slim'); /** * Install requirements described in requirementsPath to targetFolder @@ -122,6 +123,14 @@ function installRequirements( cmd = pipCmd[0]; cmdOptions = pipCmd.slice(1); } + + // If enabled slimming, strip out the caches, tests and dist-infos + if (options.slim === true || options.slim === 'true') { + const preparedPath = dockerPathForWin(options, targetRequirementsFolder); + const slimCmd = getSlimPackageCommands(options, preparedPath); + cmdOptions.push(...slimCmd); + } + const res = spawnSync(cmd, cmdOptions, { cwd: servicePath, shell: true }); if (res.error) { if (res.error.code === 'ENOENT') { diff --git a/lib/slim.js b/lib/slim.js new file mode 100644 index 00000000..e42a884f --- /dev/null +++ b/lib/slim.js @@ -0,0 +1,59 @@ +const isWsl = require('is-wsl'); + +/** + * Get commands to slim the installed requirements + * only for non-windows platforms: + * works for docker builds and when run on UNIX platforms (wsl included) + * @param {Object} options + * @param {string} folderPath + * @return {Array.} + */ +function getSlimPackageCommands(options, folderPath) { + let stripCmd = []; + + // Default stripping is done for non-windows environments + if (process.platform !== 'win32' || isWsl) { + stripCmd = getDefaultSLimOptions(folderPath); + + // If specified any custom patterns to remove + if (options.slimPatterns instanceof Array) { + // Add the custom specified patterns to remove to the default commands + const customPatterns = options.slimPatterns.map(pattern => { + return getRemovalCommand(folderPath, pattern); + }); + stripCmd = stripCmd.concat(customPatterns); + } + } + return stripCmd; +} + +/** + * Gets the commands to slim the default (safe) files: + * including removing caches, stripping compiled files, removing dist-infos + * @param {String} folderPath + * @return {Array} + */ +function getDefaultSLimOptions(folderPath) { + return [ + `&& find ${folderPath} -name "*.so" -exec strip {} \\;`, + `&& find ${folderPath} -name "*.py[c|o]" -exec rm -rf {} +`, + `&& find ${folderPath} -type d -name "__pycache__*" -exec rm -rf {} +`, + `&& find ${folderPath} -type d -name "*.dist-info*" -exec rm -rf {} +` + ]; +} + +/** + * Get the command created fromt he find and remove template: + * returns a string in form `&& find -name "" -exec rm -rf {} +` + * @param {String} folderPath + * @param {String} removalMatch + * @return {String} + */ +function getRemovalCommand(folderPath, removalMatch) { + return `&& find ${folderPath} -type d -name "${removalMatch}" -exec rm -rf {} +`; +} + +module.exports = { + getSlimPackageCommands, + getDefaultSLimOptions +}; diff --git a/test.bats b/test.bats index 3ce2b990..4501ba52 100755 --- a/test.bats +++ b/test.bats @@ -32,6 +32,27 @@ teardown() { ! ls puck/flask } +@test "py3.6 can package flask with slim options" { + cd tests/base + npm i $(npm pack ../..) + sls --slim=true package + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 +} + +@test "py3.6 can package flask with slim & slimPatterns options" { + cd tests/base + mv _slimPatterns.yml slimPatterns.yml + npm i $(npm pack ../..) + sls --slim=true package + mv slimPatterns.yml _slimPatterns.yml + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 + test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 +} + @test "py3.6 doesn't package boto3 by default" { cd tests/base npm i $(npm pack ../..) @@ -59,6 +80,15 @@ teardown() { ls puck/.requirements.zip puck/unzip_requirements.py } +@test "py3.6 can package flask with zip & slim & dockerizePip option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + sls --dockerizePip=true --zip=true --slim=true package + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/.requirements.zip puck/unzip_requirements.py +} + @test "py3.6 can package flask with dockerizePip option" { cd tests/base npm i $(npm pack ../..) @@ -68,6 +98,29 @@ teardown() { ls puck/flask } +@test "py3.6 can package flask with slim & dockerizePip option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + sls --dockerizePip=true --slim=true package + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 +} + +@test "py3.6 can package flask with slim & dockerizePip & slimPatterns options" { + cd tests/base + mv _slimPatterns.yml slimPatterns.yml + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + sls --dockerizePip=true --slim=true package + mv slimPatterns.yml _slimPatterns.yml + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 + test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 +} + @test "py3.6 uses cache with dockerizePip option" { cd tests/base npm i $(npm pack ../..) @@ -77,6 +130,17 @@ teardown() { ls .requirements-cache/http } +@test "py3.6 uses cache with dockerizePip & slim option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n pipCmdExtraArgs: ["--cache-dir", ".requirements-cache"]/' serverless.yml + sls --dockerizePip=true --slim=true package + ls .requirements-cache/http + test $(find puck -name "*.pyc" | wc -l) -eq 0 +} + + @test "py2.7 can package flask with default options" { cd tests/base npm i $(npm pack ../..) @@ -85,6 +149,15 @@ teardown() { ls puck/flask } +@test "py2.7 can package flask with slim option" { + cd tests/base + npm i $(npm pack ../..) + sls --runtime=python2.7 --slim=true package + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 +} + @test "py2.7 can package flask with zip option" { cd tests/base npm i $(npm pack ../..) @@ -93,6 +166,18 @@ teardown() { ls puck/.requirements.zip puck/unzip_requirements.py } +@test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" { + cd tests/base + mv _slimPatterns.yml slimPatterns.yml + npm i $(npm pack ../..) + sls --runtime=python2.7 --slim=true packag + mv slimPatterns.yml _slimPatterns.yml + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 + test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 +} + @test "py2.7 doesn't package boto3 by default" { cd tests/base npm i $(npm pack ../..) @@ -119,6 +204,15 @@ teardown() { ls puck/.requirements.zip puck/unzip_requirements.py } +@test "py2.7 can package flask with zip & slim & dockerizePip option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + sls --dockerizePip=true --runtime=python2.7 --zip=true --slim=true package + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/.requirements.zip puck/unzip_requirements.py +} + @test "py2.7 can package flask with dockerizePip option" { cd tests/base npm i $(npm pack ../..) @@ -128,6 +222,29 @@ teardown() { ls puck/flask } +@test "py2.7 can package flask with slim & dockerizePip option" { + cd tests/base + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + sls --dockerizePip=true --slim=true --runtime=python2.7 package + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 +} + +@test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" { + cd tests/base + mv _slimPatterns.yml slimPatterns.yml + npm i $(npm pack ../..) + ! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" + sls --dockerizePip=true --slim=true --runtime=python2.7 package + mv slimPatterns.yml _slimPatterns.yml + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 + test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 +} + @test "pipenv py3.6 can package flask with default options" { cd tests/pipenv npm i $(npm pack ../..) @@ -136,6 +253,27 @@ teardown() { ls puck/flask } +@test "pipenv py3.6 can package flask with slim option" { + cd tests/pipenv + npm i $(npm pack ../..) + sls --slim=true package + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 +} + +@test "pipenv py3.6 can package flask with slim & slimPatterns option" { + cd tests/pipenv + npm i $(npm pack ../..) + mv _slimPatterns.yml slimPatterns.yml + sls --slim=true package + mv slimPatterns.yml _slimPatterns.yml + unzip .serverless/sls-py-req-test.zip -d puck + ls puck/flask + test $(find puck -name "*.pyc" | wc -l) -eq 0 + test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 +} + @test "pipenv py3.6 can package flask with zip option" { cd tests/pipenv npm i $(npm pack ../..) @@ -182,6 +320,20 @@ teardown() { ! ls puck3/flask } +@test "py3.6 can package flask with package individually & slim option" { + cd tests/base + npm i $(npm pack ../..) + sls --individually=true --slim=true package + unzip .serverless/hello.zip -d puck + unzip .serverless/hello2.zip -d puck2 + unzip .serverless/hello3.zip -d puck3 + ls puck/flask + ls puck2/flask + ! ls puck3/flask + test $(find "puck*" -name "*.pyc" | wc -l) -eq 0 +} + + @test "py2.7 can package flask with package individually option" { cd tests/base npm i $(npm pack ../..) @@ -194,6 +346,20 @@ teardown() { ! ls puck3/flask } +@test "py2.7 can package flask with package individually & slim option" { + cd tests/base + npm i $(npm pack ../..) + sls --individually=true --slim=true --runtime=python2.7 package + unzip .serverless/hello.zip -d puck + unzip .serverless/hello2.zip -d puck2 + unzip .serverless/hello3.zip -d puck3 + ls puck/flask + ls puck2/flask + ! ls puck3/flask + test $(find puck* -name "*.pyc" | wc -l) -eq 0 +} + + @test "py3.6 can package only requirements of module" { cd tests/individually npm i $(npm pack ../..) diff --git a/tests/base/_slimPatterns.yml b/tests/base/_slimPatterns.yml new file mode 100644 index 00000000..ffc3c134 --- /dev/null +++ b/tests/base/_slimPatterns.yml @@ -0,0 +1,2 @@ +slimPatterns: + - "*.egg-info*" \ No newline at end of file diff --git a/tests/base/serverless.yml b/tests/base/serverless.yml index ea22c46e..7c864714 100644 --- a/tests/base/serverless.yml +++ b/tests/base/serverless.yml @@ -10,8 +10,12 @@ custom: pythonRequirements: zip: ${opt:zip, self:custom.defaults.zip} dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip} + slim: ${opt:slim, self:custom.defaults.slim} + slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} vendor: ${opt:vendor, ''} defaults: + slim: false + slimPatterns: false zip: false dockerizePip: false individually: false diff --git a/tests/pipenv/_slimPatterns.yml b/tests/pipenv/_slimPatterns.yml new file mode 100644 index 00000000..ffc3c134 --- /dev/null +++ b/tests/pipenv/_slimPatterns.yml @@ -0,0 +1,2 @@ +slimPatterns: + - "*.egg-info*" \ No newline at end of file diff --git a/tests/pipenv/serverless.yml b/tests/pipenv/serverless.yml index b8ebb38b..feb7f9de 100644 --- a/tests/pipenv/serverless.yml +++ b/tests/pipenv/serverless.yml @@ -9,9 +9,13 @@ plugins: custom: pythonRequirements: zip: ${opt:zip, self:custom.defaults.zip} + slim: ${opt:slim, self:custom.defaults.slim} + slimPatterns: ${file(./slimPatterns.yml):slimPatterns, self:custom.defaults.slimPatterns} dockerizePip: ${opt:dockerizePip, self:custom.defaults.dockerizePip} defaults: zip: false + slimPatterns: false + slim: false dockerizePip: false package: