New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Reduce size of installed dependencies by slimming #191
Changes from all commits
91d80c3
76a92b0
7b7c429
4362153
8389e2b
b2d50fa
f500910
82c1088
780af23
81e2663
c406b5f
4302c17
1f9ae75
eeee343
e121d17
f12d600
474b823
d3a6dc8
26216d8
83b170c
671cc72
2230ae4
97212b2
89c46ba
c71be73
33433aa
e6a4273
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
const isWsl = require('is-wsl'); | ||
|
||
/** | ||
* Get commands to slim the installed requirements | ||
* only for non-windows platforms: | ||
* works for docker builds and when run on UNIX platforms (wsl included) | ||
* @param {Object} options | ||
* @param {string} folderPath | ||
* @return {Array.<String>} | ||
*/ | ||
function getSlimPackageCommands(options, folderPath) { | ||
let stripCmd = []; | ||
|
||
// Default stripping is done for non-windows environments | ||
if (process.platform !== 'win32' || isWsl) { | ||
stripCmd = getDefaultSLimOptions(folderPath); | ||
|
||
// If specified any custom patterns to remove | ||
if (options.slimPatterns instanceof Array) { | ||
// Add the custom specified patterns to remove to the default commands | ||
const customPatterns = options.slimPatterns.map(pattern => { | ||
return getRemovalCommand(folderPath, pattern); | ||
}); | ||
stripCmd = stripCmd.concat(customPatterns); | ||
} | ||
} | ||
return stripCmd; | ||
} | ||
|
||
/** | ||
* Gets the commands to slim the default (safe) files: | ||
* including removing caches, stripping compiled files, removing dist-infos | ||
* @param {String} folderPath | ||
* @return {Array} | ||
*/ | ||
function getDefaultSLimOptions(folderPath) { | ||
return [ | ||
`&& find ${folderPath} -name "*.so" -exec strip {} \\;`, | ||
`&& find ${folderPath} -name "*.py[c|o]" -exec rm -rf {} +`, | ||
`&& find ${folderPath} -type d -name "__pycache__*" -exec rm -rf {} +`, | ||
`&& find ${folderPath} -type d -name "*.dist-info*" -exec rm -rf {} +` | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe removing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @dee-me-tree-or-love test folders?
|
||
]; | ||
} | ||
|
||
/** | ||
* Get the command created fromt he find and remove template: | ||
* returns a string in form `&& find <folder> -name "<match>" -exec rm -rf {} +` | ||
* @param {String} folderPath | ||
* @param {String} removalMatch | ||
* @return {String} | ||
*/ | ||
function getRemovalCommand(folderPath, removalMatch) { | ||
return `&& find ${folderPath} -type d -name "${removalMatch}" -exec rm -rf {} +`; | ||
} | ||
|
||
module.exports = { | ||
getSlimPackageCommands, | ||
getDefaultSLimOptions | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -32,6 +32,27 @@ teardown() { | |
! ls puck/flask | ||
} | ||
|
||
@test "py3.6 can package flask with slim options" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
sls --slim=true package | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
@test "py3.6 can package flask with slim & slimPatterns options" { | ||
cd tests/base | ||
mv _slimPatterns.yml slimPatterns.yml | ||
npm i $(npm pack ../..) | ||
sls --slim=true package | ||
mv slimPatterns.yml _slimPatterns.yml | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 | ||
} | ||
|
||
@test "py3.6 doesn't package boto3 by default" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -59,6 +80,15 @@ teardown() { | |
ls puck/.requirements.zip puck/unzip_requirements.py | ||
} | ||
|
||
@test "py3.6 can package flask with zip & slim & dockerizePip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" | ||
sls --dockerizePip=true --zip=true --slim=true package | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/.requirements.zip puck/unzip_requirements.py | ||
} | ||
|
||
@test "py3.6 can package flask with dockerizePip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -68,6 +98,29 @@ teardown() { | |
ls puck/flask | ||
} | ||
|
||
@test "py3.6 can package flask with slim & dockerizePip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" | ||
sls --dockerizePip=true --slim=true package | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🙌 Thanks for including tests too!! Could you add this to also check that the unzipped services doesn't contain any pyc files?
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sure, that's an awesome one, thanks, didn't know of this, will do now! ✌️ There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should I put There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yup. My mistake. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @dee-me-tree-or-love, @dschep Wouldn't you want to dump the .py and not .pyc? |
||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
@test "py3.6 can package flask with slim & dockerizePip & slimPatterns options" { | ||
cd tests/base | ||
mv _slimPatterns.yml slimPatterns.yml | ||
npm i $(npm pack ../..) | ||
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" | ||
sls --dockerizePip=true --slim=true package | ||
mv slimPatterns.yml _slimPatterns.yml | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 | ||
} | ||
|
||
@test "py3.6 uses cache with dockerizePip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -77,6 +130,17 @@ teardown() { | |
ls .requirements-cache/http | ||
} | ||
|
||
@test "py3.6 uses cache with dockerizePip & slim option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" | ||
perl -p -i'.bak' -e 's/(pythonRequirements:$)/\1\n pipCmdExtraArgs: ["--cache-dir", ".requirements-cache"]/' serverless.yml | ||
sls --dockerizePip=true --slim=true package | ||
ls .requirements-cache/http | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
|
||
@test "py2.7 can package flask with default options" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -85,6 +149,15 @@ teardown() { | |
ls puck/flask | ||
} | ||
|
||
@test "py2.7 can package flask with slim option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
sls --runtime=python2.7 --slim=true package | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
@test "py2.7 can package flask with zip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -93,6 +166,18 @@ teardown() { | |
ls puck/.requirements.zip puck/unzip_requirements.py | ||
} | ||
|
||
@test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" { | ||
cd tests/base | ||
mv _slimPatterns.yml slimPatterns.yml | ||
npm i $(npm pack ../..) | ||
sls --runtime=python2.7 --slim=true packag | ||
mv slimPatterns.yml _slimPatterns.yml | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 | ||
} | ||
|
||
@test "py2.7 doesn't package boto3 by default" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -119,6 +204,15 @@ teardown() { | |
ls puck/.requirements.zip puck/unzip_requirements.py | ||
} | ||
|
||
@test "py2.7 can package flask with zip & slim & dockerizePip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" | ||
sls --dockerizePip=true --runtime=python2.7 --zip=true --slim=true package | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/.requirements.zip puck/unzip_requirements.py | ||
} | ||
|
||
@test "py2.7 can package flask with dockerizePip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -128,6 +222,29 @@ teardown() { | |
ls puck/flask | ||
} | ||
|
||
@test "py2.7 can package flask with slim & dockerizePip option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" | ||
sls --dockerizePip=true --slim=true --runtime=python2.7 package | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
@test "py2.7 can package flask with slim & dockerizePip & slimPatterns options" { | ||
cd tests/base | ||
mv _slimPatterns.yml slimPatterns.yml | ||
npm i $(npm pack ../..) | ||
! uname -sm|grep Linux || groups|grep docker || id -u|egrep '^0$' || skip "can't dockerize on linux if not root & not in docker group" | ||
sls --dockerizePip=true --slim=true --runtime=python2.7 package | ||
mv slimPatterns.yml _slimPatterns.yml | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 | ||
} | ||
|
||
@test "pipenv py3.6 can package flask with default options" { | ||
cd tests/pipenv | ||
npm i $(npm pack ../..) | ||
|
@@ -136,6 +253,27 @@ teardown() { | |
ls puck/flask | ||
} | ||
|
||
@test "pipenv py3.6 can package flask with slim option" { | ||
cd tests/pipenv | ||
npm i $(npm pack ../..) | ||
sls --slim=true package | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
@test "pipenv py3.6 can package flask with slim & slimPatterns option" { | ||
cd tests/pipenv | ||
npm i $(npm pack ../..) | ||
mv _slimPatterns.yml slimPatterns.yml | ||
sls --slim=true package | ||
mv slimPatterns.yml _slimPatterns.yml | ||
unzip .serverless/sls-py-req-test.zip -d puck | ||
ls puck/flask | ||
test $(find puck -name "*.pyc" | wc -l) -eq 0 | ||
test $(find puck -type d -name "*.egg-info*" | wc -l) -eq 0 | ||
} | ||
|
||
@test "pipenv py3.6 can package flask with zip option" { | ||
cd tests/pipenv | ||
npm i $(npm pack ../..) | ||
|
@@ -182,6 +320,20 @@ teardown() { | |
! ls puck3/flask | ||
} | ||
|
||
@test "py3.6 can package flask with package individually & slim option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
sls --individually=true --slim=true package | ||
unzip .serverless/hello.zip -d puck | ||
unzip .serverless/hello2.zip -d puck2 | ||
unzip .serverless/hello3.zip -d puck3 | ||
ls puck/flask | ||
ls puck2/flask | ||
! ls puck3/flask | ||
test $(find "puck*" -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
|
||
@test "py2.7 can package flask with package individually option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
|
@@ -194,6 +346,20 @@ teardown() { | |
! ls puck3/flask | ||
} | ||
|
||
@test "py2.7 can package flask with package individually & slim option" { | ||
cd tests/base | ||
npm i $(npm pack ../..) | ||
sls --individually=true --slim=true --runtime=python2.7 package | ||
unzip .serverless/hello.zip -d puck | ||
unzip .serverless/hello2.zip -d puck2 | ||
unzip .serverless/hello3.zip -d puck3 | ||
ls puck/flask | ||
ls puck2/flask | ||
! ls puck3/flask | ||
test $(find puck* -name "*.pyc" | wc -l) -eq 0 | ||
} | ||
|
||
|
||
@test "py3.6 can package only requirements of module" { | ||
cd tests/individually | ||
npm i $(npm pack ../..) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
slimPatterns: | ||
- "*.egg-info*" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
slimPatterns: | ||
- "*.egg-info*" |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Now the preparing of the strip command happens inside of this method - in case when the environment does not meet expectations (
win32
systems) returns an empty array.To be honest, I am slightly worried about this - theoretically running these commands inside of CYGWIN or Git Bash (or other bash emulators) should be just fine - but I couldn't find a way yet to check for this... would appreciate any advice! :)