diff --git a/.gitignore b/.gitignore index 71a86646..5b7a0db7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ coverage # OSX .DS_Store + +# Python +*.pyc \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index e9a932ce..9ac8ef6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: node_js + node_js: - "10" - "8" @@ -7,8 +8,9 @@ sudo: required services: - docker +cache: npm + addons: - chrome: stable apt: sources: - ubuntu-toolchain-r-test diff --git a/.vscode/launch.json b/.vscode/launch.json index 54410e47..9daf5665 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,33 +8,39 @@ "type": "node", "request": "launch", "name": "SQS", - "program": "${workspaceFolder}/node_modules/.bin/serverless", - "cwd": "${workspaceFolder}/packages/serverless-offline-sqs/example", + "program": "${workspaceFolder}/packages/serverless-offline-sqs-integration/node_modules/.bin/serverless", + "cwd": "${workspaceFolder}/packages/serverless-offline-sqs-integration", + "env": { + "PATH": "${env:PATH}:${workspaceFolder}/packages/serverless-offline-sqs-integration/node_modules/.bin" + }, "args": [ - "offline", - "start" + "offline" ] }, { "type": "node", "request": "launch", "name": "DynamoDB Streams", - "program": "${workspaceFolder}/node_modules/.bin/serverless", - "cwd": "${workspaceFolder}/packages/serverless-offline-dynamodb-streams/example", + "program": "${workspaceFolder}/packages/serverless-offline-dynamodb-streams-integration/node_modules/.bin/serverless", + "cwd": "${workspaceFolder}/packages/serverless-offline-dynamodb-streams-integration", + "env": { + "PATH": "${env:PATH}:${workspaceFolder}/packages/serverless-offline-dynamodb-streams-integration/node_modules/.bin" + }, "args": [ - "offline", - "start" + "offline" ] }, { "type": "node", "request": "launch", "name": "Kinesis", - "program": "${workspaceFolder}/node_modules/.bin/serverless", - "cwd": "${workspaceFolder}/packages/serverless-offline-kinesis/example", + "program": "${workspaceFolder}/packages/serverless-offline-kinesis-integration/node_modules/.bin/serverless", + "cwd": "${workspaceFolder}/packages/serverless-offline-kinesis-integration", + "env": { + "PATH": "${env:PATH}:${workspaceFolder}/packages/serverless-offline-kinesis-integration/node_modules/.bin" + }, "args": [ - "offline", - "start" + "offline" ] } ] diff --git a/packages/dynamodb-streams-readable/src/index.js b/packages/dynamodb-streams-readable/src/index.js index 59033cda..d70986f5 100644 --- a/packages/dynamodb-streams-readable/src/index.js +++ b/packages/dynamodb-streams-readable/src/index.js @@ -105,6 +105,12 @@ function DynamoDBStreamReadable(client, arn, options) { pending--; if (err) { + if (err.name === 'TrimmedDataAccessException') { + return describeStream(function(e) { + if (e) return checkpoint.emit('error', e); + read(callback); + }); + } return callback(err); } diff --git a/packages/serverless-apigateway-access-logs/package.json b/packages/serverless-apigateway-access-logs/package.json index f8cb9e30..ac608e50 100644 --- a/packages/serverless-apigateway-access-logs/package.json +++ b/packages/serverless-apigateway-access-logs/package.json @@ -17,7 +17,7 @@ "author": "Adrien Becchis @Coorpacademy (https://github.com/AdrieanKhisbe)", "license": "MIT", "dependencies": { - "aws-sdk": "^2.443.0", + "aws-sdk": "^2.444.0", "lodash": "^4.17.11" }, "keywords": [ diff --git a/packages/serverless-offline-dynamodb-streams/example/README.md b/packages/serverless-offline-dynamodb-streams-integration/README.md similarity index 100% rename from packages/serverless-offline-dynamodb-streams/example/README.md rename to packages/serverless-offline-dynamodb-streams-integration/README.md diff --git a/packages/serverless-offline-dynamodb-streams/example/create-tables.sh b/packages/serverless-offline-dynamodb-streams-integration/create-tables.sh similarity index 96% rename from packages/serverless-offline-dynamodb-streams/example/create-tables.sh rename to packages/serverless-offline-dynamodb-streams-integration/create-tables.sh index cc88302b..705635e3 100644 --- a/packages/serverless-offline-dynamodb-streams/example/create-tables.sh +++ b/packages/serverless-offline-dynamodb-streams-integration/create-tables.sh @@ -16,8 +16,8 @@ do --provisioned-throughput ReadCapacityUnits=1,WriteCapacityUnits=1 \ --stream-specification StreamEnabled=true,StreamViewType=NEW_AND_OLD_IMAGES \ > /dev/null 2> /dev/null - done -done & + done & +done wait diff --git a/packages/serverless-offline-dynamodb-streams/docker-compose.yml b/packages/serverless-offline-dynamodb-streams-integration/docker-compose.yml similarity index 76% rename from packages/serverless-offline-dynamodb-streams/docker-compose.yml rename to packages/serverless-offline-dynamodb-streams-integration/docker-compose.yml index 814a8ed0..463c3166 100644 --- a/packages/serverless-offline-dynamodb-streams/docker-compose.yml +++ b/packages/serverless-offline-dynamodb-streams-integration/docker-compose.yml @@ -10,8 +10,8 @@ services: - dynamodb entrypoint: sh volumes: - - ./example/:/project - - ./example/create-tables.sh:/project/create-tables.sh:ro + - ./:/project + - ./create-tables.sh:/project/create-tables.sh:ro environment: - AWS_ACCESS_KEY_ID=local - AWS_SECRET_ACCESS_KEY=local diff --git a/packages/serverless-offline-dynamodb-streams/example/handler.js b/packages/serverless-offline-dynamodb-streams-integration/handler.js similarity index 100% rename from packages/serverless-offline-dynamodb-streams/example/handler.js rename to packages/serverless-offline-dynamodb-streams-integration/handler.js diff --git a/packages/serverless-offline-dynamodb-streams-integration/handler.py b/packages/serverless-offline-dynamodb-streams-integration/handler.py new file mode 100644 index 00000000..c779f0fd --- /dev/null +++ b/packages/serverless-offline-dynamodb-streams-integration/handler.py @@ -0,0 +1,5 @@ +import json + +def handler(event, context): + print(event) + return \ No newline at end of file diff --git a/packages/serverless-offline-dynamodb-streams-integration/package-lock.json b/packages/serverless-offline-dynamodb-streams-integration/package-lock.json new file mode 100644 index 00000000..b121c9b6 --- /dev/null +++ b/packages/serverless-offline-dynamodb-streams-integration/package-lock.json @@ -0,0 +1,103 @@ +{ + "name": "serverless-offline-dynamodb-streams-integration", + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "aws-sdk": { + "version": "2.444.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.444.0.tgz", + "integrity": "sha512-3vdC7l5BJ3zHzVNgtIxD+TDviti/sAA/1T8zAXAm2XhZ7AePR5lYIMNAwqu+J44Nm6PFSK1QNSzRQ6A4/6b9eA==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.8", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "base64-js": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" + }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "ieee754": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", + "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } + } + } +} diff --git a/packages/serverless-offline-dynamodb-streams-integration/package.json b/packages/serverless-offline-dynamodb-streams-integration/package.json new file mode 100644 index 00000000..35e0956e --- /dev/null +++ b/packages/serverless-offline-dynamodb-streams-integration/package.json @@ -0,0 +1,19 @@ +{ + "name": "serverless-offline-dynamodb-streams-integration", + "private": true, + "scripts": { + "start": "sls offline", + "pretest": "docker-compose stop dynamodb && docker-compose rm -f dynamodb && docker-compose run dynamodb-create", + "test": "node ./test.js" + }, + "dependencies": { + "aws-sdk": "^2.444.0", + "figures": "^3.0.0", + "lodash": "^4.17.11", + "serverless": "^1.41.1", + "serverless-offline": "^4.9.4", + "serverless-offline-dynamodb-streams": "1.5.1", + "signal-exit": "^3.0.2" + } + } + \ No newline at end of file diff --git a/packages/serverless-offline-dynamodb-streams/example/serverless.yml b/packages/serverless-offline-dynamodb-streams-integration/serverless.yml similarity index 92% rename from packages/serverless-offline-dynamodb-streams/example/serverless.yml rename to packages/serverless-offline-dynamodb-streams-integration/serverless.yml index ddda1f77..98a89396 100644 --- a/packages/serverless-offline-dynamodb-streams/example/serverless.yml +++ b/packages/serverless-offline-dynamodb-streams-integration/serverless.yml @@ -29,6 +29,10 @@ functions: Fn::GetAtt: - MyThirdTable - Arn + myPythonHandler: + runtime: python2.7 + handler: handler.handler + events: - stream: type: dynamodb arn: diff --git a/packages/serverless-offline-dynamodb-streams-integration/test.js b/packages/serverless-offline-dynamodb-streams-integration/test.js new file mode 100755 index 00000000..909a8209 --- /dev/null +++ b/packages/serverless-offline-dynamodb-streams-integration/test.js @@ -0,0 +1,79 @@ +/* eslint-disable unicorn/no-process-exit */ +const {Writable} = require('stream'); +const {spawn} = require('child_process'); +const onExit = require('signal-exit'); +const {DynamoDB} = require('aws-sdk'); + +const client = new DynamoDB({ + region: 'eu-west-1', + accessKeyId: 'local', + secretAccessKey: 'local', + endpoint: 'http://localhost:8000' +}); + +const putItems = () => { + return Promise.all([ + client + .putItem({ + Item: {id: {S: 'MyFirstId'}}, + TableName: 'MyFirstTable' + }) + .promise(), + client + .putItem({ + Item: {id: {S: 'MySecondId'}}, + TableName: 'MySecondTable' + }) + .promise(), + client + .putItem({ + Item: {id: {S: 'MyThirdId'}}, + TableName: 'MyThirdTable' + }) + .promise(), + client + .putItem({ + Item: {id: {S: 'MyFourthId'}}, + TableName: 'MyFourthTable' + }) + .promise() + ]); +}; + +const serverless = spawn('sls', ['offline'], { + stdio: ['pipe', 'pipe', 'pipe'], + cwd: __dirname +}); + +serverless.stdout.pipe( + new Writable({ + write(chunk, enc, cb) { + const output = chunk.toString(); + + if (/Offline listening on/.test(output)) { + putItems(); + } + + this.count = (this.count || 0) + (output.match(/\[✔\]/g) || []).length; + if (this.count === 4) serverless.kill(); + cb(); + } + }) +); + +serverless.stdout.on('data', data => { + console.log(data.toString()); +}); + +serverless.stderr.on('data', data => { + console.error(data.toString()); + process.exit(1); +}); + +serverless.on('close', code => { + process.exit(code); +}); + +onExit((code, signal) => { + if (signal) serverless.kill(signal); +}); diff --git a/packages/serverless-offline-dynamodb-streams/README.md b/packages/serverless-offline-dynamodb-streams/README.md index 2ec516fb..fc5f0e76 100644 --- a/packages/serverless-offline-dynamodb-streams/README.md +++ b/packages/serverless-offline-dynamodb-streams/README.md @@ -23,7 +23,7 @@ plugins: - serverless-offline ``` -[See example](./example/README.md) +[See example](../serverless-offline-dynamodb-streams-integration/README.md) ## Configure diff --git a/packages/serverless-offline-dynamodb-streams/example/package.json b/packages/serverless-offline-dynamodb-streams/example/package.json deleted file mode 100644 index e9f5e918..00000000 --- a/packages/serverless-offline-dynamodb-streams/example/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "scripts": { - "start": "../../../node_modules/.bin/serverless offline", - "pretest": "docker-compose stop dynamodb && docker-compose rm -f dynamodb && docker-compose run dynamodb-create", - "test": "if test $(npm run start -- --exec ./test.sh | sed -n '/[✔]/p' | wc -l) != 4 ; then exit 1; fi;", - "test": "npm run start -- --exec ./test.sh" - } -} \ No newline at end of file diff --git a/packages/serverless-offline-dynamodb-streams/example/test.sh b/packages/serverless-offline-dynamodb-streams/example/test.sh deleted file mode 100755 index 21cd0ecd..00000000 --- a/packages/serverless-offline-dynamodb-streams/example/test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -trap "exit 1" INT - -aws dynamodb --endpoint-url http://localhost:8000 put-item --table-name MyFirstTable --item '{"id": {"S": "MyFirstId"}}' & -aws dynamodb --endpoint-url http://localhost:8000 put-item --table-name MySecondTable --item '{"id": {"S": "MySecondId"}}' & -aws dynamodb --endpoint-url http://localhost:8000 put-item --table-name MyThirdTable --item '{"id": {"S": "MyThirdId"}}' & -aws dynamodb --endpoint-url http://localhost:8000 put-item --table-name MyFourthTable --item '{"id": {"S": "MyFourthId"}}' & -wait - -trap - INT \ No newline at end of file diff --git a/packages/serverless-offline-dynamodb-streams/package.json b/packages/serverless-offline-dynamodb-streams/package.json index e7830511..e7585b95 100644 --- a/packages/serverless-offline-dynamodb-streams/package.json +++ b/packages/serverless-offline-dynamodb-streams/package.json @@ -16,11 +16,8 @@ ], "author": "godu", "license": "MIT", - "scripts": { - "test": "cd example && npm test" - }, "dependencies": { - "aws-sdk": "^2.443.0", + "aws-sdk": "^2.444.0", "dynamodb-streams-readable": "^1.0.3", "figures": "^3.0.0", "lodash": "^4.17.11", diff --git a/packages/serverless-offline-kinesis/example/README.md b/packages/serverless-offline-kinesis-integration/README.md similarity index 100% rename from packages/serverless-offline-kinesis/example/README.md rename to packages/serverless-offline-kinesis-integration/README.md diff --git a/packages/serverless-offline-kinesis/example/create-streams.sh b/packages/serverless-offline-kinesis-integration/create-streams.sh similarity index 95% rename from packages/serverless-offline-kinesis/example/create-streams.sh rename to packages/serverless-offline-kinesis-integration/create-streams.sh index 64bff316..2f9e6012 100644 --- a/packages/serverless-offline-kinesis/example/create-streams.sh +++ b/packages/serverless-offline-kinesis-integration/create-streams.sh @@ -13,8 +13,8 @@ do --stream-name ${STREAM} \ --shard-count 1 \ > /dev/null 2> /dev/null - done -done & + done & +done wait diff --git a/packages/serverless-offline-kinesis/docker-compose.yml b/packages/serverless-offline-kinesis-integration/docker-compose.yml similarity index 75% rename from packages/serverless-offline-kinesis/docker-compose.yml rename to packages/serverless-offline-kinesis-integration/docker-compose.yml index 55d7b851..e3b49ef1 100644 --- a/packages/serverless-offline-kinesis/docker-compose.yml +++ b/packages/serverless-offline-kinesis-integration/docker-compose.yml @@ -10,8 +10,8 @@ services: - kinesis entrypoint: sh volumes: - - ./example/:/project - - ./example/create-streams.sh:/project/create-streams.sh:ro + - ./:/project + - ./create-streams.sh:/project/create-streams.sh:ro environment: - AWS_ACCESS_KEY_ID=local - AWS_SECRET_ACCESS_KEY=local diff --git a/packages/serverless-offline-kinesis/example/handler.js b/packages/serverless-offline-kinesis-integration/handler.js similarity index 100% rename from packages/serverless-offline-kinesis/example/handler.js rename to packages/serverless-offline-kinesis-integration/handler.js diff --git a/packages/serverless-offline-kinesis-integration/handler.py b/packages/serverless-offline-kinesis-integration/handler.py new file mode 100644 index 00000000..c779f0fd --- /dev/null +++ b/packages/serverless-offline-kinesis-integration/handler.py @@ -0,0 +1,5 @@ +import json + +def handler(event, context): + print(event) + return \ No newline at end of file diff --git a/packages/serverless-offline-kinesis-integration/package.json b/packages/serverless-offline-kinesis-integration/package.json new file mode 100644 index 00000000..2fa19356 --- /dev/null +++ b/packages/serverless-offline-kinesis-integration/package.json @@ -0,0 +1,19 @@ +{ + + "name": "serverless-offline-kinesis-integration", + "private": true, + "scripts": { + "start": "sls offline", + "pretest": "docker-compose stop kinesis && docker-compose rm -f kinesis && docker-compose run kinesis-create", + "test": "node ./test.js" + }, + "dependencies": { + "aws-sdk": "^2.444.0", + "figures": "^3.0.0", + "lodash": "^4.17.11", + "serverless": "^1.41.1", + "serverless-offline": "^4.9.4", + "serverless-offline-kinesis": "1.5.1", + "signal-exit": "^3.0.2" + } +} diff --git a/packages/serverless-offline-kinesis/example/serverless.yml b/packages/serverless-offline-kinesis-integration/serverless.yml similarity index 90% rename from packages/serverless-offline-kinesis/example/serverless.yml rename to packages/serverless-offline-kinesis-integration/serverless.yml index 5b5773df..b1e26064 100644 --- a/packages/serverless-offline-kinesis/example/serverless.yml +++ b/packages/serverless-offline-kinesis-integration/serverless.yml @@ -29,6 +29,10 @@ functions: Fn::GetAtt: - MyThirdStream - Arn + myPythonHandler: + runtime: python2.7 + handler: handler.handler + events: - stream: type: kinesis arn: diff --git a/packages/serverless-offline-kinesis-integration/test.js b/packages/serverless-offline-kinesis-integration/test.js new file mode 100755 index 00000000..a509abe0 --- /dev/null +++ b/packages/serverless-offline-kinesis-integration/test.js @@ -0,0 +1,83 @@ +/* eslint-disable unicorn/no-process-exit */ +const {Writable} = require('stream'); +const {spawn} = require('child_process'); +const onExit = require('signal-exit'); +const {Kinesis} = require('aws-sdk'); + +const client = new Kinesis({ + region: 'eu-west-1', + accessKeyId: 'local', + secretAccessKey: 'local', + endpoint: 'http://localhost:4567' +}); + +const putRecords = () => { + return Promise.all([ + client + .putRecord({ + StreamName: 'MyFirstStream', + PartitionKey: 'MyFirstMessage', + Data: 'MyFirstMessage' + }) + .promise(), + client + .putRecord({ + StreamName: 'MySecondStream', + PartitionKey: 'MySecondMessage', + Data: 'MySecondMessage' + }) + .promise(), + client + .putRecord({ + StreamName: 'MyThirdStream', + PartitionKey: 'MyThirdMessage', + Data: 'MyThirdMessage' + }) + .promise(), + client + .putRecord({ + StreamName: 'MyFourthStream', + PartitionKey: 'MyFourthMessage', + Data: 'MyFourthMessage' + }) + .promise() + ]); +}; + +const serverless = spawn('sls', ['offline'], { + stdio: ['pipe', 'pipe', 'pipe'], + cwd: __dirname +}); + +serverless.stdout.pipe( + new Writable({ + write(chunk, enc, cb) { + const output = chunk.toString(); + + if (/Offline listening on/.test(output)) { + putRecords(); + } + + this.count = (this.count || 0) + (output.match(/\[✔\]/g) || []).length; + if (this.count === 4) serverless.kill(); + cb(); + } + }) +); + +serverless.stdout.on('data', data => { + console.log(data.toString()); +}); + +serverless.stderr.on('data', data => { + console.error(data.toString()); + process.exit(1); +}); + +serverless.on('close', code => { + process.exit(code); +}); + +onExit((code, signal) => { + if (signal) serverless.kill(signal); +}); diff --git a/packages/serverless-offline-kinesis/README.md b/packages/serverless-offline-kinesis/README.md index a5e3a8c3..9ec68255 100644 --- a/packages/serverless-offline-kinesis/README.md +++ b/packages/serverless-offline-kinesis/README.md @@ -23,7 +23,7 @@ plugins: - serverless-offline ``` -[See example](./example/README.md) +[See example](../serverless-offline-kinesis-integration/README.md) ## Configure diff --git a/packages/serverless-offline-kinesis/example/package.json b/packages/serverless-offline-kinesis/example/package.json deleted file mode 100644 index 98a80bf2..00000000 --- a/packages/serverless-offline-kinesis/example/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "scripts": { - "start": "../../../node_modules/.bin/serverless offline", - "pretest": "docker-compose stop kinesis && docker-compose rm -f kinesis && docker-compose run kinesis-create", - "test": "if test $(npm run start -- --exec ./test.sh | sed -n '/[✔]/p' | wc -l) != 4 ; then exit 1; fi;", - "test": "npm run start -- --exec ./test.sh" - } -} \ No newline at end of file diff --git a/packages/serverless-offline-kinesis/example/test.sh b/packages/serverless-offline-kinesis/example/test.sh deleted file mode 100755 index 106a5fe6..00000000 --- a/packages/serverless-offline-kinesis/example/test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -trap "exit 1" INT - -aws kinesis --endpoint-url http://localhost:4567 put-record --stream-name MyFirstStream --partition-key "MyFirstMessage" --data "MyFirstMessage" & -aws kinesis --endpoint-url http://localhost:4567 put-record --stream-name MySecondStream --partition-key "MySecondMessage" --data "MySecondMessage" & -aws kinesis --endpoint-url http://localhost:4567 put-record --stream-name MyThirdStream --partition-key "MyThirdMessage" --data "MyThirdMessage" & -aws kinesis --endpoint-url http://localhost:4567 put-record --stream-name MyFourthStream --partition-key "MyFourthMessage" --data "MyFourthMessage" & -wait - -trap - INT \ No newline at end of file diff --git a/packages/serverless-offline-kinesis/package.json b/packages/serverless-offline-kinesis/package.json index 052ce8ff..f7e87621 100644 --- a/packages/serverless-offline-kinesis/package.json +++ b/packages/serverless-offline-kinesis/package.json @@ -16,11 +16,8 @@ ], "author": "godu", "license": "MIT", - "scripts": { - "test": "cd example && npm test" - }, "dependencies": { - "aws-sdk": "^2.443.0", + "aws-sdk": "^2.444.0", "figures": "^3.0.0", "kinesis-readable": "^1.2.0", "lodash": "^4.17.11", diff --git a/packages/serverless-offline-sqs/example/README.md b/packages/serverless-offline-sqs-integration/README.md similarity index 100% rename from packages/serverless-offline-sqs/example/README.md rename to packages/serverless-offline-sqs-integration/README.md diff --git a/packages/serverless-offline-sqs/example/create-queues.sh b/packages/serverless-offline-sqs-integration/create-queues.sh similarity index 94% rename from packages/serverless-offline-sqs/example/create-queues.sh rename to packages/serverless-offline-sqs-integration/create-queues.sh index 637710b5..f02ac728 100644 --- a/packages/serverless-offline-sqs/example/create-queues.sh +++ b/packages/serverless-offline-sqs-integration/create-queues.sh @@ -12,8 +12,8 @@ do aws sqs --endpoint-url ${AWS_ENDPOINT_URL} create-queue \ --queue-name ${QUEUE} \ > /dev/null 2> /dev/null - done -done & + done & +done wait diff --git a/packages/serverless-offline-sqs/docker-compose.yml b/packages/serverless-offline-sqs-integration/docker-compose.yml similarity index 74% rename from packages/serverless-offline-sqs/docker-compose.yml rename to packages/serverless-offline-sqs-integration/docker-compose.yml index 9f5d94d0..9b8f82b5 100644 --- a/packages/serverless-offline-sqs/docker-compose.yml +++ b/packages/serverless-offline-sqs-integration/docker-compose.yml @@ -10,8 +10,8 @@ services: - sqs entrypoint: sh volumes: - - ./example/:/project - - ./example/create-queues.sh:/project/create-queues.sh:ro + - ./:/project + - ./create-queues.sh:/project/create-queues.sh:ro environment: - AWS_ACCESS_KEY_ID=local - AWS_SECRET_ACCESS_KEY=local diff --git a/packages/serverless-offline-sqs/example/handler.js b/packages/serverless-offline-sqs-integration/handler.js similarity index 100% rename from packages/serverless-offline-sqs/example/handler.js rename to packages/serverless-offline-sqs-integration/handler.js diff --git a/packages/serverless-offline-sqs-integration/handler.py b/packages/serverless-offline-sqs-integration/handler.py new file mode 100644 index 00000000..c779f0fd --- /dev/null +++ b/packages/serverless-offline-sqs-integration/handler.py @@ -0,0 +1,5 @@ +import json + +def handler(event, context): + print(event) + return \ No newline at end of file diff --git a/packages/serverless-offline-sqs-integration/package.json b/packages/serverless-offline-sqs-integration/package.json new file mode 100644 index 00000000..f4730f12 --- /dev/null +++ b/packages/serverless-offline-sqs-integration/package.json @@ -0,0 +1,18 @@ +{ + "name": "serverless-offline-sqs-integration", + "private": true, + "scripts": { + "start": "sls offline", + "pretest": "docker-compose stop sqs && docker-compose rm -f sqs && docker-compose run sqs-create", + "test": "node ./test.js" + }, + "dependencies": { + "aws-sdk": "^2.444.0", + "figures": "^3.0.0", + "lodash": "^4.17.11", + "serverless": "^1.41.1", + "serverless-offline": "^4.9.4", + "serverless-offline-sqs": "1.6.1", + "signal-exit": "^3.0.2" + } +} diff --git a/packages/serverless-offline-sqs/example/serverless.yml b/packages/serverless-offline-sqs-integration/serverless.yml similarity index 89% rename from packages/serverless-offline-sqs/example/serverless.yml rename to packages/serverless-offline-sqs-integration/serverless.yml index 3e03832d..91ce4033 100644 --- a/packages/serverless-offline-sqs/example/serverless.yml +++ b/packages/serverless-offline-sqs-integration/serverless.yml @@ -27,6 +27,10 @@ functions: Fn::GetAtt: - MyThirdQueue - Arn + myPythonHandler: + runtime: python2.7 + handler: handler.handler + events: - sqs: arn: Fn::GetAtt: diff --git a/packages/serverless-offline-sqs-integration/test.js b/packages/serverless-offline-sqs-integration/test.js new file mode 100755 index 00000000..45a7739f --- /dev/null +++ b/packages/serverless-offline-sqs-integration/test.js @@ -0,0 +1,79 @@ +/* eslint-disable unicorn/no-process-exit */ +const {Writable} = require('stream'); +const {spawn} = require('child_process'); +const onExit = require('signal-exit'); +const {SQS} = require('aws-sdk'); + +const client = new SQS({ + region: 'eu-west-1', + accessKeyId: 'local', + secretAccessKey: 'local', + endpoint: 'http://localhost:9324' +}); + +const sendMessages = () => { + return Promise.all([ + client + .sendMessage({ + QueueUrl: 'http://localhost:9324/queue/MyFirstQueue', + MessageBody: 'MyFirstMessage' + }) + .promise(), + client + .sendMessage({ + QueueUrl: 'http://localhost:9324/queue/MySecondQueue', + MessageBody: 'MySecondMessage' + }) + .promise(), + client + .sendMessage({ + QueueUrl: 'http://localhost:9324/queue/MyThirdQueue', + MessageBody: 'MyThirdMessage' + }) + .promise(), + client + .sendMessage({ + QueueUrl: 'http://localhost:9324/queue/MyFourthQueue', + MessageBody: 'MyFourthMessage' + }) + .promise() + ]); +}; + +const serverless = spawn('sls', ['offline'], { + stdio: ['pipe', 'pipe', 'pipe'], + cwd: __dirname +}); + +serverless.stdout.pipe( + new Writable({ + write(chunk, enc, cb) { + const output = chunk.toString(); + + if (/Offline listening on/.test(output)) { + sendMessages(); + } + + this.count = (this.count || 0) + (output.match(/\[✔\]/g) || []).length; + if (this.count === 4) serverless.kill(); + cb(); + } + }) +); + +serverless.stdout.on('data', data => { + console.log(data.toString()); +}); + +serverless.stderr.on('data', data => { + console.error(data.toString()); + process.exit(1); +}); + +serverless.on('close', code => { + process.exit(code); +}); + +onExit((code, signal) => { + if (signal) serverless.kill(signal); +}); diff --git a/packages/serverless-offline-sqs/README.md b/packages/serverless-offline-sqs/README.md index e0beeaaa..e2944daf 100644 --- a/packages/serverless-offline-sqs/README.md +++ b/packages/serverless-offline-sqs/README.md @@ -24,13 +24,13 @@ plugins: - serverless-offline ``` -[See example](./example/README.md) +[See example](../serverless-offline-sqs-integration/README.md) ## How it works? To be able to emulate AWS SQS queue on local machine there should be some queue system actually running. One of the existing implementations suitable for the task is [ElasticMQ](https://github.com/adamw/elasticmq). -[ElasticMQ](https://github.com/adamw/elasticmq) is a standalone in-memory queue system, which implements AWS SQS compatible interface. It can be run either stand-alone or inside Docker container. See [example](./example/docker-compose.yml) `sqs` service setup. +[ElasticMQ](https://github.com/adamw/elasticmq) is a standalone in-memory queue system, which implements AWS SQS compatible interface. It can be run either stand-alone or inside Docker container. See [example](../serverless-offline-sqs-integration/docker-compose.yml) `sqs` service setup. We also need to setup actual queue in ElasticMQ server, we can use [AWS cli](https://aws.amazon.com/cli/) tools for that. In example, we spawn-up another container with `aws-cli` pre-installed and run initialization script, against ElasticMQ server in separate container. diff --git a/packages/serverless-offline-sqs/example/package.json b/packages/serverless-offline-sqs/example/package.json deleted file mode 100644 index 2ee3015f..00000000 --- a/packages/serverless-offline-sqs/example/package.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "scripts": { - "start": "../../../node_modules/.bin/serverless offline", - "pretest": "docker-compose stop sqs && docker-compose rm -f sqs && docker-compose run sqs-create", - "test": "if test $(npm run start -- --exec ./test.sh | sed -n '/[✔]/p' | wc -l) != 4 ; then exit 1; fi;", - "test": "npm run start -- --exec ./test.sh" - } -} \ No newline at end of file diff --git a/packages/serverless-offline-sqs/example/test.sh b/packages/serverless-offline-sqs/example/test.sh deleted file mode 100755 index ab4b6841..00000000 --- a/packages/serverless-offline-sqs/example/test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -trap "exit 1" INT - -aws sqs --endpoint-url http://localhost:9324 send-message --queue-url http://localhost:9324/queue/MyFirstQueue --message-body "MyFirstMessage" & -aws sqs --endpoint-url http://localhost:9324 send-message --queue-url http://localhost:9324/queue/MySecondQueue --message-body "MySecondMessage" & -aws sqs --endpoint-url http://localhost:9324 send-message --queue-url http://localhost:9324/queue/MyThirdQueue --message-body "MyThirdMessage" & -aws sqs --endpoint-url http://localhost:9324 send-message --queue-url http://localhost:9324/queue/MyFourthQueue --message-body "MyFourthMessage" & -wait - -trap - INT \ No newline at end of file diff --git a/packages/serverless-offline-sqs/package.json b/packages/serverless-offline-sqs/package.json index 081ad19a..87c5f433 100644 --- a/packages/serverless-offline-sqs/package.json +++ b/packages/serverless-offline-sqs/package.json @@ -16,11 +16,8 @@ ], "author": "godu", "license": "MIT", - "scripts": { - "test": "cd example && npm test" - }, "dependencies": { - "aws-sdk": "^2.443.0", + "aws-sdk": "^2.444.0", "figures": "^3.0.0", "lodash": "^4.17.11", "serverless-offline": "^4.9.4"