From 631a3c50776c08c522cbdb84c0fcc39e811a8f4b Mon Sep 17 00:00:00 2001 From: Hui Yang Date: Wed, 9 Oct 2019 21:53:34 -0700 Subject: [PATCH] Update Greengrass Hello World examples --- .../aws-greengrass-core-sdk/index.js | 7 + .../aws-greengrass-core-sdk/iotdata.js | 148 +++++++++++ .../aws-greengrass-core-sdk/lambda.js | 116 +++++++++ .../aws-greengrass-core-sdk/secretsmanager.js | 95 +++++++ .../aws-greengrass-core-sdk/util.js | 41 +++ .../greengrass-hello-world-nodejs/index.js | 6 +- .../template.yaml | 8 +- .../greengrassHelloWorld.py | 16 +- .../greengrass_common/__init__.py | 6 - .../greengrass_common/common_log_appender.py | 16 -- .../greengrass_common/env_vars.py | 10 - .../greengrass_common/function_arn_fields.py | 46 ---- .../greengrass_common/greengrass_message.py | 76 ------ .../local_cloudwatch_handler.py | 133 ---------- .../greengrass_ipc_python_sdk/__init__.py | 5 - .../greengrass_ipc_python_sdk/ipc_client.py | 238 ------------------ .../utils/__init__.py | 0 .../utils/exponential_backoff.py | 116 --------- .../greengrasssdk/IoTDataPlane.py | 7 +- .../greengrasssdk/Lambda.py | 14 +- .../greengrasssdk/SecretsManager.py | 160 ++++++++++++ .../greengrasssdk/__init__.py | 5 +- .../greengrasssdk/client.py | 2 + .../greengrasssdk/utils/__init__.py | 0 .../greengrasssdk/utils/testing.py | 0 .../apps/greengrass-hello-world/template.yaml | 8 +- 26 files changed, 608 insertions(+), 671 deletions(-) create mode 100755 examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/index.js create mode 100755 examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/iotdata.js create mode 100755 examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/lambda.js create mode 100755 examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/secretsmanager.js create mode 100755 examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/util.js delete mode 100644 examples/apps/greengrass-hello-world/greengrass_common/__init__.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_common/common_log_appender.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_common/env_vars.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_common/function_arn_fields.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_common/greengrass_message.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_common/local_cloudwatch_handler.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/__init__.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/ipc_client.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/utils/__init__.py delete mode 100644 examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/utils/exponential_backoff.py mode change 100644 => 100755 examples/apps/greengrass-hello-world/greengrasssdk/IoTDataPlane.py mode change 100644 => 100755 examples/apps/greengrass-hello-world/greengrasssdk/Lambda.py create mode 100755 examples/apps/greengrass-hello-world/greengrasssdk/SecretsManager.py mode change 100644 => 100755 examples/apps/greengrass-hello-world/greengrasssdk/__init__.py mode change 100644 => 100755 examples/apps/greengrass-hello-world/greengrasssdk/client.py mode change 100644 => 100755 examples/apps/greengrass-hello-world/greengrasssdk/utils/__init__.py mode change 100644 => 100755 examples/apps/greengrass-hello-world/greengrasssdk/utils/testing.py diff --git a/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/index.js b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/index.js new file mode 100755 index 0000000000..dd1190699d --- /dev/null +++ b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/index.js @@ -0,0 +1,7 @@ +/* + * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + */ +exports.GreengrassInterfaceVersion = '1.3'; +exports.Lambda = require('./lambda'); +exports.IotData = require('./iotdata'); +exports.SecretsManager = require('./secretsmanager'); diff --git a/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/iotdata.js b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/iotdata.js new file mode 100755 index 0000000000..c40c80e42f --- /dev/null +++ b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/iotdata.js @@ -0,0 +1,148 @@ +/* + * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + */ + +const Buffer = require('buffer').Buffer; + +const Lambda = require('./lambda'); +const Util = require('./util'); +const GreengrassCommon = require('aws-greengrass-common-js'); + +const envVars = GreengrassCommon.envVars; +const MY_FUNCTION_ARN = envVars.MY_FUNCTION_ARN; +const SHADOW_FUNCTION_ARN = envVars.SHADOW_FUNCTION_ARN; +const ROUTER_FUNCTION_ARN = envVars.ROUTER_FUNCTION_ARN; + +class IotData { + constructor() { + this.lambda = new Lambda(); + } + + getThingShadow(params, callback) { + /* + * Call shadow lambda to obtain current shadow state. + * @param {object} params object contains parameters for the call + * REQUIRED: 'thingName' the name of the thing + */ + const thingName = Util.getRequiredParameter(params, 'thingName'); + if (thingName === undefined) { + callback(new Error('"thingName" is a required parameter.'), null); + return; + } + + const payload = ''; + this._shadowOperation('get', thingName, payload, callback); + } + + updateThingShadow(params, callback) { + /* + * Call shadow lambda to update current shadow state. + * @param {object} params object contains parameters for the call + * REQUIRED: 'thingName' the name of the thing + * 'payload' the state information in JSON format + */ + const thingName = Util.getRequiredParameter(params, 'thingName'); + if (thingName === undefined) { + callback(new Error('"thingName" is a required parameter.'), null); + return; + } + + const payload = Util.getRequiredParameter(params, 'payload'); + if (payload === undefined) { + callback(new Error('"payload" is a required parameter.'), null); + return; + } + + this._shadowOperation('update', thingName, payload, callback); + } + + deleteThingShadow(params, callback) { + /* + * Call shadow lambda to delete the shadow state. + * @param {object} params object contains parameters for the call + * REQUIRED: 'thingName' the name of the thing + */ + const thingName = Util.getRequiredParameter(params, 'thingName'); + if (thingName === undefined) { + callback(new Error('"thingName" is a required parameter.'), null); + return; + } + + const payload = ''; + this._shadowOperation('delete', thingName, payload, callback); + } + + publish(params, callback) { + /* + * Publishes state information. + * @param {object} params object contains parameters for the call + * REQUIRED: 'topic' the topic name to be published + * 'payload' the state information in JSON format + */ + const topic = Util.getRequiredParameter(params, 'topic'); + if (topic === undefined) { + callback(new Error('"topic" is a required parameter'), null); + return; + } + + const payload = Util.getRequiredParameter(params, 'payload'); + if (payload === undefined) { + callback(new Error('"payload" is a required parameter'), null); + return; + } + + const context = { + custom: { + source: MY_FUNCTION_ARN, + subject: topic, + }, + }; + + const buff = Buffer.from(JSON.stringify(context)); + const clientContext = buff.toString('base64'); + + const invokeParams = { + FunctionName: ROUTER_FUNCTION_ARN, + InvocationType: 'Event', + ClientContext: clientContext, + Payload: payload, + }; + + console.log(`Publishing message on topic "${topic}" with Payload "${payload}"`); + + this.lambda.invoke(invokeParams, (err, data) => { + if (err) { + callback(err, null); // an error occurred + } else { + callback(null, data); // successful response + } + }); + } + + _shadowOperation(operation, thingName, payload, callback) { + const topic = `$aws/things/${thingName}/shadow/${operation}`; + const context = { + custom: { + subject: topic, + }, + }; + + const clientContext = Buffer.from(JSON.stringify(context)).toString('base64'); + const invokeParams = { + FunctionName: SHADOW_FUNCTION_ARN, + ClientContext: clientContext, + Payload: payload, + }; + + console.log(`Calling shadow service on topic "${topic}" with payload "${payload}"`); + this.lambda.invoke(invokeParams, (err, data) => { + if (err) { + callback(err, null); + } else { + callback(null, data); + } + }); + } +} + +module.exports = IotData; diff --git a/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/lambda.js b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/lambda.js new file mode 100755 index 0000000000..1380b93d1e --- /dev/null +++ b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/lambda.js @@ -0,0 +1,116 @@ +/* + * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + */ + +const Util = require('./util'); +const IPCClient = require('aws-greengrass-ipc-sdk-js'); +const GreengrassCommon = require('aws-greengrass-common-js'); +const logging = require('aws-greengrass-common-js').logging; + +const AUTH_TOKEN = GreengrassCommon.envVars.AUTH_TOKEN; + +const logger = new logging.LocalWatchLogger(); + +class Lambda { + constructor() { + this.ipc = new IPCClient(AUTH_TOKEN); + } + + invoke(params, callback) { + const functionName = Util.getRequiredParameter(params, 'FunctionName'); + if (functionName === undefined) { + callback(new Error('"FunctionName" is a required parameter'), null); + return; + } + + let arnFields; + try { + arnFields = new GreengrassCommon.FunctionArnFields(functionName); + } catch (e) { + callback(new Error(`FunctionName is malformed: ${e}`), null); + return; + } + + let invocationType; + if (params.InvocationType === undefined || params.InvocationType === null) { + invocationType = 'RequestResponse'; + } else { + invocationType = params.InvocationType; + } + + if (invocationType !== 'Event' && invocationType !== 'RequestResponse') { + callback(new Error(`InvocationType '${invocationType}' is incorrect, should be 'Event' or 'RequestResponse'`), null); + return; + } + + const clientContext = params.ClientContext ? params.ClientContext : ''; + const payload = params.Payload; + const qualifier = params.Qualifier; + + if (!Util.isValidQualifier(qualifier)) { + callback(new Error(`Qualifier '${qualifier}' is incorrect`), null); + return; + } + + const qualifierInternal = arnFields.qualifier; + + // generate the right full function arn with qualifier + if (qualifierInternal && qualifier && qualifierInternal !== qualifier) { + callback(new Error(`Qualifier '${qualifier}' does not match the version in FunctionName`), null); + return; + } + + const finalQualifier = qualifierInternal === undefined || qualifierInternal == null ? qualifier : qualifierInternal; + + let functionArn; + if (typeof GreengrassCommon.buildFunctionArn === 'function') { + // GGC v1.9.0 or newer + functionArn = GreengrassCommon.buildFunctionArn( + arnFields.unqualifiedArn, + finalQualifier); + } else { + // older version of GGC + throw new Error('Function buildFunctionArn not found. buildFunctionArn is introduced in GGC v1.9.0. ' + + 'Please check your GGC version.'); + } + + // verify client context is base64 encoded + if (Object.prototype.hasOwnProperty.call(params, 'ClientContext')) { + const cxt = params.ClientContext; + if (!Util.isValidContext(cxt)) { + callback(new Error('Client Context is invalid'), null); + return; + } + } + + logger.debug(`Invoking local lambda ${functionArn} with payload ${payload} and client context ${clientContext}`); + + this.ipc.postWork(functionArn, payload, clientContext, invocationType, (postWorkErr, invocationId) => { + if (postWorkErr) { + logger.error(`Failed to invoke function due to ${postWorkErr}`); + callback(postWorkErr, null); + return; + } + + if (invocationType === 'RequestResponse') { + this.ipc.getWorkResult(functionArn, invocationId, (getWorkResultErr, body, functionErr, statusCode) => { + if (getWorkResultErr) { + logger.error(`Failed to get work result due to ${getWorkResultErr}`); + callback(getWorkResultErr, null); + return; + } + const data = { + FunctionError: functionErr, + StatusCode: statusCode, + Payload: body, + }; + callback(null, data); + }); + } else { + callback(null, invocationId); + } + }); + } +} + +module.exports = Lambda; diff --git a/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/secretsmanager.js b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/secretsmanager.js new file mode 100755 index 0000000000..80a7c15493 --- /dev/null +++ b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/secretsmanager.js @@ -0,0 +1,95 @@ +/* + * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + */ +const Buffer = require('buffer').Buffer; + +const Lambda = require('./lambda'); +const Util = require('./util'); +const GreengrassCommon = require('aws-greengrass-common-js'); + +const KEY_SECRET_ID = 'SecretId'; +const KEY_VERSION_ID = 'VersionId'; +const KEY_VERSION_STAGE = 'VersionStage'; +const KEY_SECRET_ARN = 'ARN'; +const KEY_SECRET_NAME = 'Name'; +const KEY_CREATED_DATE = 'CreatedDate'; + +const envVars = GreengrassCommon.envVars; +const SECRETS_MANAGER_FUNCTION_ARN = envVars.SECRETS_MANAGER_FUNCTION_ARN; + +class SecretsManager { + constructor() { + this.lambda = new Lambda(); + } + + getSecretValue(params, callback) { + const secretId = Util.getRequiredParameter(params, KEY_SECRET_ID); + const versionId = Util.getRequiredParameter(params, KEY_VERSION_ID); + const versionStage = Util.getRequiredParameter(params, KEY_VERSION_STAGE); + + if (secretId === undefined) { + callback(new Error(`"${KEY_SECRET_ID}" is a required parameter`), null); + return; + } + // TODO: Remove this once we support query by VersionId + if (versionId !== undefined) { + callback(new Error('Query by VersionId is not yet supported'), null); + return; + } + if (versionId !== undefined && versionStage !== undefined) { + callback(new Error('VersionId and VersionStage cannot both be specified at the same time'), null); + return; + } + + const getSecretValueRequestBytes = + SecretsManager._generateGetSecretValueRequestBytes(secretId, versionId, versionStage); + + const invokeParams = { + FunctionName: SECRETS_MANAGER_FUNCTION_ARN, + Payload: getSecretValueRequestBytes, + }; + + console.log(`Getting secret value from secrets manager: ${getSecretValueRequestBytes}`); + + this.lambda.invoke(invokeParams, (err, data) => { + if (err) { + callback(err, null); // an error occurred + } else if (SecretsManager._is200Response(data.Payload)) { + callback(null, data.Payload); // successful response + } else { + callback(new Error(JSON.stringify(data.Payload)), null); // error response + } + }); + } + + static _generateGetSecretValueRequestBytes(secretId, versionId, versionStage) { + const request = { + SecretId: secretId, + }; + + if (versionStage !== undefined) { + request.VersionStage = versionStage; + } + + if (versionId !== undefined) { + request.VersionId = versionId; + } + + return Buffer.from(JSON.stringify(request)); + } + + static _is200Response(payload) { + const hasSecretArn = this._stringContains(payload, KEY_SECRET_ARN); + const hasSecretName = this._stringContains(payload, KEY_SECRET_NAME); + const hasVersionId = this._stringContains(payload, KEY_VERSION_ID); + const hasCreatedDate = this._stringContains(payload, KEY_CREATED_DATE); + + return hasSecretArn && hasSecretName && hasVersionId && hasCreatedDate; + } + + static _stringContains(src, target) { + return src.indexOf(target) > -1; + } +} + +module.exports = SecretsManager; diff --git a/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/util.js b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/util.js new file mode 100755 index 0000000000..b02fa8e65d --- /dev/null +++ b/examples/apps/greengrass-hello-world-nodejs/aws-greengrass-core-sdk/util.js @@ -0,0 +1,41 @@ +/* + * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + */ + +const base64Regex = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/; +const qualifierRegex = /(|[a-zA-Z0-9$_-]+)/; + +exports.getRequiredParameter = function _getRequiredParameter(params, requiredParam) { + if (!Object.prototype.hasOwnProperty.call(params, requiredParam)) { + return; + } + return params[requiredParam]; +}; + +exports.isValidJSON = function _isValidJSON(str) { + try { + JSON.parse(str); + } catch (e) { + return false; + } + return true; +}; + +exports.isValidContext = function _isValidContext(context) { + if (!base64Regex.test(context)) { + return false; + } + try { + JSON.stringify(context); + } catch (e) { + return false; + } + return true; +}; + +exports.isValidQualifier = function _isValidQualifier(qualifier) { + if (!qualifierRegex.test(qualifier)) { + return false; + } + return true; +}; diff --git a/examples/apps/greengrass-hello-world-nodejs/index.js b/examples/apps/greengrass-hello-world-nodejs/index.js index 9edcab9be3..041a05babc 100644 --- a/examples/apps/greengrass-hello-world-nodejs/index.js +++ b/examples/apps/greengrass-hello-world-nodejs/index.js @@ -1,5 +1,5 @@ /* - * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. */ /* @@ -18,8 +18,8 @@ const os = require('os'); const util = require('util'); function publishCallback(err, data) { - console.log(err); - console.log(data); + console.log('Publish response: '+data); + console.log('Publish error: '+err); } const myPlatform = util.format('%s-%s', os.platform(), os.release()); diff --git a/examples/apps/greengrass-hello-world-nodejs/template.yaml b/examples/apps/greengrass-hello-world-nodejs/template.yaml index bf0bb24d7d..65e3052a31 100644 --- a/examples/apps/greengrass-hello-world-nodejs/template.yaml +++ b/examples/apps/greengrass-hello-world-nodejs/template.yaml @@ -3,15 +3,15 @@ Transform: 'AWS::Serverless-2016-10-31' Description: >- Deploy this lambda to a Greengrass core where it will send a hello world message to a topic -Parameters: - IdentityNameParameter: +Parameters: + IdentityNameParameter: Type: String Resources: greengrasshelloworldnodejs: Type: 'AWS::Serverless::Function' Properties: Handler: index.handler - Runtime: nodejs6.10 + Runtime: nodejs8.10 CodeUri: . Description: >- Deploy this lambda to a Greengrass core where it will send a hello world @@ -20,4 +20,4 @@ Resources: Timeout: 3 Policies: - SESSendBouncePolicy: - IdentityName: !Ref IdentityNameParameter \ No newline at end of file + IdentityName: !Ref IdentityNameParameter diff --git a/examples/apps/greengrass-hello-world/greengrassHelloWorld.py b/examples/apps/greengrass-hello-world/greengrassHelloWorld.py index 270df1bb57..be838e3853 100644 --- a/examples/apps/greengrass-hello-world/greengrassHelloWorld.py +++ b/examples/apps/greengrass-hello-world/greengrassHelloWorld.py @@ -1,5 +1,5 @@ # -# Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # greengrassHelloWorld.py @@ -10,10 +10,10 @@ # long-lived it will run forever when deployed to a Greengrass core. The handler # will NOT be invoked in our example since the we are executing an infinite loop. +import logging import greengrasssdk import platform from threading import Timer -import time # Creating a greengrass core sdk client @@ -21,6 +21,10 @@ # Retrieving platform information to send from Greengrass Core my_platform = platform.platform() +if not my_platform: + payload_str = 'Hello world! Sent from Greengrass Core.' +else: + payload_str = 'Hello world! Sent from Greengrass Core running on platform: {}'.format(my_platform) # When deployed to a Greengrass core, this code will be executed immediately @@ -31,10 +35,12 @@ # a result. def greengrass_hello_world_run(): - if not my_platform: - client.publish(topic='hello/world', payload='Hello world! Sent from Greengrass Core.') + try: + response = client.publish(topic='hello/world', payload=payload_str) + except Exception as e: + logging.error("Failed to publish the message, error: {}".format(e)) else: - client.publish(topic='hello/world', payload='Hello world! Sent from Greengrass Core running on platform: {}'.format(my_platform)) + logging.info("Successfully publish the message, response: {}".format(response)) # Asynchronously schedule this function to be run again in 5 seconds Timer(5, greengrass_hello_world_run).start() diff --git a/examples/apps/greengrass-hello-world/greengrass_common/__init__.py b/examples/apps/greengrass-hello-world/greengrass_common/__init__.py deleted file mode 100644 index 8b1d457eef..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_common/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -from .function_arn_fields import FunctionArnFields -from .greengrass_message import GreengrassMessage diff --git a/examples/apps/greengrass-hello-world/greengrass_common/common_log_appender.py b/examples/apps/greengrass-hello-world/greengrass_common/common_log_appender.py deleted file mode 100644 index 1fe1c9bdbd..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_common/common_log_appender.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# this log appender is shared among all components of python lambda runtime, including: -# greengrass_common/greengrass_message.py, greengrass_ipc_python_sdk/ipc_client.py, -# greengrass_ipc_python_sdk/utils/exponential_backoff.py, lambda_runtime/lambda_runtime.py. -# so that all log records will be emitted to local Cloudwatch. -import logging.handlers - -from greengrass_common.local_cloudwatch_handler import LocalCloudwatchLogHandler - -# https://docs.python.org/2/library/logging.html#logrecord-attributes -LOCAL_CLOUDWATCH_FORMAT = '[%(levelname)s]-%(filename)s:%(lineno)d,%(message)s' - -local_cloudwatch_handler = LocalCloudwatchLogHandler('GreengrassSystem', 'python_runtime') -local_cloudwatch_handler.setFormatter(logging.Formatter(LOCAL_CLOUDWATCH_FORMAT)) diff --git a/examples/apps/greengrass-hello-world/greengrass_common/env_vars.py b/examples/apps/greengrass-hello-world/greengrass_common/env_vars.py deleted file mode 100644 index aae4721621..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_common/env_vars.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -import os - -AUTH_TOKEN = os.getenv('AWS_CONTAINER_AUTHORIZATION_TOKEN') -MY_FUNCTION_ARN = os.getenv('MY_FUNCTION_ARN') -SHADOW_FUNCTION_ARN = os.getenv('SHADOW_FUNCTION_ARN') -ROUTER_FUNCTION_ARN = os.getenv('ROUTER_FUNCTION_ARN') diff --git a/examples/apps/greengrass-hello-world/greengrass_common/function_arn_fields.py b/examples/apps/greengrass-hello-world/greengrass_common/function_arn_fields.py deleted file mode 100644 index 2e5928c265..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_common/function_arn_fields.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -import re - -ARN_FIELD_REGEX = \ - 'arn:aws:lambda:([a-z]{2}-[a-z]+-\d{1}):(\d{12}):function:([a-zA-Z0-9-_]+)(?::(\$LATEST|[a-zA-Z0-9-_]+))?' - - -class FunctionArnFields: - """ - This class takes in a string representing a Lambda function's ARN (the qualifier is optional), parses that string - into individual fields for region, account_id, name and qualifier. It also has a static method for creating a - Function ARN string from those subfields. - """ - @staticmethod - def build_arn_string(region, account_id, name, qualifier): - if qualifier: - return 'arn:aws:lambda:{region}:{account_id}:function:{name}:{qualifier}'.format( - region=region, account_id=account_id, name=name, qualifier=qualifier - ) - else: - return 'arn:aws:lambda:{region}:{account_id}:function:{name}'.format( - region=region, account_id=account_id, name=name - ) - - def __init__(self, function_arn_string): - self.parse_function_arn(function_arn_string) - - def parse_function_arn(self, function_arn_string): - regex_match = re.match(ARN_FIELD_REGEX, function_arn_string) - if regex_match: - region, account_id, name, qualifier = map( - lambda s: s.replace(':', '') if s else s, regex_match.groups() - ) - else: - raise ValueError('Cannot parse given string as a function ARN.') - - self.region = region - self.account_id = account_id - self.name = name - self.qualifier = qualifier - - def to_arn_string(self): - return FunctionArnFields.build_arn_string(self.region, self.account_id, self.name, self.qualifier) diff --git a/examples/apps/greengrass-hello-world/greengrass_common/greengrass_message.py b/examples/apps/greengrass-hello-world/greengrass_common/greengrass_message.py deleted file mode 100644 index 23906daa79..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_common/greengrass_message.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -import base64 -import json -import logging -from greengrass_common.common_log_appender import local_cloudwatch_handler - -# Log messages here are not part of customer's log because anything that -# goes wrong here has nothing to do with customer's lambda code. Since we configured -# the root logger to log to customer's log, we need to turn off propagation. -runtime_logger = logging.getLogger(__name__) -runtime_logger.addHandler(local_cloudwatch_handler) -runtime_logger.propagate = False -# set to the lowest possible level so all log messages will be sent to local cloudwatch handler -runtime_logger.setLevel(logging.DEBUG) - - -class GreengrassMessage: - """ - Holds the payload and extension_map fields making up messages exchanged over the IPC. Provides methods for encoding - and decoding to/from strings. - """ - - def __init__(self, payload=b'', **extension_map): - self.payload = payload - self.extension_map = extension_map - - @classmethod - def decode(cls, encoded_string): - if encoded_string: - try: - data_map = json.loads(encoded_string) - except (ValueError, TypeError) as e: - runtime_logger.exception(e) - raise ValueError('Could not load provided encoded string "{}" as JSON due to exception: {}'.format( - repr(encoded_string), str(e) - )) - - try: - payload = base64.b64decode(data_map['Payload']) - except (ValueError, TypeError) as e: - runtime_logger.exception(e) - raise ValueError( - 'Could not decode payload of Greengrass Message data' - '"{}" from base64 due to exception: {}'.format(repr(data_map), str(e)) - ) - - extension_map = data_map['ExtensionMap_'] - else: - payload = None - extension_map = {} - - return cls(payload, **extension_map) - - def encode(self): - try: - # .decode to convert bytes -> string - payload = base64.b64encode(self.payload).decode() - except (ValueError, TypeError) as e: - runtime_logger.exception(e) - raise ValueError('Could not encode Greengrass Message payload "{}" as base64 due to exception: {}'.format( - repr(self.payload), str(e) - )) - - try: - return json.dumps({'Payload': payload, 'ExtensionMap_': self.extension_map}) - except (ValueError, TypeError) as e: - runtime_logger.exception(e) - raise ValueError('Could not encode Greengrass Message fields "{}" as JSON due to exception: {}'.format( - str(self), str(e) - )) - - def __str__(self): - return str({'Payload': self.payload, 'ExtensionMap_': self.extension_map}) diff --git a/examples/apps/greengrass-hello-world/greengrass_common/local_cloudwatch_handler.py b/examples/apps/greengrass-hello-world/greengrass_common/local_cloudwatch_handler.py deleted file mode 100644 index d17dd59659..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_common/local_cloudwatch_handler.py +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -from __future__ import print_function - -try: - # Python 3 - from urllib.request import urlopen, Request - from urllib.error import URLError -except ImportError: - # Python 2 - from urllib2 import urlopen, Request, URLError - -import functools -import inspect -import json -import logging -import os.path -import sys -import time -import traceback - -from greengrass_common.env_vars import AUTH_TOKEN - -HEADER_AUTH_TOKEN = 'Authorization' -LOCAL_CLOUDWATCH_API_VERSION = '2016-11-01' -LOCAL_CLOUDWATCH_ENDPOINT = 'http://localhost:8000/{version}/cloudwatch/logs/'.format(version=LOCAL_CLOUDWATCH_API_VERSION) - -# http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html -MAX_REQUEST_SIZE = 1024 * 1024 -LOG_EVENT_OVERHEAD = 26 -BUFFER_SIZE = 10000 -SECONDS_IN_ONE_DAY = 86400 -# local Cloudwatch uses the log4j logging levels, so we need to convert Python's logging.WARNING -# and loggging.CRITICAL to levels understandable by local Cloudwatch, which is WARN and FATAL. -LOG_LEVEL_WARNING_TO_REPLACE = '[WARNING]' -LOG_LEVEL_CRITICAL_TO_REPLACE = '[CRITICAL]' - - -def wrap_urllib_exceptions(func): - @functools.wraps(func) - def wrapped(*args, **kwargs): - try: - return func(*args, **kwargs) - except URLError: - exc_type, exc_value, exc_traceback = sys.exc_info() - full_stack_trace = traceback.format_exception(exc_type, exc_value, exc_traceback) - full_stack_trace.insert(0, 'Failed to send to Localwatch due to exception:\n') - # when we can't talk to local cloudwatch, print to the STDERR - # this will go to Daemon's STDERR - print(''.join(full_stack_trace), file=sys.__stderr__) - - return wrapped - - -class LocalCloudwatchLogHandler(logging.Handler): - - def __init__(self, component_type, component_name, *args, **kwargs): - logging.Handler.__init__(self, *args, **kwargs) - self.oldest_time_stamp = time.time() - self.total_log_event_byte_size = 0 - self.events_buffer = [] - self.log_group_name = os.path.join('/', component_type, component_name) - self.auth_token = AUTH_TOKEN - - def write(self, data): - data = str(data) - if data == '\n': - # when print(data) is invoked, it invokes write() twice. First, - # writes the data, then writes a new line. This is to avoid - # emitting log record with just a new-line character. - return - - # creates https://docs.python.org/2/library/logging.html#logrecord-objects - file_name, line_number = inspect.getouterframes(inspect.currentframe())[1][1:3] - record = logging.makeLogRecord({"created": time.time(), - "msg": data, - "filename": os.path.basename(file_name), - "lineno": line_number, - "levelname": "DEBUG", - "levelno": logging.DEBUG}) - self.emit(record) - - def _should_send(self, message, created_time): - if created_time >= self.oldest_time_stamp + SECONDS_IN_ONE_DAY: - return True - elif self.total_log_event_byte_size + len(message) + LOG_EVENT_OVERHEAD > MAX_REQUEST_SIZE: - return True - elif len(self.events_buffer) == BUFFER_SIZE: - return True - else: - return False - - def emit(self, record): - # This is an implementation of the logging handler interface: - # https://docs.python.org/2/library/logging.html#handler-objects - msg = self.format(record) - - if msg.startswith(LOG_LEVEL_WARNING_TO_REPLACE): - msg = ''.join(('[WARN]', msg[len(LOG_LEVEL_WARNING_TO_REPLACE):])) - elif msg.startswith(LOG_LEVEL_CRITICAL_TO_REPLACE): - msg = ''.join(('[FATAL]', msg[len(LOG_LEVEL_CRITICAL_TO_REPLACE):])) - - # TODO: Revert GG-5168, re-introduce _should_send check here and avoid - # flushing per emit - self.total_log_event_byte_size += len(msg) + LOG_EVENT_OVERHEAD - self.events_buffer.append({'timestamp': int(round(record.created * 1000)), 'message': msg}) - self.flush() - - @wrap_urllib_exceptions - def _send_to_local_cw(self): - # construct a putLogEvents request and send it - # http://boto3.readthedocs.io/en/latest/reference/services/logs.html#CloudWatchLogs.Client.put_log_events - request_data = { - 'logGroupName': self.log_group_name, - 'logStreamName': 'fromPythonAppender', - 'logEvents': self.events_buffer - } - request = Request(LOCAL_CLOUDWATCH_ENDPOINT, json.dumps(request_data).encode('utf-8')) - request.add_header(HEADER_AUTH_TOKEN, self.auth_token) - - urlopen(request) - self._clear_buffer() - - def flush(self): - if len(self.events_buffer) > 0: - # don't bother to send a request if there's nothing to send - # otherwise you'll just get an HTTP 400 - self._send_to_local_cw() - - def _clear_buffer(self): - self.total_log_event_byte_size = 0 - del self.events_buffer[:] diff --git a/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/__init__.py b/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/__init__.py deleted file mode 100644 index 0b0389ac4f..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -from .ipc_client import IPCClient diff --git a/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/ipc_client.py b/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/ipc_client.py deleted file mode 100644 index 4f46f70c6a..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/ipc_client.py +++ /dev/null @@ -1,238 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -import collections -import functools -import logging -import json - -try: - # Python 3 - from urllib.request import urlopen, Request - from urllib.error import URLError -except ImportError: - # Python 2 - from urllib2 import urlopen, Request, URLError - -from greengrass_common.env_vars import AUTH_TOKEN -from greengrass_common.common_log_appender import local_cloudwatch_handler - -# Log messages in the ipc client are not part of customer's log because anything that -# goes wrong here has have nothing to do with customer's lambda code. Since we configured -# the root logger to log to customer's log, we need to turn off propagation. -runtime_logger = logging.getLogger(__name__) -runtime_logger.addHandler(local_cloudwatch_handler) -runtime_logger.propagate = False -# set to the lowest possible level so all log messages will be sent to local cloudwatch handler -runtime_logger.setLevel(logging.DEBUG) - -HEADER_INVOCATION_ID = 'X-Amz-InvocationId' -HEADER_CLIENT_CONTEXT = 'X-Amz-Client-Context' -HEADER_AUTH_TOKEN = 'Authorization' -HEADER_INVOCATION_TYPE = 'X-Amz-Invocation-Type' -HEADER_FUNCTION_ERR_TYPE = 'X-Amz-Function-Error' -IPC_API_VERSION = '2016-11-01' - - -def wrap_urllib_exceptions(func): - @functools.wraps(func) - def wrapped(*args, **kwargs): - try: - return func(*args, **kwargs) - except URLError as e: - runtime_logger.exception(e) - raise IPCException(str(e)) - - return wrapped - - -class IPCException(Exception): - pass - - -WorkItem = collections.namedtuple('WorkItem', ['invocation_id', 'payload', 'client_context']) -GetWorkResultOutput = collections.namedtuple('GetWorkResultOutput', ['payload', 'func_err']) - - -class IPCClient: - """ - Client for IPC that provides methods for getting/posting work for functions, - as well as getting/posting results of the work. - """ - - def __init__(self, endpoint='localhost', port=8000): - """ - :param endpoint: Endpoint used to connect to IPC. - Generally, IPC and functions always run on the same box, - so endpoint should always be 'localhost' in production. - You can override it for testing purposes. - :type endpoint: str - - :param port: Port number used to connect to the :code:`endpoint`. - Similarly to :code:`endpoint`, can be overridden for testing purposes. - :type port: int - """ - self.endpoint = endpoint - self.port = port - self.auth_token = AUTH_TOKEN - - @wrap_urllib_exceptions - def post_work(self, function_arn, input_bytes, client_context, invocation_type="RequestResponse"): - """ - Send work item to specified :code:`function_arn`. - - :param function_arn: Arn of the Lambda function intended to receive the work for processing. - :type function_arn: string - - :param input_bytes: The data making up the work being posted. - :type input_bytes: bytes - - :param client_context: The base64 encoded client context byte string that will be provided to the Lambda - function being invoked. - :type client_context: bytes - - :returns: Invocation ID for obtaining result of the work. - :type returns: str - """ - url = self._get_url(function_arn) - runtime_logger.info('Posting work for function [{}] to {}'.format(function_arn, url)) - - request = Request(url, input_bytes or b'') - request.add_header(HEADER_CLIENT_CONTEXT, client_context) - request.add_header(HEADER_AUTH_TOKEN, self.auth_token) - request.add_header(HEADER_INVOCATION_TYPE, invocation_type) - - response = urlopen(request) - - invocation_id = response.info().get(HEADER_INVOCATION_ID) - runtime_logger.info('Work posted with invocation id [{}]'.format(invocation_id)) - return invocation_id - - @wrap_urllib_exceptions - def get_work(self, function_arn): - """ - Retrieve the next work item for specified :code:`function_arn`. - - :param function_arn: Arn of the Lambda function intended to receive the work for processing. - :type function_arn: string - - :returns: Next work item to be processed by the function. - :type returns: WorkItem - """ - url = self._get_work_url(function_arn) - runtime_logger.info('Getting work for function [{}] from {}'.format(function_arn, url)) - - request = Request(url) - request.add_header(HEADER_AUTH_TOKEN, self.auth_token) - - response = urlopen(request) - - invocation_id = response.info().get(HEADER_INVOCATION_ID) - client_context = response.info().get(HEADER_CLIENT_CONTEXT) - - runtime_logger.info('Got work item with invocation id [{}]'.format(invocation_id)) - return WorkItem( - invocation_id=invocation_id, - payload=response.read(), - client_context=client_context) - - @wrap_urllib_exceptions - def post_work_result(self, function_arn, work_item): - """ - Post the result of processing work item by :code:`function_arn`. - - :param function_arn: Arn of the Lambda function intended to receive the work for processing. - :type function_arn: string - - :param work_item: The WorkItem holding the results of the work being posted. - :type work_item: WorkItem - - :returns: None - """ - url = self._get_work_url(function_arn) - - runtime_logger.info('Posting work result for invocation id [{}] to {}'.format(work_item.invocation_id, url)) - request = Request(url, work_item.payload or b'') - - request.add_header(HEADER_INVOCATION_ID, work_item.invocation_id) - request.add_header(HEADER_AUTH_TOKEN, self.auth_token) - - urlopen(request) - - runtime_logger.info('Posted work result for invocation id [{}]'.format(work_item.invocation_id)) - - @wrap_urllib_exceptions - def post_handler_err(self, function_arn, invocation_id, handler_err): - """ - Post the error message from executing the function handler for :code:`function_arn` - with specifid :code:`invocation_id` - - - :param function_arn: Arn of the Lambda function which has the handler error message. - :type function_arn: string - - :param invocation_id: Invocation ID of the work that is being requested - :type invocation_id: string - - :param handler_err: the error message caught from handler - :type handler_err: string - """ - url = self._get_work_url(function_arn) - - runtime_logger.info('Posting handler error for invocation id [{}] to {}'.format(invocation_id, url)) - - payload = json.dumps({ - "errorMessage": handler_err, - }).encode('utf-8') - - request = Request(url, payload) - request.add_header(HEADER_INVOCATION_ID, invocation_id) - request.add_header(HEADER_FUNCTION_ERR_TYPE, "Handled") - request.add_header(HEADER_AUTH_TOKEN, self.auth_token) - - urlopen(request) - - runtime_logger.info('Posted handler error for invocation id [{}]'.format(invocation_id)) - - @wrap_urllib_exceptions - def get_work_result(self, function_arn, invocation_id): - """ - Retrieve the result of the work processed by :code:`function_arn` - with specified :code:`invocation_id`. - - :param function_arn: Arn of the Lambda function intended to receive the work for processing. - :type function_arn: string - - :param invocation_id: Invocation ID of the work that is being requested - :type invocation_id: string - - :returns: The get work result output contains result payload and function error type if the invoking is failed. - :type returns: GetWorkResultOutput - """ - url = self._get_url(function_arn) - - runtime_logger.info('Getting work result for invocation id [{}] from {}'.format(invocation_id, url)) - - request = Request(url) - request.add_header(HEADER_INVOCATION_ID, invocation_id) - request.add_header(HEADER_AUTH_TOKEN, self.auth_token) - - response = urlopen(request) - - runtime_logger.info('Got result for invocation id [{}]'.format(invocation_id)) - - payload = response.read() - func_err = response.info().get(HEADER_FUNCTION_ERR_TYPE) - - return GetWorkResultOutput( - payload=payload, - func_err=func_err) - - def _get_url(self, function_arn): - return 'http://{endpoint}:{port}/{version}/functions/{function_arn}'.format( - endpoint=self.endpoint, port=self.port, version=IPC_API_VERSION, function_arn=function_arn - ) - - def _get_work_url(self, function_arn): - return '{base_url}/work'.format(base_url=self._get_url(function_arn)) diff --git a/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/utils/__init__.py b/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/utils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/utils/exponential_backoff.py b/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/utils/exponential_backoff.py deleted file mode 100644 index 674d8cd302..0000000000 --- a/examples/apps/greengrass-hello-world/greengrass_ipc_python_sdk/utils/exponential_backoff.py +++ /dev/null @@ -1,116 +0,0 @@ -# -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# - -from functools import wraps -import logging -import random -import sys -import time -import traceback - -from greengrass_common.common_log_appender import local_cloudwatch_handler - -# Log messages in the ipc client are not part of customer's log because anything that -# goes wrong here has nothing to do with customer's lambda code. Since we configured -# the root logger to log to customer's log, we need to turn off propagation. -runtime_logger = logging.getLogger(__name__) -runtime_logger.addHandler(local_cloudwatch_handler) -runtime_logger.propagate = False -# set to the lowest possible level so all log messages will be sent to local cloudwatch handler -runtime_logger.setLevel(logging.DEBUG) - - -class RetryTimeoutException(Exception): - """ - Information regarding a timed-out task. - """ - - def __init__(self, task_name, have_tried, max_attempts, total_wait_time, multiplier, backoff_coefficient, - jitter_enabled, retry_errors): - self.task_name = task_name - self.have_tried = have_tried - self.max_attempts = max_attempts - self.total_wait_time = total_wait_time - self.multiplier = multiplier - self.backoff_coefficient = backoff_coefficient - self.jitter_enabled = jitter_enabled - self.retry_errors = retry_errors - - def __str__(self): - return 'Task has timed out, task name: {0}, total retry count: {1}, total wait time: {2}, ' \ - 'jitter enabled: {3}, retry errors: {4}'.format(self.task_name, self.max_attempts, - self.total_wait_time, self.jitter_enabled, - self.retry_errors) - - -def retry(time_unit, multiplier, backoff_coefficient, max_delay, max_attempts, expiration_duration, enable_jitter): - """ - The retry function will keep retrying `task_to_try` until either: - (1) it returns None, then retry() finishes - (2) `max_attempts` is reached, then retry() raises an exception. - (3) if retrying one more time will cause total wait time to go above: `expiration_duration`, then - retry() raises an exception - - Beware that any exception raised by task_to_try won't get surfaced until (2) or (3) is satisfied. - - At step n, it sleeps for [0, delay), where delay is defined as the following: - `delay = min(max_delay, multiplier * (backoff_coefficient ** (n - 1))) * time_unit` seconds - - Additionally, if you enable jitter, for each retry, the function will instead sleep for: - random.random() * sleep, that is [0, sleep) seconds. - - :param time_unit: This field represents a fraction of a second, which is used as a - multiplier to compute the amount of time to sleep. - :type time_unit: float - - :param multiplier: The initial wait duration for the first retry. - :type multiplier: float - - :param backoff_coefficient: the base value for exponential retry. - :type backoff_coefficient: float - - :param max_delay: The maximum amount of time to wait per try. - :type max_delay: float - - :param max_attempts: This method will retry up to this value. - :type max_attempts: int - - :param expiration_duration: the maximum amount of time retry can wait. - :type expiration_duration: float - - :param enable_jitter: Setting this to true will add jitter. - :type enable_jitter: bool - """ - - def deco_retry(task_to_try): - @wraps(task_to_try) - def retry_impl(*args, **kwargs): - total_wait_time = 0 - have_tried = 0 - retry_errors = [] - while have_tried < max_attempts: - try: - task_to_try(*args, **kwargs) - return - except Exception as e: - retry_errors.append(e) - going_to_sleep_for = min(max_delay, multiplier * (backoff_coefficient ** have_tried)) - if enable_jitter: - going_to_sleep_for = random.random() * going_to_sleep_for - - duration = going_to_sleep_for * time_unit - if total_wait_time + duration > expiration_duration: - raise RetryTimeoutException(task_to_try.__name__, have_tried, max_attempts, total_wait_time, - multiplier, backoff_coefficient, enable_jitter, retry_errors) - - runtime_logger.warn('Retrying [{0}], going to sleep for {1} seconds, exception stacktrace:\n{2}' - .format(task_to_try.__name__, duration, traceback.format_exc())) - time.sleep(duration) - total_wait_time += duration - have_tried += 1 - raise RetryTimeoutException(task_to_try.__name__, have_tried, max_attempts, total_wait_time, multiplier, - backoff_coefficient, enable_jitter, retry_errors) - - return retry_impl - return deco_retry diff --git a/examples/apps/greengrass-hello-world/greengrasssdk/IoTDataPlane.py b/examples/apps/greengrass-hello-world/greengrasssdk/IoTDataPlane.py old mode 100644 new mode 100755 index b2bd578f02..a04bd6d5c1 --- a/examples/apps/greengrass-hello-world/greengrasssdk/IoTDataPlane.py +++ b/examples/apps/greengrass-hello-world/greengrasssdk/IoTDataPlane.py @@ -112,11 +112,12 @@ def publish(self, **kwargs): } } - customer_logger.info('Publishing message on topic "{}" with Payload "{}"'.format(topic, payload)) + customer_logger.debug('Publishing message on topic "{}" with Payload "{}"'.format(topic, payload)) self.lambda_client._invoke_internal( function_arn, payload, - base64.b64encode(json.dumps(client_context).encode()) + base64.b64encode(json.dumps(client_context).encode()), + 'Event' ) def _get_required_parameter(self, parameter_name, **kwargs): @@ -135,7 +136,7 @@ def _shadow_op(self, op, thing_name, payload): } } - customer_logger.info('Calling shadow service on topic "{}" with payload "{}"'.format(topic, payload)) + customer_logger.debug('Calling shadow service on topic "{}" with payload "{}"'.format(topic, payload)) response = self.lambda_client._invoke_internal( function_arn, payload, diff --git a/examples/apps/greengrass-hello-world/greengrasssdk/Lambda.py b/examples/apps/greengrass-hello-world/greengrasssdk/Lambda.py old mode 100644 new mode 100755 index 22b7d88a1e..9fdeaf0030 --- a/examples/apps/greengrass-hello-world/greengrasssdk/Lambda.py +++ b/examples/apps/greengrass-hello-world/greengrasssdk/Lambda.py @@ -56,9 +56,13 @@ def invoke(self, **kwargs): final_qualifier = arn_qualifier if arn_qualifier else extraneous_qualifier - function_arn = FunctionArnFields.build_arn_string( - arn_fields.region, arn_fields.account_id, arn_fields.name, final_qualifier - ) + try: + # GGC v1.9.0 or newer + function_arn = FunctionArnFields.build_function_arn(arn_fields.unqualified_arn, final_qualifier) + except AttributeError: + # older GGC version + raise AttributeError('class FunctionArnFields has no attribute \'build_function_arn\'. build_function_arn ' + 'is introduced in GGC v1.9.0. Please check your GGC version.') # ClientContext must be base64 if given, but is an option parameter try: @@ -76,7 +80,7 @@ def invoke(self, **kwargs): # Payload is an optional parameter payload = kwargs.get('Payload', b'') invocation_type = kwargs.get('InvocationType', 'RequestResponse') - customer_logger.info('Invoking local lambda "{}" with payload "{}" and client context "{}"'.format( + customer_logger.debug('Invoking local lambda "{}" with payload "{}" and client context "{}"'.format( function_arn, payload, client_context)) # Post the work to IPC and return the result of that work @@ -89,7 +93,7 @@ def _invoke_internal(self, function_arn, payload, client_context, invocation_typ give this Lambda client a raw payload/client context to invoke with, rather than having it built for them. This lets you include custom ExtensionMap_ values like subject which are needed for our internal pinned Lambdas. """ - customer_logger.info('Invoking Lambda function "{}" with Greengrass Message "{}"'.format(function_arn, payload)) + customer_logger.debug('Invoking Lambda function "{}" with Greengrass Message "{}"'.format(function_arn, payload)) try: invocation_id = self.ipc.post_work(function_arn, payload, client_context, invocation_type) diff --git a/examples/apps/greengrass-hello-world/greengrasssdk/SecretsManager.py b/examples/apps/greengrass-hello-world/greengrasssdk/SecretsManager.py new file mode 100755 index 0000000000..2a8385e256 --- /dev/null +++ b/examples/apps/greengrass-hello-world/greengrasssdk/SecretsManager.py @@ -0,0 +1,160 @@ +# +# Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# + +import json +import logging +from datetime import datetime +from decimal import Decimal + +from greengrasssdk import Lambda +from greengrass_common.env_vars import MY_FUNCTION_ARN, SECRETS_MANAGER_FUNCTION_ARN + +# Log messages in the SDK are part of customer's log because they're helpful for debugging +# customer's lambdas. Since we configured the root logger to log to customer's log and set the +# propagate flag of this logger to True. The log messages submitted from this logger will be +# sent to the customer's local Cloudwatch handler. +customer_logger = logging.getLogger(__name__) +customer_logger.propagate = True + +KEY_NAME_PAYLOAD = 'Payload' +KEY_NAME_STATUS = 'Status' +KEY_NAME_MESSAGE = 'Message' +KEY_NAME_SECRET_ID = 'SecretId' +KEY_NAME_VERSION_ID = 'VersionId' +KEY_NAME_VERSION_STAGE = 'VersionStage' +KEY_NAME_CREATED_DATE = "CreatedDate" + + +class SecretsManagerError(Exception): + pass + + +class Client: + def __init__(self): + self.lambda_client = Lambda.Client() + + def get_secret_value(self, **kwargs): + r""" + Call secrets manager lambda to obtain the requested secret value. + + :Keyword Arguments: + * *SecretId* (``string``) -- + [REQUIRED] + Specifies the secret containing the version that you want to retrieve. You can specify either the + Amazon Resource Name (ARN) or the friendly name of the secret. + * *VersionId* (``string``) -- + Specifies the unique identifier of the version of the secret that you want to retrieve. If you + specify this parameter then don't specify ``VersionStage`` . If you don't specify either a + ``VersionStage`` or ``SecretVersionId`` then the default is to perform the operation on the version + with the ``VersionStage`` value of ``AWSCURRENT`` . + + This value is typically a UUID-type value with 32 hexadecimal digits. + * *VersionStage* (``string``) -- + Specifies the secret version that you want to retrieve by the staging label attached to the + version. + + Staging labels are used to keep track of different versions during the rotation process. If you + use this parameter then don't specify ``SecretVersionId`` . If you don't specify either a + ``VersionStage`` or ``SecretVersionId`` , then the default is to perform the operation on the + version with the ``VersionStage`` value of ``AWSCURRENT`` . + + :returns: (``dict``) -- + * *ARN* (``string``) -- + The ARN of the secret. + * *Name* (``string``) -- + The friendly name of the secret. + * *VersionId* (``string``) -- + The unique identifier of this version of the secret. + * *SecretBinary* (``bytes``) -- + The decrypted part of the protected secret information that was originally provided as + binary data in the form of a byte array. The response parameter represents the binary data + as a base64-encoded string. + + This parameter is not used if the secret is created by the Secrets Manager console. + + If you store custom information in this field of the secret, then you must code your Lambda + rotation function to parse and interpret whatever you store in the ``SecretString`` or + ``SecretBinary`` fields. + * *SecretString* (``string``) -- + The decrypted part of the protected secret information that was originally provided as a + string. + + If you create this secret by using the Secrets Manager console then only the ``SecretString`` + parameter contains data. Secrets Manager stores the information as a JSON structure of + key/value pairs that the Lambda rotation function knows how to parse. + + If you store custom information in the secret by using the CreateSecret , UpdateSecret , or + PutSecretValue API operations instead of the Secrets Manager console, or by using the + *Other secret type* in the console, then you must code your Lambda rotation function to + parse and interpret those values. + * *VersionStages* (``list``) -- + A list of all of the staging labels currently attached to this version of the secret. + * (``string``) -- + * *CreatedDate* (``datetime``) -- + The date and time that this version of the secret was created. + """ + + secret_id = self._get_required_parameter(KEY_NAME_SECRET_ID, **kwargs) + version_id = kwargs.get(KEY_NAME_VERSION_ID, '') + version_stage = kwargs.get(KEY_NAME_VERSION_STAGE, '') + + if version_id: # TODO: Remove this once we support query by VersionId + raise SecretsManagerError('Query by VersionId is not yet supported') + if version_id and version_stage: + raise ValueError('VersionId and VersionStage cannot both be specified at the same time') + + request_payload_bytes = self._generate_request_payload_bytes(secret_id=secret_id, + version_id=version_id, + version_stage=version_stage) + + customer_logger.debug('Retrieving secret value with id "{}", version id "{}" version stage "{}"' + .format(secret_id, version_id, version_stage)) + response = self.lambda_client._invoke_internal( + SECRETS_MANAGER_FUNCTION_ARN, + request_payload_bytes, + b'', # We do not need client context for Secrets Manager back-end lambda + ) # Use Request/Response here as we are mimicking boto3 Http APIs for SecretsManagerService + + payload = response[KEY_NAME_PAYLOAD].read() + payload_dict = json.loads(payload.decode('utf-8')) + + # All customer facing errors are presented within the response payload. For example: + # { + # "code": 404, + # "message": "Resource not found" + # } + if KEY_NAME_STATUS in payload_dict and KEY_NAME_MESSAGE in payload_dict: + raise SecretsManagerError('Request for secret value returned error code {} with message {}'.format( + payload_dict[KEY_NAME_STATUS], payload_dict[KEY_NAME_MESSAGE] + )) + + # Time is serialized as epoch timestamp (int) upon IPC routing. We need to deserialize it back to datetime object in Python + payload_dict[KEY_NAME_CREATED_DATE] = datetime.fromtimestamp( + # Cloud response contains timestamp in milliseconds while datetime.fromtimestamp is expecting seconds + Decimal(payload_dict[KEY_NAME_CREATED_DATE]) / Decimal(1000) + ) + + return payload_dict + + def _generate_request_payload_bytes(self, secret_id, version_id, version_stage): + request_payload = { + KEY_NAME_SECRET_ID: secret_id, + } + if version_stage: + request_payload[KEY_NAME_VERSION_STAGE] = version_stage + + # TODO: Add VersionId once we support query by VersionId + + # The allowed chars for secret id and version stage are strictly enforced when customers are configuring them + # through Secrets Manager Service in the cloud: + # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_CreateSecret.html#API_CreateSecret_RequestSyntax + return json.dumps(request_payload).encode() + + @staticmethod + def _get_required_parameter(parameter_name, **kwargs): + if parameter_name not in kwargs: + raise ValueError('Parameter "{parameter_name}" is a required parameter but was not provided.'.format( + parameter_name=parameter_name + )) + return kwargs[parameter_name] diff --git a/examples/apps/greengrass-hello-world/greengrasssdk/__init__.py b/examples/apps/greengrass-hello-world/greengrasssdk/__init__.py old mode 100644 new mode 100755 index 456f7e14c2..916d0dc264 --- a/examples/apps/greengrass-hello-world/greengrasssdk/__init__.py +++ b/examples/apps/greengrass-hello-world/greengrasssdk/__init__.py @@ -1,6 +1,9 @@ # -# Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. # from .client import client from .Lambda import StreamingBody + +__version__ = '1.4.0' +INTERFACE_VERSION = '1.3' diff --git a/examples/apps/greengrass-hello-world/greengrasssdk/client.py b/examples/apps/greengrass-hello-world/greengrasssdk/client.py old mode 100644 new mode 100755 index 9ca6a7df72..53c92d0ff8 --- a/examples/apps/greengrass-hello-world/greengrasssdk/client.py +++ b/examples/apps/greengrass-hello-world/greengrasssdk/client.py @@ -8,6 +8,8 @@ def client(client_type, *args): from .Lambda import Client elif client_type == 'iot-data': from .IoTDataPlane import Client + elif client_type == 'secretsmanager': + from .SecretsManager import Client else: raise Exception('Client type {} is not recognized.'.format(repr(client_type))) diff --git a/examples/apps/greengrass-hello-world/greengrasssdk/utils/__init__.py b/examples/apps/greengrass-hello-world/greengrasssdk/utils/__init__.py old mode 100644 new mode 100755 diff --git a/examples/apps/greengrass-hello-world/greengrasssdk/utils/testing.py b/examples/apps/greengrass-hello-world/greengrasssdk/utils/testing.py old mode 100644 new mode 100755 diff --git a/examples/apps/greengrass-hello-world/template.yaml b/examples/apps/greengrass-hello-world/template.yaml index 8e33c96baf..e91a69e741 100644 --- a/examples/apps/greengrass-hello-world/template.yaml +++ b/examples/apps/greengrass-hello-world/template.yaml @@ -2,15 +2,15 @@ AWSTemplateFormatVersion: '2010-09-09' Transform: 'AWS::Serverless-2016-10-31' Description: >- Deploy this lambda to a Greengrass core where it will send a hello world message to a topic -Parameters: - IdentityNameParameter: +Parameters: + IdentityNameParameter: Type: String Resources: greengrasshelloworld: Type: 'AWS::Serverless::Function' Properties: Handler: greengrassHelloWorld.function_handler - Runtime: python2.7 + Runtime: python3.7 CodeUri: . Description: >- Deploy this lambda to a Greengrass core where it will send a hello world message to a topic @@ -18,4 +18,4 @@ Resources: Timeout: 3 Policies: - SESSendBouncePolicy: - IdentityName: !Ref IdentityNameParameter \ No newline at end of file + IdentityName: !Ref IdentityNameParameter