updated sqs client and env vars

README.md

@@ -67,8 +67,7 @@ If you are using Memory as your scaling mode:
 
 If you are using SQS as your scaling mode:
 
-    AS_SQS_NAME # name of the aws simple queue service
-    AS_SQS_ENDPOINT # endpoint url of the sqs service
+    AS_QUEUE_URL # full URL of the SQS queue
     AWS_ACCESS_KEY_ID # aws access key
     AWS_SECRET_ACCESS_KEY # aws secret key
     AWS_DEFAULT_REGION # aws region
@@ -160,8 +159,7 @@ The following examples execute the python application from the command line.
 
 #### SQS message queue length as autoscale trigger
 
-    export AS_SQS_NAME=testQueue
-    export AS_SQS_ENDPOINT=https://sqs.us-east-1.amazonaws.com
+    export AS_QUEUE_URL=
     export AWS_ACCESS_KEY_ID=
     export AWS_SECRET_ACCESS_KEY=
     export AWS_DEFAULT_REGION=us-east-1
@@ -180,6 +178,6 @@ The following examples execute the python application from the command line.
 
     python marathon_autoscaler.py --dcos-master https://leader.mesos --trigger_mode and --autoscale_multiplier 1.5 --max_instances 5 --marathon-app /group1/testapp --min_instances 1 --cool_down_factor 4 --scale_up_factor 3 --interval 10 --min_range 55.0,40.0 --max_range 75.0,60.0
 
-#### OR (CPU and Memory) as autoscale trigger
+#### OR (CPU or Memory) as autoscale trigger
 
     python marathon_autoscaler.py --dcos-master https://leader.mesos --trigger_mode or --autoscale_multiplier 1.5 --max_instances 5 --marathon-app /group1/testapp --min_instances 1 --cool_down_factor 4 --scale_up_factor 3 --interval 10 --min_range 55.0,55.0 --max_range 75.0,75.0

autoscaler/modes/scalesqs.py

@@ -2,7 +2,7 @@
 import sys
 import logging
 
-from boto3 import resource
+from boto3 import client
 from botocore.errorfactory import ClientError
 
 from autoscaler.modes.abstractmode import AbstractMode
@@ -18,37 +18,33 @@ def __init__(self, api_client=None, app=None, dimension=None):
         logger.setLevel(logging.ERROR)
 
         # Verify environment vars for SQS config exist
-        if 'AS_SQS_NAME' not in os.environ.keys():
-            self.log.error("AS_SQS_NAME env var is not set.")
+        if 'AS_QUEUE_URL' not in os.environ.keys():
+            self.log.error("AS_QUEUE_URL env var is not set.")
             sys.exit(1)
 
-        if 'AS_SQS_ENDPOINT' not in os.environ.keys():
-            self.log.error("AS_SQS_ENDPOINT env var is not set.")
-            sys.exit(1)
+        """Boto3 will use the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, 
+        and AWS_DEFAULT_REGION env vars as it's credentials
+        """
+        self.sqs = client('sqs')
+        self.url = os.environ.get('AS_QUEUE_URL')
 
     def get_value(self):
         """Get the approximate number of visible messages in a SQS queue
         """
-        endpoint_url = os.environ.get('AS_SQS_ENDPOINT')
-        queue_name = os.environ.get('AS_SQS_NAME')
-
-        self.log.debug("SQS queue name:  %s", queue_name)
 
         try:
-            """Boto3 will use the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, 
-                and AWS_DEFAULT_REGION env vars as it's credentials
-            """
-            sqs = resource(
-                'sqs',
-                endpoint_url=endpoint_url
+
+            attributes = self.sqs.get_queue_attributes(
+                QueueUrl=self.url,
+                AttributeNames=['ApproximateNumberOfMessages']
             )
-            queue = sqs.get_queue_by_name(QueueName=queue_name)
-            value = float(queue.attributes.get('ApproximateNumberOfMessages'))
+
+            value = float(attributes['Attributes']['ApproximateNumberOfMessages'])
+
         except ClientError as e:
             raise ValueError("Boto3 client error: %s", e.response)
 
-        self.log.info("Current available messages for queue %s = %s",
-                      queue_name, value)
+        self.log.info("Current available messages for queue is %s", value)
 
         return value
 

env.list

@@ -17,8 +17,7 @@ AS_USERID=
 AS_PASSWORD=
 AS_SECRET=
 # Scale based on SQS
-AS_SQS_NAME=
-AS_SQS_ENDPOINT=
+AS_QUEUE_URL=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_DEFAULT_REGION=

marathon_autoscaler.py

@@ -106,7 +106,7 @@ def timer(self):
     def autoscale(self, direction):
         """ Determine if scaling mode direction is below or above scaling
         factor. If scale_up/cool_down cycle count exceeds scaling
-        factor threshold, autoscale (up/down) will be triggered.
+        factor, autoscale (up/down) will be triggered.
         """
 
         if direction == 1:
@@ -241,7 +241,6 @@ def env_or_req(key):
 
     def run(self):
         """Main function
-        Runs the query - compute - act cycle
         """
         self.cool_down = 0
         self.scale_up = 0

marathon_definitions/autoscale_examples/autoscale-sqs-noauth-marathon.json

@@ -46,8 +46,7 @@
     "AS_SCALE_UP_FACTOR": "3",
     "AS_MARATHON_APP": "/group1/hello-dcos",
     "AS_INTERVAL": "30",
-    "AS_SQS_NAME": "testQueue",
-    "AS_SQS_ENDPOINT": "https://sqs.us-east-1.amazonaws.com",
+    "AS_QUEUE_URL": "https://sqs.us-east-1.amazonaws.com/ACCOUNT_NUMBER/test-auto-scale",
     "AS_MIN_RANGE": "3",
     "AS_MAX_RANGE": "10",
     "AWS_DEFAULT_REGION": "us-east-1",

marathon_definitions/target_app_examples/stress_memory.json

@@ -3,5 +3,5 @@
   "instances": 1,
   "cpus": 0.1,
   "mem": 16,
-  "cmd": "tail -f /dev/null;"
+  "cmd": "for i in $(seq 5); do BLOB=$(dd if=/dev/urandom bs=1k count=1024k); sleep 10s; done"
 }

marathon_definitions/target_app_examples/stress_sqs.json

@@ -0,0 +1,46 @@
+{
+  "id": "/test/stress-sqs",
+  "instances": 1,
+  "cpus": 0.1,
+  "mem": 128,
+  "disk": 0,
+  "gpus": 0,
+  "constraints": [],
+  "fetch": [],
+  "storeUrls": [],
+  "backoffSeconds": 1,
+  "backoffFactor": 1.15,
+  "maxLaunchDelaySeconds": 3600,
+  "container": {
+    "type": "DOCKER",
+    "volumes": [],
+    "docker": {
+      "image": "adamiezzi/stress-sqs:latest",
+      "portMappings": [],
+      "privileged": false,
+      "parameters": [],
+      "forcePullImage": true
+    }
+  },
+  "healthChecks": [],
+  "readinessChecks": [],
+  "dependencies": [],
+  "upgradeStrategy": {
+    "minimumHealthCapacity": 1,
+    "maximumOverCapacity": 1
+  },
+  "unreachableStrategy": {
+    "inactiveAfterSeconds": 300,
+    "expungeAfterSeconds": 600
+  },
+  "killSelection": "YOUNGEST_FIRST",
+  "portDefinitions": [],
+  "requirePorts": false,
+  "env": {
+    "INTERVAL": "10",
+    "QUEUE_URL": "https://sqs.us-east-1.amazonaws.com/ACCOUNT_ID/test-auto-scale",
+    "AWS_DEFAULT_REGION": "us-east-1",
+    "AWS_ACCESS_KEY_ID": "CHANGEME",
+    "AWS_SECRET_ACCESS_KEY": "CHANGEME"
+  }
+}