Bean-counter added

skarlekar · Apr 22, 2019 · 237c0b0 · 237c0b0
1 parent 0a52976
commit 237c0b0
Show file tree

Hide file tree

Showing 17 changed files with 330 additions and 0 deletions.
diff --git a/bean-counter/bean-counter/Dockerfile b/bean-counter/bean-counter/Dockerfile
@@ -0,0 +1,20 @@
+FROM ubuntu:16.04
+
+MAINTAINER Srini Karlekar <skarlekar@yahoo.com>
+
+RUN apt-get -y update
+RUN apt-get -y install ipython python-opencv python-scipy python-numpy python-setuptools python-pip python-pygame
+RUN apt-get -y install libjpeg-dev
+RUN apt-get -y install libjpeg8-dev
+RUN pip install https://github.com/sightmachine/SimpleCV/zipball/master
+RUN pip install svgwrite
+RUN pip install flask
+RUN pip install --no-cache-dir -I Pillow==2.1.0
+
+WORKDIR /app
+COPY app/app.py /app
+
+EXPOSE 8080
+
+ENTRYPOINT ["python"]
+CMD ["app.py"]
diff --git a/bean-counter/bean-counter/app/app.py b/bean-counter/bean-counter/app/app.py
@@ -0,0 +1,84 @@
+from flask import Flask
+from SimpleCV import Image, Blob, Display
+import numpy as np
+
+app = Flask(__name__)
+
+testImageUrl = "https://s3.amazonaws.com/skarlekar-ffmpeg/raw/53_cents_small.jpg"
+s3BucketUrl = "https://s3.amazonaws.com/skarlekar-ffmpeg/raw/{0}"
+
+coin_diameter_values = np.array([
+[ 19.05, 0.01],
+[ 21.21, 0.05],
+[ 17.7, 0.10],
+[ 24.26, 0.25]]);
+
+@app.route('/')
+def usage():
+  usageStr = "<html><head><title>Bean Counter Usage</title><h1>Bean Counter</h1></head><body><p><b>Usage:</b><br/><em>countme-test</em>: Runs a test on the image at this url: <a href='{0}'>Test image</a><br/><em>countme&#47;<i>image-url</i></em>: Counts the coins in the image passed as image-url<br/></p></body></html>".format(testImageUrl)
+  return usageStr
+
+@app.route('/countme/<input_str>')
+def count_me(input_str):
+    processUrl = s3BucketUrl.format(input_str)
+    print "Processing image at: {0}".format(processUrl)
+    try:
+        value = process_image(processUrl)
+    except:
+        print "Cannot read image at: {0}".format(processUrl)
+        value = "Bad Image"
+    return value
+
+@app.route('/countme-test/')
+def coin_count():
+    return process_image(testImageUrl)
+
+
+
+def process_image(image_url):
+        # Print the url for image that is being processed.
+        print "Processing image: {}".format(image_url)
+
+        # Construct the Image object from the image URL
+        img = Image(image_url)
+
+        # Find the coin blobs from the image after inverting the image
+        coins = img.invert().findBlobs(minsize = 500)
+
+        # Use a quarter to calibrate the largest coin in the coin blobs.
+        # Logic:
+        # coin_diameter_values[3,0] is the size of the US Mint Quarter.
+        # Find the largest radius of all the coins in the image. This will be our reference Quarter.
+        # usmint_quarter/radius_of_our_quarter will provide the calibration factor.
+        # In other words: convert pixels to millimeter.
+        px2mm = coin_diameter_values[3,0] / max([c.radius()*2 for c in coins])
+
+        # Initialize index & total value
+        i=0
+        value = 0.0
+
+        # For each blob in the coins blob list
+        for c in coins:
+            i=i+1
+            # Find the diameter of this coin blob & normalize to the calibration factor
+            # ie., find the diameter in millimeter of this coin.
+            diameter_in_mm = c.radius() * 2 * px2mm
+            # Get an array of values for difference between this coin and all the US Mint coins.
+            distance = np.abs(diameter_in_mm - coin_diameter_values[:,0])
+            #print "Coin diameter: " , diameter_in_mm, " Distance: ", distance
+            # Find the coin with the smallest difference. This is our best guess on the coin type.
+            index = np.where(distance == np.min(distance))[0][0]
+
+            # Get the value of the coin and add it to the total amount
+            coinValue = coin_diameter_values[index, 1]
+            value += coinValue
+            #coinImg = c.crop()
+            #coinImg.drawText(str(coinValue))
+            #coinImg.save("Results/coin"+str(i)+".png")
+
+        message = "The total value of the coins in the image is ${0}".format(value)
+        print message
+        return "{}".format(value)
+
+if __name__ == '__main__':
+  app.run(host='0.0.0.0', port=8080)
diff --git a/bean-counter/create-bean-counter-cluster.sh b/bean-counter/create-bean-counter-cluster.sh
@@ -0,0 +1 @@
+aws ecs create-cluster --cluster-name bean-counter-cluster
diff --git a/bean-counter/create-bean-counter-log-group.sh b/bean-counter/create-bean-counter-log-group.sh
@@ -0,0 +1 @@
+aws logs create-log-group --log-group-name /ecs/bean-counter-service
diff --git a/bean-counter/create-bean-counter-repository.sh b/bean-counter/create-bean-counter-repository.sh
@@ -0,0 +1,9 @@
+#export ECR_REPO_URI=$(aws ecr create-repository --repository-name bean-counter |  jq '.repository.repositoryUri' | sed "s/\"//g")
+aws ecr create-repository --repository-name bean-counter 
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error: Repo may already exist"
+fi
+export ECR_REPO_URI=$(aws ecr describe-repositories | jq '.repositories[] | if .repositoryName == "bean-counter" then .repositoryUri else null end' | grep -v null | sed "s/\"//g")
+
+
diff --git a/bean-counter/create-bean-counter-service-def-template.json b/bean-counter/create-bean-counter-service-def-template.json
@@ -0,0 +1,46 @@
+{
+    "cluster": "bean-counter-cluster",
+    "serviceName": "bean-counter-service",
+    "taskDefinition": "bean-counter-service-task",
+    "loadBalancers": [
+        {
+            "targetGroupArn": "ALB_TARGET_GROUP_ARN",
+            "containerName": "bean-counter-container",
+            "containerPort": 8080
+        }
+    ],
+    "desiredCount": 2,
+    "clientToken": "",
+    "launchType": "FARGATE",
+    "platformVersion": "LATEST",
+    "role": "",
+    "deploymentConfiguration": {
+        "maximumPercent": 100,
+        "minimumHealthyPercent": 50
+    },
+    "networkConfiguration": {
+        "awsvpcConfiguration": {
+            "subnets": [
+		"SUBNET_1",
+		"SUBNET_2"
+            ],
+            "securityGroups": [
+		"SEC_GROUP"
+            ],
+            "assignPublicIp": "ENABLED"
+        }
+    },
+    "healthCheckGracePeriodSeconds": 0,
+    "schedulingStrategy": "REPLICA",
+    "deploymentController": {
+        "type": "ECS"
+    },
+    "tags": [
+        {
+            "key": "Purpose",
+            "value": "Demo"
+        }
+    ],
+    "enableECSManagedTags": true,
+    "propagateTags": "SERVICE"
+}
diff --git a/bean-counter/create-bean-counter-service.sh b/bean-counter/create-bean-counter-service.sh
@@ -0,0 +1 @@
+aws ecs create-service --cli-input-json file://temp/create-bean-counter-service-definition.json
diff --git a/bean-counter/generate-bean-counter-service-definition.sh b/bean-counter/generate-bean-counter-service-definition.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+: ${SUBNET1? "Need to set environment variable SUBNET1"}
+: ${SUBNET2? "Need to set environment variable SUBNET2"}
+: ${SECURITYGROUP? "Need to set environment variable SECURITYGROUP"}
+: ${TG_ARN? "Need to set environment variable TG_ARN"}
+
+DIRECTORY=temp
+if [ ! -d "$DIRECTORY" ]; then
+  echo Creating temp directory
+  mkdir temp
+fi
+
+cat create-bean-counter-service-def-template.json| jq '.loadBalancers[0].targetGroupArn = env.TG_ARN' | jq '.networkConfiguration.awsvpcConfiguration.subnets = [env.SUBNET1, env.SUBNET2]'| jq '.networkConfiguration.awsvpcConfiguration.securityGroups = [env.SECURITYGROUP]' > temp/create-bean-counter-service-definition.json
+
diff --git a/bean-counter/generate-bean-counter-task-definition.sh b/bean-counter/generate-bean-counter-task-definition.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+: ${TASK_ROLE_ARN? "Need to set environment variable TASK_ROLE_ARN"}
+
+DIRECTORY=temp
+if [ ! -d "$DIRECTORY" ]; then
+  echo Creating temp directory
+  mkdir temp
+fi
+
+export ECR_IMAGE_LATEST=$ECR_REPO_URI:latest
+cat register-bean-counter-task-def-template.json | jq '.containerDefinitions[0].image = env.ECR_IMAGE_LATEST' | jq '.executionRoleArn = env.TASK_ROLE_ARN'| jq '.taskRoleArn = env.TASK_ROLE_ARN' > temp/register-bean-counter-task-definition.json
+
diff --git a/bean-counter/push-to-ecr.sh b/bean-counter/push-to-ecr.sh
@@ -0,0 +1,4 @@
+$(aws ecr get-login --no-include-email --region us-east-1)
+docker build -t bean-counter bean-counter
+docker tag bean-counter:latest $ECR_REPO_URI:latest
+docker push $ECR_REPO_URI:latest
diff --git a/bean-counter/register-bean-counter-task-def-template.json b/bean-counter/register-bean-counter-task-def-template.json
@@ -0,0 +1,32 @@
+{
+    "requiresCompatibilities": [
+        "FARGATE"
+    ],
+    "containerDefinitions": [
+        {
+            "name": "bean-counter-container",
+            "image": "MY_ECR_IMAGE:latest",
+            "essential": true,
+            "portMappings": [
+		{
+			"containerPort": 8080,
+			"hostPort": 8080
+		}
+	     ],
+            "logConfiguration": {
+                "logDriver": "awslogs",
+                "options": {
+                    "awslogs-group": "/ecs/bean-counter-service",
+                    "awslogs-region": "us-east-1",
+                    "awslogs-stream-prefix": "ecs"
+                }
+            }
+        }
+    ],
+    "networkMode": "awsvpc",
+    "memory": "1024",
+    "cpu": "256",
+    "executionRoleArn": "MY_EXECUTION_ROLE_ARN",
+    "family": "bean-counter-service-task",
+    "taskRoleArn": "MY_TASK_ROLE_ARN"
+}
diff --git a/bean-counter/register-bean-counter-task.sh b/bean-counter/register-bean-counter-task.sh
@@ -0,0 +1 @@
+aws ecs register-task-definition --cli-input-json file://temp/register-bean-counter-task-definition.json
diff --git a/bean-counter/scaling-policy.json b/bean-counter/scaling-policy.json
@@ -0,0 +1,8 @@
+{
+     "TargetValue": 75.0,
+     "PredefinedMetricSpecification": {
+         "PredefinedMetricType": "ECSServiceAverageCPUUtilization"
+     },
+     "ScaleOutCooldown": 60,
+    "ScaleInCooldown": 60
+}
diff --git a/bean-counter/set-scaling-policy.sh b/bean-counter/set-scaling-policy.sh
@@ -0,0 +1,14 @@
+aws application-autoscaling register-scalable-target \
+--service-namespace ecs \
+--scalable-dimension ecs:service:DesiredCount \
+--resource-id service/bean-counter-cluster/bean-counter-service \
+--min-capacity 2 \
+--max-capacity 4
+
+aws application-autoscaling put-scaling-policy \
+--service-namespace ecs \
+--scalable-dimension ecs:service:DesiredCount \
+--resource-id service/bean-counter-cluster/bean-counter-service \
+--policy-name cpu75-target-tracking-scaling-policy \
+--policy-type TargetTrackingScaling \
+--target-tracking-scaling-policy-configuration file://scaling-policy.json
diff --git a/bean-counter/temp/create-bean-counter-service-definition.json b/bean-counter/temp/create-bean-counter-service-definition.json
@@ -0,0 +1,46 @@
+{
+  "cluster": "bean-counter-cluster",
+  "serviceName": "bean-counter-service",
+  "taskDefinition": "bean-counter-service-task",
+  "loadBalancers": [
+    {
+      "targetGroupArn": "arn:aws:elasticloadbalancing:us-east-1:219104658389:targetgroup/My-Fargate-TG/718fcdec1728a987",
+      "containerName": "bean-counter-container",
+      "containerPort": 8080
+    }
+  ],
+  "desiredCount": 2,
+  "clientToken": "",
+  "launchType": "FARGATE",
+  "platformVersion": "LATEST",
+  "role": "",
+  "deploymentConfiguration": {
+    "maximumPercent": 100,
+    "minimumHealthyPercent": 50
+  },
+  "networkConfiguration": {
+    "awsvpcConfiguration": {
+      "subnets": [
+        "subnet-0ff6d2dc15aaa56b3",
+        "subnet-059ec4683f49bd24f"
+      ],
+      "securityGroups": [
+        "sg-026753693033a9130"
+      ],
+      "assignPublicIp": "ENABLED"
+    }
+  },
+  "healthCheckGracePeriodSeconds": 0,
+  "schedulingStrategy": "REPLICA",
+  "deploymentController": {
+    "type": "ECS"
+  },
+  "tags": [
+    {
+      "key": "Purpose",
+      "value": "Demo"
+    }
+  ],
+  "enableECSManagedTags": true,
+  "propagateTags": "SERVICE"
+}
diff --git a/bean-counter/temp/register-bean-counter-task-definition.json b/bean-counter/temp/register-bean-counter-task-definition.json
@@ -0,0 +1,32 @@
+{
+  "requiresCompatibilities": [
+    "FARGATE"
+  ],
+  "containerDefinitions": [
+    {
+      "name": "bean-counter-container",
+      "image": "219104658389.dkr.ecr.us-east-1.amazonaws.com/bean-counter:latest",
+      "essential": true,
+      "portMappings": [
+        {
+          "containerPort": 8080,
+          "hostPort": 8080
+        }
+      ],
+      "logConfiguration": {
+        "logDriver": "awslogs",
+        "options": {
+          "awslogs-group": "/ecs/bean-counter-service",
+          "awslogs-region": "us-east-1",
+          "awslogs-stream-prefix": "ecs"
+        }
+      }
+    }
+  ],
+  "networkMode": "awsvpc",
+  "memory": "1024",
+  "cpu": "256",
+  "executionRoleArn": "arn:aws:iam::219104658389:role/my-ecs-tasks-role",
+  "family": "bean-counter-service-task",
+  "taskRoleArn": "arn:aws:iam::219104658389:role/my-ecs-tasks-role"
+}
diff --git a/bean-counter/test-scaling.sh b/bean-counter/test-scaling.sh
@@ -0,0 +1,3 @@
+export DNS=$(aws elbv2 describe-load-balancers | jq '.LoadBalancers[] | if .LoadBalancerName == "My-Fargate-ALB" then .DNSName else null end' | grep -v null | sed "s/\"//g")
+
+ab -n 100000 -c 100 -s 120 http://$DNS/