Merge pull request #17 from cristim/self-contained

Self contained execution, without any dependencies

.gitignore

@@ -1,3 +1,6 @@
 build
 autospotting
 coverage.out
+lambda/GIT_SHA
+lambda/autospotting_lambda
+lambda/instances.json

.travis.yml

@@ -7,8 +7,6 @@ go:
 - tip
 
 before_install:
-- sudo apt-get -qq update
-- sudo apt-get install -y upx
 - go get github.com/mattn/goveralls
 - go get golang.org/x/tools/cmd/cover
 

Makefile

@@ -1,32 +1,46 @@
 BINARY=autospotting
 LAMBDA_BINARY=autospotting_lambda
 LOCAL_PATH=build/s3/dv
+BUCKET_NAME=cloudprowess
+
+# upstream data
+#INSTANCES_URL="https://raw.githubusercontent.com/powdahound/ec2instances.info/master/www/instances.json"
+# my Github fork
+INSTANCES_URL="https://raw.githubusercontent.com/cristim/ec2instances.info/master/www/instances.json"
 
 all: build_local
 
 release: upload
 
-build_local:
+check_deps:
+	./check_deps.sh
+
+build_local: check_deps
 	go build $(GOFLAGS)
 
-build_lambda:
+build_lambda_binary: check_deps
 	GOOS=linux GOARCH=amd64 go build -o ${LAMBDA_BINARY}
 
-strip: build_lambda
+strip: build_lambda_binary
 	strip ${LAMBDA_BINARY}
-	upx --lzma --best ${LAMBDA_BINARY}
 
-install: strip
-	CHECKSUM=$$(sha256sum ${LAMBDA_BINARY} | cut -f1 -d " " ); \
-	FILENAME=${BINARY}_$$CHECKSUM ; \
-	mkdir -p ${LOCAL_PATH} ; \
-	mv ${LAMBDA_BINARY} ${LOCAL_PATH}/$$FILENAME ; \
-	echo $$FILENAME > ${LOCAL_PATH}/latest_agent
+lambda: strip
+	git rev-parse HEAD | cut -c 1-8 > lambda/GIT_SHA
+	mv ${LAMBDA_BINARY} lambda/
+	curl ${INSTANCES_URL} --output lambda/instances.json
+	zip -9 -v -j lambda lambda/*
+	rm -rf ${LOCAL_PATH}
+	mkdir -p ${LOCAL_PATH}
+	mv lambda.zip ${LOCAL_PATH}
+
+install: lambda
 
 upload: install
-	aws s3 sync build/s3/ s3://cloudprowess/
+	aws s3 sync build/s3/ s3://${BUCKET_NAME}/
 
 test: build_local
-	./autospotting -e  core/test_data/event.json -c core/test_data/context.json
+	./autospotting core/test_data/json_instance/instances.json
+
 cover:
-	go test -covermode=count -coverprofile=coverage.out ./core && go tool cover -html=coverage.out
+	go test -covermode=count -coverprofile=coverage.out ./core
+	go tool cover -html=coverage.out

autospotting.go

@@ -2,22 +2,24 @@ package main
 
 import (
 	"fmt"
+	"log"
+	"os"
 
 	autospotting "github.com/cristim/autospotting/core"
 )
 
-// upstream data
-// const instancesURL = "https://raw.githubusercontent.com/powdahound/ec2instances.info/master/www/instances.json"
-
-// my Github fork
-const instancesURL = "https://raw.githubusercontent.com/cristim/ec2instances.info/master/www/instances.json"
-
 // Main intefaces with the lambda event handler wrapper written
 // in javascript.
 func main() {
 	fmt.Println("Starting agent...")
 
-	autospotting.Run(instancesURL)
+	if len(os.Args) != 2 {
+		log.Fatal("The program needs a command line argument: ",
+			os.Args[0], " <instances.json>")
+	}
+	instancesFile := os.Args[1]
+
+	autospotting.Run(instancesFile)
 
 	fmt.Println("Exiting main, nothing left to do")
 }

check_deps.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+function check_dep {
+  command -v $1 >/dev/null 2>&1 || {
+    echo >&2 "The required dependency program '$1' is not installed. Aborting."
+    exit 1
+  }
+}
+
+DEPS="aws curl git go strip zip"
+
+echo "Checking for the presence of the following dependencies: ${DEPS}"
+
+for DEP in $DEPS
+do
+  check_dep $DEP && echo "Found ${DEP}"
+done
+echo "All dependencies were successfully found, we're good to go!"

core/json_instance.go

@@ -3,7 +3,7 @@ package autospotting
 import (
 	"encoding/json"
 	"io/ioutil"
-	"net/http"
+	"log"
 )
 
 // AWS Instances JSON Structure Definitions
@@ -67,20 +67,11 @@ type regionPrices struct {
 
 type jsonInstances []jsonInstance
 
-func (i *jsonInstances) loadFromURL(url string) error {
+func (i *jsonInstances) loadFromFile(fileName string) error {
 
-	response, err := http.Get(url)
+	contents, err := ioutil.ReadFile(fileName)
 	if err != nil {
-		logger.Println(err.Error())
-		return err
-	}
-
-	defer response.Body.Close()
-
-	contents, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		logger.Println(err.Error())
-		return err
+		log.Fatal(err.Error())
 	}
 
 	// logger.Println(string(contents))

core/json_instance_test.go

@@ -47,23 +47,3 @@ func runTestCase(t *testing.T, tc testCase) {
 	}
 
 }
-
-func testLoadFromURL(t *testing.T) {
-	dataFile := "test_data/json_instance/instances.json"
-
-	ts := httpMock(dataFile)
-	defer ts.Close()
-
-	var i jsonInstances
-	i.loadFromURL(ts.URL)
-
-	testCases := []testCase{
-		// i, index, instanceType, numCPU, RAM, onDemandUSEast
-		{i[14], "t1.micro", 1, 0.613, "0.02"},
-	}
-
-	for _, tc := range testCases {
-		runTestCase(t, tc)
-	}
-
-}

core/main.go

@@ -3,7 +3,7 @@ package autospotting
 // Run starts processing all AWS regions looking for AutoScaling groups
 // enabled and taking action by replacing more pricy on-demand instances with
 // compatible and cheaper spot instances.
-func Run(instancesURL string) {
+func Run(instancesFile string) {
 
 	initLogger()
 
@@ -13,7 +13,7 @@ func Run(instancesURL string) {
 	var jsonInst jsonInstances
 
 	logger.Println("Loading on-demand instance pricing information")
-	jsonInst.loadFromURL(instancesURL)
+	jsonInst.loadFromFile(instancesFile)
 
 	// logger.Println(spew.Sdump(jsonInst))
 

lambda/wrapper.py

@@ -5,78 +5,24 @@
 """
 
 import json
-import os.path
-import sys
 
-from urlparse import urljoin
-from urllib2 import URLError, urlopen
-from subprocess import call, PIPE, STDOUT, check_output
-
-URL_PATH = 'https://cdn.cloudprowess.com/dv/'
+from subprocess import call, STDOUT
 
+BINARY = './autospotting_lambda'
+JSON_INSTANCES = 'instances.json'
+GIT_SHA = 'GIT_SHA'
 
 def lambda_handler(event, context):
     """ Main entry point for Lambda """
-    print 'Received event: ' + json.dumps(event, indent=2)
-    print "Context log stream: " + context.log_stream_name
-
-    try:
-        filename = get_latest_agent_filename()
-        download_agent_if_missing(filename)
-        run_agent(filename)
-
-    except URLError as ex:
-        print 'Error: ', ex
-
-
-def get_latest_agent_filename():
-    """Determines the filename of the latest released agent golang binary"""
-    return urlopen(
-        urljoin(
-            URL_PATH,
-            'latest_agent'
-        )
-    ).read().strip()
-
 
-def download_agent_if_missing(filename):
-    """ Downloads the agent if missing from the current Lambda run """
-    if file_missing(filename):
-        print filename+'is missing, downloading it first'
-        download(filename)
-
-
-def file_missing(filename):
-    """Checks file for existence"""
-    return not os.path.isfile(filename)
-
-
-def download(filename):
-    """ Downloads a file from the base URL path into /tmp/<filename> """
-    print "Downloading", filename
-    file_content = urlopen(
-        urljoin(URL_PATH, filename)
-    )
-    write_data_to_file(
-        file_content.read(),
-        os.path.join(
-            '/tmp',
-            filename
-            )
-        )
-
-
-def write_data_to_file(data, filename):
-    """ Writes data to filename """
-    with open(filename, 'wb') as outfile:
-        outfile.write(data)
+    with open(GIT_SHA, 'r') as sha:
+        print 'Starting AutoSpotting, built from the git SHA', sha.read()
 
+    print 'Received event: ' + json.dumps(event, indent=2)
 
-def run_agent(filename):
-    """ Runs the agent witn the required params """
-    print "Running", filename
+    print "Running", BINARY, JSON_INSTANCES
+    call([BINARY, JSON_INSTANCES], stderr=STDOUT)
 
-    binary_path = os.path.join('/tmp', filename)
 
-    os.chmod(binary_path, 0755)
-    call([binary_path], stderr=STDOUT)
+if __name__ == '__main__':
+    lambda_handler(None, None)