Merge fba1840 into 2febace

Dockerfile

@@ -15,15 +15,13 @@ LABEL maintainer='Rahul Verma <rv404674@gmail.com>'
 # Set the Current Working Directory inside the container
 WORKDIR /app
 
-# install dependencies
-COPY go.mod go.sum ./
+# copy the source from the current directory to the working directory inside the container
+# this will copy both go.mod go.sum as well. So no need to copy again
+COPY . .
 
 # Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed
 RUN go mod download
 
-# copy the source from the current directory to the working directory inside the container
-COPY . .
-
 # Build the Go app
 # RUN go build -o main .
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o main .
@@ -36,10 +34,13 @@ RUN apk --no-cache add ca-certificates
 WORKDIR /root/
 
 # COPY the Prebuilt binary file from the previous stage
+# "COPY --from=builder /app/main/ ." No need to do this as we are copy whole app including the executables.
+# we need the whole app as well, because apart from executable our environment variables are stored in a
+# .env file.
 COPY --from=builder /app/ .
-COPY --from=builder /app/main/ .
 
 # Expose port 8080 to the outside world
+# This port should be the same one, as exposed by the app server (goRubu).
 EXPOSE 8080
 
 CMD [ "./main" ]

cache/cacheConnection.go

@@ -0,0 +1,78 @@
+package cache
+
+import (
+	"log"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/bradfitz/gomemcache/memcache"
+	"github.com/joho/godotenv"
+)
+
+// NOTE:
+// Try to establish a connection with Memcached container or Local Memcached
+
+// EXPIRY_TIME - TTL for an item int cache
+var EXPIRY_TIME int
+
+func init() {
+	dir, _ := os.Getwd()
+	envFile := "variables.env"
+	if strings.Contains(dir, "test") {
+		envFile = "../variables.env"
+	}
+
+	if err := godotenv.Load(envFile); err != nil {
+		log.Fatal("Error: No Environment File Found, cacheConnection.go", err)
+	}
+
+	// in seconds
+	EXPIRY_TIME, _ = strconv.Atoi(os.Getenv("EXPIRATION_TIME"))
+}
+
+func tryMemcached(domain string) *memcache.Client {
+	mc := memcache.New(domain)
+
+	inputUrl := "https://stackoverflow.com/questions/58442596/golang-base64-to-hex-conversion"
+	newUrl := "https://goRubu/MTAyNDE="
+
+	err := mc.Set(&memcache.Item{
+		Key:        newUrl,
+		Value:      []byte(inputUrl),
+		Expiration: int32(EXPIRY_TIME),
+	})
+
+	if err != nil {
+		log.Printf("Err: %v, Domain: %v", err, domain)
+		return nil
+	}
+
+	return mc
+}
+
+// CreateCon - Create Memcached Connection
+// as this is called by mainService init function only once, hence checking whether local/docker memcache
+// is up will happen only once.
+func CreateCon() *memcache.Client {
+	var cacheDomain = os.Getenv("MEMCACHED_DOMAIN_DOCKER")
+	var client *memcache.Client
+
+	client = tryMemcached(cacheDomain)
+
+	if client == nil {
+		log.Println("Connection Failed while trying to connect with Memcached Container")
+
+		cacheDomain = os.Getenv("MEMCACHED_DOMAIN_LOCALHOST")
+		client = tryMemcached(cacheDomain)
+		if client == nil {
+			log.Fatal("Connection Failed while Trying to connect with Local Memcached")
+		}
+
+		log.Println("Connected to Local Memcached!!")
+	} else {
+		log.Println("Connected to Memcached Container!!")
+	}
+
+	return client
+}

commands_benchmarks.md

@@ -8,33 +8,40 @@ get https://goRubu/MTAyMTk=
 this will give the original url, to corresponding to the shorten url.
 If you get nothing, it means there is not entry in memcached.
 
-2.  To run tests.
+2. To check Indexed in Mongo.
+```bash
+db.collection.getIndexes()
+```
+
+3. To run tests.
 ```bash
 go test ./tests -v
 go test ./tests -v - cover
 ```
 
 **NOTE** - Normally these two commands work, but with "go 1.13" they are not working.
 Hence use these
+```bash
 go test ./... -v -coverpkg=./... -coverprofile=cover.txt
 go tool cover -html=cover.txt -o cover.html
+```
 
 ## Docker 
 
-1. ```bash
+1. This will give the logs of that container.
+```bash
 docker logs container_id
 ```
-This will give the logs of that container
 
-2. ```bash
+2. Get into the container and see what is happening.
+```bash
 docker exec -it container_id /bin/sh
 ```
-Get into the container and see what is happening.
 
-3. ```bash
+3. Build an image with some name("gorubuimage") from dockerfile
+```bash
 docker build -t gorubuimage .
 ```
-Build an image with some name("gorubuimage") from dockerfile
 
 ## Benchmarks
 

daos/mainDao.go

@@ -34,7 +34,7 @@ func init() {
 	envFile := "variables.env"
 	if strings.Contains(dir, "test") {
 		envFile = "../variables.env"
-		// TODO Remove .. , it is a security threat if done from root
+		// TODO: Remove .. , it is a security threat if done from root
 	}
 
 	if err := godotenv.Load(envFile); err != nil {
@@ -112,10 +112,10 @@ func CleanDb(uid int) {
 		log.Fatal("Error while deleting a doc", err)
 	}
 	// if you directly do string(deleteResult.DeletedCount) for deleteResult.DeletedCount = 67. you will get C. Hence use strcov
-	log.Println("**Deleted " + strconv.FormatInt(deleteResult.DeletedCount, 10) + " documents ")
+	log.Println("** Deleted " + strconv.FormatInt(deleteResult.DeletedCount, 10) + " documents ")
 }
 
-// GetCounterValue - update counter field in second collections - incrementer
+// GetCounterValue - update counter field in second collection - incrementer
 // { "_id" : ObjectId("5e9b7c0e7b3a8740a2f828c4"), "uniqueid" : "counter", "value" : 10000 }
 func GetCounterValue() int {
 	// as there will be one row only

database/dbConnection.go

@@ -18,10 +18,8 @@ func init() {
 	// go test was unable to find it.
 
 	// when doing "go test ./tests -v", I am getting "pwd" as "/Users/home/goRubu/tests"
-	// when doing make execute or go run main.go, I am getting "pwd" as
-	// "Users/home/goRubu"
+	// when doing make execute or go run main.go, I am getting "pwd" as "Users/home/goRubu"
 	dir, _ := os.Getwd()
-	log.Print(dir)
 	envFile := "variables.env"
 	if strings.Contains(dir, "test") {
 		envFile = "../variables.env"
@@ -42,7 +40,7 @@ func tryMongo(dbDomain string) *mongo.Client {
 	err = client.Ping(context.TODO(), nil)
 
 	if err != nil {
-		log.Printf("Dbdomain %v", dbDomain)
+		log.Printf("Err:%v, Dbdomain:%v", dbDomain, err)
 		return nil
 	}
 
@@ -51,6 +49,8 @@ func tryMongo(dbDomain string) *mongo.Client {
 
 // CreateCon - create db connection
 // support both mongo - one on localhost and other on docker
+// as this function is called in dao init function, hence it will be only ran once.
+// Advantages - we will try to establish connection with docker/local only once, not again again.
 func CreateCon() *mongo.Client {
 	var dbDomain = os.Getenv("DB_DOMAIN_DOCKER")
 	var client *mongo.Client
@@ -66,7 +66,7 @@ func CreateCon() *mongo.Client {
 			log.Fatal("Connection to both Mongo Container and Local Mongo Failed")
 		}
 
-		log.Println("Connected to Local Mongo !")
+		log.Println("Connected to Local Mongo!")
 	} else {
 		log.Println("Connected to Mongo Container!")
 	}

handlers/route.go

@@ -106,14 +106,15 @@ func New() http.Handler {
 
 	hist := HistPrometheus{}
 	hist.Populate()
-	// prometheusMonitoring and CheckApiKey will be applied to all middlewares prefixed by all
+	// prometheusMonitoring ,CheckApiKey, Logger will be applied to all middlewares prefixed by all
 	mainRoute.Use(hist.PrometheusMonitoring)
 	mainRoute.Use(middlewares.CheckApiKey)
 	mainRoute.Use(middlewares.Logger)
 
-	// WILL expose default metrics for go application
-	// also as we won't be passing "url" in body, so to prevent missing key from
-	// being returned from checkApiKey middleware, we wont prefix this with all
+	// WILL expose default metrics, along with our custom metrics for go application
+	// NOTE: Prometheus follows a Pull based Mechanism instead of Push Based.
+	// Monitored applications exposes an HTTP endpoint exposing monitoring metrics.
+	// Prometheus then periodically download the metrics.
 	route.Handle("/metrics", promhttp.Handler()).Methods("GET")
 
 	mainRoute.HandleFunc("/check", Hellohandler)

loadtesting/benchmarking.md

@@ -0,0 +1,25 @@
+# BenchMarking Shortened Url
+
+## Setup and Working
+
+You need to have [k6](https://k6.io/) installed. It's written in Go and scriptable in JS.
+
+1. K6 works with the concept of Virtual Users (VU)
+2. Each VU executes your script in a completely separate JS runtime, parallel to all of the other running VU. Code inside the default function is called VU code, and is run over and over, for as long as the test is running.
+3. Every virtual user (VU) performs the GET requests, in a continuous loop, as fast as it can.
+
+
+### Benchmarking
+
+*** For Shorten Url Endpoint ***
+1. For 1VU, in 30s.
+```bash
+k6 run -d 5s -u 1 ./load_test_shorten_url.js
+
+http_req_duration..........: avg=5ms     min=1.66ms med=3.15ms max=222.44ms p(90)=8.7ms   p(95)=12.5ms
+http_reqs..................: 934     186.794732/s
+```
+
+> Conclusion
+95% of our users got served a response in under 12.5ms.
+In the 30 second test duration we served 934 responses, at a rate of ~187 requests per second (RPS).

loadtesting/load_test_shorten_url.js

@@ -0,0 +1,27 @@
+import { check } from "k6";
+import http from "k6/http";
+import { Rate } from 'k6/metrics';
+
+
+export let errorRate = new Rate('errors');
+
+export default function() {
+    var url = "http://localhost:8080/all/shorten_url";
+    var params = {
+        headers: {
+            'Content-Type': 'application/json'
+          }
+    };
+
+    var data = JSON.stringify({
+        "Url": "https://stackoverflow.com/questions/58442596/golang-base64-to-hex-conversion"
+    });
+
+    check(http.post(url, data, params), {
+        'status is 20': r => r.status == 200
+      }) || errorRate.add(1);
+
+//   check(res, {
+//     "is status 200": (r) => r.status === 200
+//   });
+}

readme.md

@@ -52,6 +52,8 @@ Wanted to Learn Go and system design by building a project. Hence goRubu.
 ```bash
 make docker
 ```
+> Note - I haven't dockerized prometheus and grafana with it. Containerized goRubu and locally installed
+> prometheus and grafana will work fine as prometheus is only listening to "local:8080/metrics".
 
 Check the Api's Section afterwards.
 

services/mainService.go

@@ -4,6 +4,7 @@ package services
 import (
 	"context"
 	"encoding/base64"
+	cacheConnection "goRubu/cache"
 	dao "goRubu/daos"
 	model "goRubu/models"
 	"log"
@@ -35,12 +36,8 @@ func init() {
 		log.Fatal("Unable to load env file from urlCreationService Init", err)
 	}
 
-	// in seconds
-	// EXPIRY_TIME is cache expiry time and db expiry time as well
-
-	EXPIRY_TIME, _ = strconv.Atoi(os.Getenv("EXPIRATION_TIME"))
-	mc = memcache.New(os.Getenv("MEMCACHED_DOMAIN_DOCKER"))
-
+	// memcached connection
+	mc = cacheConnection.CreateCon()
 }
 
 // CreateShortenedUrl - This service shortens a give url.
@@ -58,22 +55,10 @@ func CreateShortenedUrl(inputUrl string) string {
 	})
 
 	if err != nil {
-		log.Println("Error in setting memcached value Using Memcached Container", err)
-		log.Println("Switching to Local Memcached")
-		mc = memcache.New(os.Getenv("MEMCACHED_DOMAIN_LOCALHOST"))
-
-		err = mc.Set(&memcache.Item{
-			Key:        newUrl,
-			Value:      []byte(inputUrl),
-			Expiration: int32(EXPIRY_TIME),
-		})
-
-		if err != nil {
-			log.Fatal("Error in setting memcached value using both local and containerized Memcache")
-			log.Fatal(err)
-		}
+		log.Printf("Error in setting memcached value:%v", err)
 	}
 
+	// FIXME:
 	// Race Condition - Undesirable condition where o/p of a program depends on the seq of execution of go routines
 
 	// To prevent this use Mutex - a locking mechanism, to ensure only one Go routine
@@ -84,7 +69,6 @@ func CreateShortenedUrl(inputUrl string) string {
 	dao.InsertInShortenedUrl(inputModel)
 	dao.UpdateCounter()
 	return newUrl
-
 }
 
 //UrlRedirection - will return back the original url from which the inputUrl was created

tests/main_test.go

@@ -51,13 +51,13 @@ func TestUrlcreation(t *testing.T) {
 }
 
 // check whether db cleaning service is working correctly or not.
-func TestDbpurging(t *testing.T) {
-	testUrl := "https://google.com"
-	outputUrl := commonUtility(testUrl, "yup")
-
-	if outputUrl != "" {
-		t.Errorf("Test DbPurging failed. Expected %s, got %s", "", outputUrl)
-	} else {
-		t.Logf("Success, Expected %s, got %s", "", outputUrl)
-	}
-}
+// func TestDbpurging(t *testing.T) {
+// 	testUrl := "https://google.com"
+// 	outputUrl := commonUtility(testUrl, "yup")
+
+// 	if outputUrl != "" {
+// 		t.Errorf("Test DbPurging failed. Expected %s, got %s", "", outputUrl)
+// 	} else {
+// 		t.Logf("Success, Expected %s, got %s", "", outputUrl)
+// 	}
+// }

variables.env

@@ -12,7 +12,7 @@ COLLECTION2_NAME=incrementer
 # the time after which url entries will be removed from the db and cache. 
 # an entry should expire from db and cache at the same time.
 # you can increase it if you want your urls to stay in db for a longer period
-EXPIRATION_TIME=300
+EXPIRATION_TIME=3600
 
 MEMCACHED_DOMAIN_LOCALHOST=127.0.0.1:11211
 MEMCACHED_DOMAIN_DOCKER=cache:11211