Init

V1

.gitignore

@@ -0,0 +1,17 @@
+.env
+logs
+project/project
+project/target
+target
+tmp
+.history
+dist
+/.idea
+/*.iml
+/out
+/.idea_modules
+/.classpath
+/.project
+/RUNNING_PID
+/.settings
+hk-debug

LICENSE

@@ -0,0 +1,8 @@
+This software is licensed under the Apache 2 license, quoted below.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this project except in compliance with
+the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0.
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
+language governing permissions and limitations under the License.

README.md

@@ -0,0 +1,30 @@
+# DontRefMe
+
+Don't you hate all these companies sending so many ref data to learn more things about you.
+ie:
+1. From where you opened that link
+2. How you're connected to the person who sent you the link
+3. How many times you have clicked the link 
+
+Let's try to avoid that.
+
+## Help us Improve
+
+Please feel free to create a PR if you think your code can make it better.
+
+## Running Locally
+
+Make sure you have Play and sbt installed.  Also, install the [Heroku Toolbelt](https://toolbelt.heroku.com/).
+
+```sh
+$ git clone https://github.com/rishijash/dontrefme.git
+$ cd dontrefme
+$ sbt run
+```
+
+## Documentation
+
+For more information about using Play and Scala on Heroku, see these Dev Center articles:
+
+- [Play and Scala on Heroku](https://devcenter.heroku.com/categories/language-support#scala-and-play)
+

app.json

@@ -0,0 +1,6 @@
+{
+  "name": "Start on Heroku: Scala",
+  "description": "A barebones Scala app (using the Play framework), which can easily be deployed to Heroku.",
+  "image": "heroku/scala",
+  "addons": [ "heroku-postgresql" ]
+}

app/Global.scala

@@ -0,0 +1,10 @@
+import play.api._
+import play.api.mvc._
+
+import play.api.Logger
+import scala.concurrent.Future
+import play.api.libs.concurrent.Execution.Implicits.defaultContext
+
+object Global extends WithFilters() {
+
+}

app/controllers/HomeController.scala

@@ -0,0 +1,32 @@
+package controllers
+
+import engines.UriEngine
+import javax.inject._
+import play.api.Configuration
+import play.api.mvc._
+
+@Singleton
+class HomeController @Inject()(config: Configuration) extends Controller {
+
+  val uriEngine = new UriEngine(config)
+
+  def index() = Action { implicit request: Request[AnyContent] =>
+    val maybeUriObj = uriEngine.getRequestUri[AnyContent](request)
+    if (maybeUriObj.isDefined) {
+      val newUri = uriEngine.removeRefFromUri(maybeUriObj.get)
+      Redirect(newUri)
+    } else {
+      val summary = uriEngine.getSummary()
+      Ok(views.html.index(summary))
+    }
+  }
+
+  def displayNoRefUri() = Action { implicit request: Request[AnyContent] =>
+    val formData = request.body.asFormUrlEncoded
+    val requestUri = formData.get("requestLink").headOption
+    val responseUri = requestUri.map(uriEngine.removeRefFromUri(_))
+    val summary = uriEngine.getSummary().map(s => s.copy(totalCalls = s.totalCalls + 1))
+    Ok(views.html.index(summary, responseUri))
+  }
+
+}

app/datastore/UriDataStore.scala

@@ -0,0 +1,59 @@
+package datastore
+
+import java.io.{File, FileInputStream}
+import java.util
+
+import com.google.auth.oauth2.ServiceAccountCredentials
+import com.google.cloud.firestore.{FirestoreOptions, QueryDocumentSnapshot}
+import javax.inject.Inject
+import org.slf4j.LoggerFactory
+import models.MetricsSummary
+import play.api.Configuration
+
+import scala.collection.JavaConversions._
+import java.util.UUID.randomUUID
+
+
+class UriDataStore @Inject()(config: Configuration) {
+
+  private val log = LoggerFactory.getLogger(this.getClass.getName)
+
+  private val collectionName = "metrics"
+
+  val hostKey = "host"
+  val totalParamsCountKey = "totalParamsCount"
+  val safeParamsCountKey = "safeParamsCount"
+
+
+  val f = new File("conf/dontrefmeKey.json")
+  private val firebaseClient = FirestoreOptions.newBuilder()
+    .setCredentials(ServiceAccountCredentials.fromStream(new FileInputStream("conf/dontrefmeKey.json")))
+    .build().getService
+
+  def createMetrics(host: String, totalParamsCount: Int, safeParamsCount: Int = 0): Boolean = {
+    try {
+      val docData = new util.HashMap[String, Any](Map(hostKey -> host,
+        totalParamsCountKey -> totalParamsCount, safeParamsCountKey -> safeParamsCount))
+      val id = randomUUID().toString
+      firebaseClient.collection(collectionName).document(id).set(docData)
+      true
+    } catch {
+      case e: Exception =>
+        log.error(s"Exception: ${e.getMessage} in creating metrics for host: ${host}, totalParamsCount: ${totalParamsCount}, safeParamsCount: ${safeParamsCount}")
+        false
+    }
+  }
+
+  def getMetricsSummary(): Option[List[QueryDocumentSnapshot]] = {
+    try {
+      val allData = firebaseClient.collection(collectionName).get().get()
+      val allDocuments = allData.getDocuments.toList
+      Some(allDocuments)
+    } catch {
+      case e: Exception =>
+        log.error(s"Exception: ${e.getMessage} in getting metrics.")
+        None
+    }
+  }
+
+}

app/engines/UriEngine.scala

@@ -0,0 +1,122 @@
+package engines
+
+import java.net.{URI, URLDecoder}
+
+import datastore.UriDataStore
+import javax.inject.Inject
+import models.{HostDetails, HostType, MetricsSummary, RemoveParamRes}
+import org.slf4j.LoggerFactory
+import play.api.Configuration
+import play.api.mvc.Request
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+
+class UriEngine @Inject()(config: Configuration) {
+
+  val baseUrl = config.getString("url.base").get
+  val herokuBaseUrl = config.getString("url.herokuBase").get
+  val store = new UriDataStore(config)
+
+  val log = LoggerFactory.getLogger(this.getClass.getName)
+
+  def getRequestUri[T](request: Request[T]): Option[URI] = {
+    val uriObj = getUriObj(request.rawQueryString)
+    val isSelfUriRequest = (uriObj.toString.contains(baseUrl) || uriObj.toString.contains(herokuBaseUrl))
+    if (isSelfUriRequest) None else uriObj
+  }
+
+  def removeRefFromUri(uri: String): String = {
+    removeRefFromUri(new URI(uri))
+  }
+
+  def removeRefFromUri(uriObj: URI): String = {
+    try {
+      val maybeHostTypeDetails = HostType.getHostTypeDetailsFromHostUriOpt(uriObj.getHost)
+      val refRemoverResponse = maybeHostTypeDetails match {
+        case Some(hostTypeDetails) => {
+          val hostDetails = hostTypeDetails._2
+          refRemoverWithRuleEngine(uriObj, hostDetails.safeParams)
+        }
+        case None => refRemoverWithRuleEngine(uriObj, HostType.commonSafeParams)
+      }
+      addMetrics(refRemoverResponse.host, refRemoverResponse.totalParams, refRemoverResponse.filterdParams)
+      refRemoverResponse.newUri
+    } catch {
+      case e: Exception => {
+        log.error(s"Exception in removing ref: ${e}")
+        uriObj.toString
+      }
+    }
+  }
+
+  def getSummary(): Option[MetricsSummary] = {
+    val maybeSummaryData = store.getMetricsSummary()
+    val summary = maybeSummaryData.map(allDocuments => {
+      val totalCalls = allDocuments.size
+      val totalParamsFiltered = allDocuments.map(doc => {
+        val totalParamCount = doc.getData.get(store.totalParamsCountKey).toString
+        val totalSafeCount = doc.getData.get(store.safeParamsCountKey).toString
+        val diff = totalParamCount.toInt - totalSafeCount.toInt
+        diff
+      }).sum
+      MetricsSummary(totalCalls, totalParamsFiltered)
+    })
+    summary
+  }
+
+  private def getUriObj[T](requestUri: String): Option[URI] = {
+    try {
+      var uri = requestUri
+      if (!uri.startsWith("http") && !uri.startsWith("https")) {
+        uri = s"http://${uri}"
+      }
+      val res = new URI(uri)
+      val maybeHost = Option(res.getHost)
+      maybeHost.map(_ => res)
+    } catch {
+      case _: Exception => None
+    }
+  }
+
+  private def addMetrics(host: String, totalParamsCount: Int, safeParamsCount: Int = 0): Future[Unit] = {
+    Future {
+      store.createMetrics(host, totalParamsCount, safeParamsCount)
+    }
+  }
+
+  private def refRemoverWithRuleEngine(uriObj: URI, safeParamsList: List[String]): RemoveParamRes = {
+    val queryParamsMap = getQueryParamsMap(uriObj)
+    if (queryParamsMap.nonEmpty) {
+      val uriWithNoParams = getUriWithNoParam(uriObj)
+      // Add filtered Params
+      val safeParams = queryParamsMap.filter {
+        case (k, _) => safeParamsList.contains(k)
+      }
+      val safeParamsStr = safeParams.map {
+        case (k, v) => s"${k}=${v}"
+      }.mkString("&")
+     val newUri = s"${uriWithNoParams}?${safeParamsStr}"
+      RemoveParamRes(newUri, uriObj.getHost, queryParamsMap.size, safeParams.size)
+    } else {
+      // If no query params, no need to filter anything. Just call the requested URI
+      val newUri = uriObj.toString
+      RemoveParamRes(newUri, uriObj.getHost, queryParamsMap.size, 0)
+    }
+  }
+
+  private def getUriWithNoParam(uriObj: URI): String = {
+    val rawQuery = Option(uriObj.getRawQuery)
+    val uriString = uriObj.toString.replace("?", "")
+    rawQuery.map(uriString.replace(_, "")).getOrElse(uriString)}
+
+  private def getQueryParamsMap(uriObj: URI): Map[String, String] = {
+    val queryParamsString = Option(uriObj.getRawQuery)
+    queryParamsString.map(_.split("&").map(v => {
+      val m = v.split("=", 2).map(s => URLDecoder.decode(s, "UTF-8"))
+      m(0) -> (if(m.size > 1) m(1) else "")
+    }).toMap
+    ).getOrElse(Map.empty[String, String])
+  }
+
+}

app/models/HostDetails.scala

@@ -0,0 +1,6 @@
+package models
+
+case class HostDetails (
+  uri: String,
+  safeParams: List[String]
+)

app/models/HostType.scala

@@ -0,0 +1,25 @@
+package models
+
+object HostType extends Enumeration {
+
+  type HostType = Value
+  val Google, Youtube, Amazon, Yahoo = Value
+
+  val commonSafeParams = List("g", "k", "p", "q", "v")
+
+  val hostMap = Map(
+    Google -> HostDetails("google.com", List("q", "start")),
+    Youtube -> HostDetails("youtube.com", List("search_query", "v")),
+    Amazon -> HostDetails("amazon.com", List("k")),
+    Yahoo -> HostDetails("yahoo.com", List("p"))
+  )
+
+  def withNameOpt(s: String): Option[Value] = values.find(_.toString == s)
+
+  def getHostTypeDetailsFromHostUriOpt(hostUri: String): Option[(HostType.Value, HostDetails)] = {
+    hostMap.find {
+      case (k, v) => hostUri.contains(v.uri)
+    }
+  }
+
+}

app/models/MetricsSummary.scala

@@ -0,0 +1,6 @@
+package models
+
+case class MetricsSummary (
+  totalCalls: Long,
+  totalParamsFilterd: Long
+)

app/models/RemoveParamRes.scala

@@ -0,0 +1,8 @@
+package models
+
+case class RemoveParamRes (
+  newUri: String,
+  host: String,
+  totalParams: Int,
+  filterdParams: Int
+)