Permalink
Browse files

refactor the file service to another app

  • Loading branch information...
jamesward committed May 22, 2015
1 parent 3871c3d commit 5b7cd4b8fa410cd6e4ef9824a69d28790f42e9bd
@@ -1,24 +1,20 @@
package controllers

import java.io.{BufferedInputStream, FileNotFoundException}
import java.io.FileNotFoundException
import akka.actor.{Actor, ActorRef, Props}
import akka.pattern.ask
import akka.util.Timeout
import models.{WebJarCatalog, WebJar}
import org.joda.time._
import org.joda.time.format.{DateTimeFormat, DateTimeFormatter}
import play.api.Play
import play.api.Play.current
import play.api.cache.Cache
import play.api.libs.MimeTypes
import play.api.libs.concurrent.Akka
import play.api.libs.json.{JsValue, JsObject, JsArray, Json}
import play.api.libs.json.{JsObject, JsArray, Json}
import play.api.data.Forms._
import play.api.data._
import play.api.libs.ws.WS
import play.api.mvc.Results.EmptyContent
import play.api.mvc._
import utils.MavenCentral.UnexpectedResponseException
import utils._

import scala.concurrent.ExecutionContext.Implicits.global
@@ -186,75 +182,28 @@ object Application extends Controller {
}
}
}

def listFiles(groupId: String, artifactId: String, version: String) = CorsAction {
Action.async { implicit request =>
MavenCentral.getFileList(groupId, artifactId, version).map { fileList =>
render {
case Accepts.Html() => Ok(views.html.filelist(groupId, artifactId, version, fileList))
case Accepts.Json() => Ok(Json.toJson(fileList))
}
WebJarsFileService.getFileList(groupId, artifactId, version).map { fileList =>
render {
case Accepts.Html() => Ok(views.html.filelist(groupId, artifactId, version, fileList))
case Accepts.Json() => Ok(Json.toJson(fileList))
}
} recover {
case nf: FileNotFoundException =>
NotFound(s"WebJar Not Found $groupId : $artifactId : $version")
case ure: UnexpectedResponseException =>
Status(ure.response.status)(s"Problems retrieving WebJar ($groupId : $artifactId : $version) - ${ure.response.statusText}")
case e: Exception =>
InternalServerError(e.getMessage)
InternalServerError(s"Problems retrieving WebJar ($groupId : $artifactId : $version) - ${e.getMessage}")
}
}
}

def listFilesBower(artifactId: String, version: String) = CorsAction {
Action.async { implicit request =>
Future.successful(NotImplemented)
}
}

// max 10 requests per minute
lazy val fileRateLimiter = Akka.system.actorOf(Props(classOf[RequestTracker], 10, Period.minutes(1)))

def file(groupId: String, artifactId: String, webJarVersion: String, file: String) = CorsAction {
Action.async { request =>
val pathPrefix = s"META-INF/resources/webjars/$artifactId/"

Future.fromTry {
MavenCentral.getFile(groupId, artifactId, webJarVersion).map { case (jarInputStream, inputStream) =>
Stream.continually(jarInputStream.getNextJarEntry).takeWhile(_ != null).find { jarEntry =>
// this allows for sloppyness where the webJarVersion and path differ
// todo: eventually be more strict but since this has been allowed many WebJars do not have version and path consistency
jarEntry.getName.startsWith(pathPrefix) && jarEntry.getName.endsWith(s"/$file")
}.fold {
jarInputStream.close()
inputStream.close()
NotFound(s"Found WebJar ($groupId : $artifactId : $webJarVersion) but could not find: $pathPrefix$webJarVersion/$file")
} { jarEntry =>
val bis = new BufferedInputStream(jarInputStream)
val bArray = Stream.continually(bis.read).takeWhile(_ != -1).map(_.toByte).toArray
bis.close()
jarInputStream.close()
inputStream.close()

//// From Play's Assets controller
val contentType = MimeTypes.forFileName(file).map(m => m + addCharsetIfNeeded(m)).getOrElse(BINARY)
////

Ok(bArray).as(contentType).withHeaders(
CACHE_CONTROL -> "max-age=290304000, public",
DATE -> df.print((new java.util.Date).getTime),
LAST_MODIFIED -> df.print(jarEntry.getLastModifiedTime.toMillis)
)
}
}
} recover {
case nf: FileNotFoundException =>
NotFound(s"WebJar Not Found $groupId : $artifactId : $webJarVersion")
case ure: UnexpectedResponseException =>
Status(ure.response.status)(s"Problems retrieving WebJar ($groupId : $artifactId : $webJarVersion) - ${ure.response.statusText}")
case e: Exception =>
InternalServerError(s"Could not find WebJar ($groupId : $artifactId : $webJarVersion)\n${e.getMessage}")
}
}

def file(groupId: String, artifactId: String, version: String, file: String) = Action {
MovedPermanently(s"http://webjars-file-service.herokuapp.com/files/$groupId/$artifactId/$version/$file")
}

def fileOptions(file: String) = CorsAction {
@@ -458,25 +407,4 @@ object Application extends Controller {

}

//// From Play's Asset controller

private val timeZoneCode = "GMT"

//Dateformatter is immutable and threadsafe
private val df: DateTimeFormatter =
DateTimeFormat.forPattern("EEE, dd MMM yyyy HH:mm:ss '" + timeZoneCode + "'").withLocale(java.util.Locale.ENGLISH).withZone(DateTimeZone.forID(timeZoneCode))

//Dateformatter is immutable and threadsafe
private val dfp: DateTimeFormatter =
DateTimeFormat.forPattern("EEE, dd MMM yyyy HH:mm:ss").withLocale(java.util.Locale.ENGLISH).withZone(DateTimeZone.forID(timeZoneCode))

private lazy val defaultCharSet = Play.configuration.getString("default.charset").getOrElse("utf-8")

private def addCharsetIfNeeded(mimeType: String): String =
if (MimeTypes.isText(mimeType))
"; charset=" + defaultCharSet
else ""

////

}
@@ -14,7 +14,6 @@ object WebJar {
}

object WebJarVersion {
def cacheKey(groupId: String, artifactId: String, version: String): String = groupId + "-" + artifactId + "-" + version + "-files"

// todo, this doesn't work on date-based versions that follow non-standard formats (e.g. ace)
implicit object WebJarVersionOrdering extends Ordering[WebJarVersion] {
@@ -1,90 +1,35 @@
package utils

import java.io._
import java.net.{URL, URLEncoder}
import java.nio.file.Files
import java.util.jar.JarInputStream
import java.util.zip.{DeflaterOutputStream, InflaterInputStream}

import actors.{FetchWebJars, WebJarFetcher}
import akka.actor._
import akka.pattern.ask
import akka.util.Timeout
import com.ning.http.client.providers.netty.NettyResponse
import models.WebJarCatalog
import models.WebJarCatalog.WebJarCatalog
import models.{WebJarCatalog, WebJar, WebJarVersion}
import org.webjars.WebJarAssetLocator
import play.api.Play.current
import play.api.cache.Cache
import play.api.http.Status
import play.api.libs.concurrent.Akka
import play.api.libs.json.{JsObject, Json}
import play.api.libs.ws.{WS, WSResponse}
import play.api.libs.ws.WS
import play.api.{Logger, Play}
import shade.memcached.Codec
import Memcache._

import scala.concurrent.{ExecutionContext, Future}
import scala.concurrent.duration._
import scala.io.Source
import scala.util.control.NonFatal
import scala.util.{Failure, Success, Try}
import scala.xml.{Elem, XML}
import scala.xml.Elem

object MavenCentral {

implicit val ec: ExecutionContext = Akka.system(Play.current).dispatchers.lookup("mavencentral.dispatcher")

lazy val webJarFetcher: ActorRef = Akka.system.actorOf(Props[WebJarFetcher])

lazy val tempDir: File = {
Files.createTempDirectory("webjars").toFile
}

implicit val webJarVersionReads = Json.reads[WebJarVersion]
implicit val webJarVersionWrites = Json.writes[WebJarVersion]

implicit val webJarReads = Json.reads[WebJar]
implicit val webJarWrites = Json.writes[WebJar]

// from: http://stackoverflow.com/questions/15079332/round-tripping-through-deflater-in-scala-fails

def compress(bytes: Array[Byte]): Array[Byte] = {
val deflater = new java.util.zip.Deflater
val baos = new ByteArrayOutputStream
val dos = new DeflaterOutputStream(baos, deflater)
dos.write(bytes)
dos.finish()
dos.close()
baos.close()
deflater.end()
baos.toByteArray
}

def decompress(bytes: Array[Byte]): Array[Byte] = {
val inflater = new java.util.zip.Inflater()
val bytesIn = new ByteArrayInputStream(bytes)
val in = new InflaterInputStream(bytesIn, inflater)
val out = Source.fromInputStream(in).map(_.toByte).toArray
in.close()
bytesIn.close()
inflater.end()
out
}

implicit object StringsCodec extends Codec[List[String]] {
def serialize(fileList: List[String]): Array[Byte] = compress(Json.toJson(fileList).toString().getBytes)
def deserialize(data: Array[Byte]): List[String] = Json.parse(decompress(data)).as[List[String]]
}

implicit object ElemCode extends Codec[Elem] {
def serialize(elem: Elem): Array[Byte] = compress(elem.toString().getBytes)
def deserialize(data: Array[Byte]): Elem = XML.loadString(new String(decompress(data)))
}

val primaryBaseJarUrl = Play.current.configuration.getString("webjars.jarUrl.primary").get
val fallbackBaseJarUrl = Play.current.configuration.getString("webjars.jarUrl.fallback").get

def fetchWebJarNameAndUrl(groupId: String, artifactId: String, version: String): Future[(String, String)] = {
getPom(groupId, artifactId, version).flatMap { xml =>
val artifactId = (xml \ "artifactId").text
@@ -144,7 +89,7 @@ object MavenCentral {
versions.map { version =>
catalog match {
case WebJarCatalog.CLASSIC =>
MavenCentral.getFileList(catalog.toString, artifactId, version).map { fileList =>
WebJarsFileService.getFileList(catalog.toString, artifactId, version).map { fileList =>
WebJarVersion(version, fileList.length)
}
case WebJarCatalog.BOWER | WebJarCatalog.NPM =>
@@ -217,72 +162,4 @@ object MavenCentral {
}
}

private def fetchFileList(groupId: String, artifactId: String, version: String): Try[List[String]] = {
getFile(groupId, artifactId, version).map { case (jarInputStream, inputStream) =>
val webJarFiles = Stream.continually(jarInputStream.getNextJarEntry).
takeWhile(_ != null).
filterNot(_.isDirectory).
map(_.getName).
filter(_.startsWith(WebJarAssetLocator.WEBJARS_PATH_PREFIX)).
toList
jarInputStream.close()
inputStream.close()
webJarFiles
}
}

def getFileList(groupId: String, artifactId: String, version: String): Future[List[String]] = {
val cacheKey = WebJarVersion.cacheKey(groupId, artifactId, version)
Global.memcached.get[List[String]](cacheKey).flatMap { maybeFileList =>
maybeFileList.map(Future.successful).getOrElse {
val fileListFuture = fetchFileList(groupId, artifactId, version)
fileListFuture.foreach { fileList =>
Global.memcached.set(cacheKey, fileList, Duration.Inf)
}
Future.fromTry(fileListFuture)
}
}
}

def getFile(groupId: String, artifactId: String, version: String): Try[(JarInputStream, InputStream)] = {
val tmpFile = new File(tempDir, s"$groupId-$artifactId-$version.jar")

if (tmpFile.exists()) {
val fileInputStream = Files.newInputStream(tmpFile.toPath)
Success((new JarInputStream(fileInputStream), fileInputStream))
}
else {
val fileInputStreamFuture = getFileInputStream(primaryBaseJarUrl, groupId, artifactId, version).recoverWith {
case _ =>
getFileInputStream(fallbackBaseJarUrl, groupId, artifactId, version)
}

fileInputStreamFuture.map { fileInputStream =>
// todo: not thread safe!
// write to the fs
Files.copy(fileInputStream, tmpFile.toPath)
fileInputStream.close()

val tmpFileInputStream = Files.newInputStream(tmpFile.toPath)
// read it from the fs since we've drained the http response
(new JarInputStream(tmpFileInputStream), tmpFileInputStream)
}
}
}

def getFileInputStream(baseJarUrl: String, groupId: String, artifactId: String, version: String): Try[InputStream] = {
Try {
val url = new URL(baseJarUrl.format(groupId.replace(".", "/"), artifactId, URLEncoder.encode(version, "UTF-8"), artifactId, URLEncoder.encode(version, "UTF-8")))
url.openConnection().getInputStream
}
}

case class NotFoundResponseException(response: WSResponse) extends RuntimeException {
override def getMessage: String = response.statusText
}

case class UnexpectedResponseException(response: WSResponse) extends RuntimeException {
override def getMessage: String = response.statusText
}

}
@@ -0,0 +1,49 @@
package utils

import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
import java.util.zip.{InflaterInputStream, DeflaterOutputStream}

import play.api.libs.json.Json
import shade.memcached.Codec

import scala.io.Source
import scala.xml.{XML, Elem}

object Memcache {

// from: http://stackoverflow.com/questions/15079332/round-tripping-through-deflater-in-scala-fails

def compress(bytes: Array[Byte]): Array[Byte] = {
val deflater = new java.util.zip.Deflater
val baos = new ByteArrayOutputStream
val dos = new DeflaterOutputStream(baos, deflater)
dos.write(bytes)
dos.finish()
dos.close()
baos.close()
deflater.end()
baos.toByteArray
}

def decompress(bytes: Array[Byte]): Array[Byte] = {
val inflater = new java.util.zip.Inflater()
val bytesIn = new ByteArrayInputStream(bytes)
val in = new InflaterInputStream(bytesIn, inflater)
val out = Source.fromInputStream(in).map(_.toByte).toArray
in.close()
bytesIn.close()
inflater.end()
out
}

implicit object StringsCodec extends Codec[List[String]] {
def serialize(fileList: List[String]): Array[Byte] = compress(Json.toJson(fileList).toString().getBytes)
def deserialize(data: Array[Byte]): List[String] = Json.parse(decompress(data)).as[List[String]]
}

implicit object ElemCode extends Codec[Elem] {
def serialize(elem: Elem): Array[Byte] = compress(elem.toString().getBytes)
def deserialize(data: Array[Byte]): Elem = XML.loadString(new String(decompress(data)))
}

}
Oops, something went wrong.

0 comments on commit 5b7cd4b

Please sign in to comment.