Skip to content

Commit

Permalink
Json parse on forms
Browse files Browse the repository at this point in the history
Co-authored-by: James Roper <james@jazzy.id.au>
Co-authored-by: Ignasi Marimon-Clos <ignasi@lightbend.com>
  • Loading branch information
3 people authored and octonato committed Oct 26, 2020
1 parent b71352a commit 5e3ecbf
Show file tree
Hide file tree
Showing 5 changed files with 285 additions and 26 deletions.
130 changes: 110 additions & 20 deletions core/play/src/main/scala/play/api/data/Form.scala
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@

package play.api.data

import scala.language.existentials
import akka.annotation.InternalApi
import org.slf4j.Logger
import org.slf4j.LoggerFactory

import scala.language.existentials
import play.api.data.format._
import play.api.data.validation._
import play.api.http.HttpVerbs
Expand Down Expand Up @@ -40,6 +43,8 @@ import play.api.templates.PlayMagic.translate
*/
case class Form[T](mapping: Mapping[T], data: Map[String, String], errors: Seq[FormError], value: Option[T]) {

private val logger: Logger = LoggerFactory.getLogger(classOf[Form[T]])

/**
* Constraints associated with this form, indexed by field name.
*/
Expand Down Expand Up @@ -75,7 +80,26 @@ case class Form[T](mapping: Mapping[T], data: Map[String, String], errors: Seq[F
* @param data Json data to submit
* @return a copy of this form, filled with the new data
*/
def bind(data: JsValue): Form[T] = bind(FormUtils.fromJson(js = data))
@deprecated(
"Use bind(JsValue, Int) instead to specify the maximum chars that should be consumed by the flattened form representation of the JSON",
"2.8.3"
)
def bind(data: JsValue): Form[T] = {
logger.warn(
s"Binding json field from form with a hardcoded max size of ${Form.FromJsonMaxChars} bytes. This is deprecated. Use bind(JsValue, Int) instead."
)
bind(FormUtils.fromJson(data, Form.FromJsonMaxChars))
}

/**
* Binds data to this form, i.e. handles form submission.
*
* @param data Json data to submit
* @param maxChars The maximum number of chars allowed to be used in the intermediate map representation
* of the JSON. `parse.DefaultMaxTextLength` is recommended to passed for this parameter.
* @return a copy of this form, filled with the new data
*/
def bind(data: JsValue, maxChars: Int): Form[T] = bind(FormUtils.fromJson(data, maxChars))

/**
* Binds request data to this form, i.e. handles form submission.
Expand All @@ -89,14 +113,14 @@ case class Form[T](mapping: Mapping[T], data: Map[String, String], errors: Seq[F
body.asFormUrlEncoded.orElse(body.asMultipartFormData).orElse(body.asJson).getOrElse(body)
case body => body
}
val data = unwrap match {
val data: Map[String, Seq[String]] = unwrap match {
case body: Map[_, _] => body.asInstanceOf[Map[String, Seq[String]]]
case body: MultipartFormData[_] => body.asFormUrlEncoded
case Right(body: MultipartFormData[_]) => body.asFormUrlEncoded
case body: play.api.libs.json.JsValue => FormUtils.fromJson(js = body).mapValues(Seq(_))
case body: play.api.libs.json.JsValue => FormUtils.fromJson(body, Form.FromJsonMaxChars).mapValues(Seq(_)).toMap
case _ => Map.empty
}
val method = request.method.toUpperCase match {
val method: Map[_ <: String, Seq[String]] = request.method.toUpperCase match {
case HttpVerbs.POST | HttpVerbs.PUT | HttpVerbs.PATCH => Map.empty
case _ => request.queryString
}
Expand Down Expand Up @@ -343,6 +367,15 @@ case class Field(
*/
object Form {

/**
* INTERNAL API
*
* Default maximum number of chars allowed to be used in the intermediate map representation of the
* JSON. Defaults to 100k which is the default of parser.maxMemoryBuffer
*/
@InternalApi
val FromJsonMaxChars: Long = 102400

/**
* Creates a new form from a mapping.
*
Expand Down Expand Up @@ -391,21 +424,75 @@ object Form {
}

private[data] object FormUtils {
def fromJson(prefix: String = "", js: JsValue): Map[String, String] = js match {
case JsObject(fields) =>
val prefix2 = Option(prefix).filterNot(_.isEmpty).map(_ + ".").getOrElse("")
fields.iterator
.map { case (key, value) => fromJson(prefix2 + key, value) }
.foldLeft(Map.empty[String, String])(_ ++ _)
case JsArray(values) =>
values.zipWithIndex.iterator
.map { case (value, i) => fromJson(s"$prefix[$i]", value) }
.foldLeft(Map.empty[String, String])(_ ++ _)
case JsNull => Map.empty
case JsUndefined() => Map.empty
case JsBoolean(value) => Map(prefix -> value.toString)
case JsNumber(value) => Map(prefix -> value.toString)
case JsString(value) => Map(prefix -> value.toString)
def fromJson(js: JsValue, maxChars: Long): Map[String, String] = doFromJson(FromJsonRoot(js), Map.empty, 0, maxChars)

@annotation.tailrec
private def doFromJson(
context: FromJsonContext,
form: Map[String, String],
cumulativeChars: Int,
maxChars: Long
): Map[String, String] = context match {
case FromJsonTerm => form
case ctx: FromJsonContextValue =>
// Ensure this contexts next is initialised, this prevents unbounded recursion.
ctx.next
ctx.value match {
case obj: JsObject if obj.fields.nonEmpty =>
doFromJson(FromJsonObject(ctx, obj.fields.toIndexedSeq, 0), form, cumulativeChars, maxChars)
case JsArray(values) if values.nonEmpty =>
doFromJson(FromJsonArray(ctx, values, 0), form, cumulativeChars, maxChars)
case JsNull | JsArray(_) | JsObject(_) =>
doFromJson(ctx.next, form, cumulativeChars, maxChars)
case simple =>
val value = simple match {
case JsString(v) => v
case JsNumber(v) => v.toString
case JsBoolean(v) => v.toString
}
val prefix = ctx.prefix
val newCumulativeChars = cumulativeChars + prefix.length + value.length
if (newCumulativeChars > maxChars) {
throw FormJsonExpansionTooLarge(maxChars)
}
doFromJson(ctx.next, form.updated(prefix, value), newCumulativeChars, maxChars)
}
}

private sealed trait FromJsonContext
private sealed trait FromJsonContextValue extends FromJsonContext {
def value: JsValue
def prefix: String
def next: FromJsonContext
}
private case object FromJsonTerm extends FromJsonContext
private case class FromJsonRoot(value: JsValue) extends FromJsonContextValue {
override def prefix = ""
override def next: FromJsonContext = FromJsonTerm
}
private case class FromJsonArray(parent: FromJsonContextValue, values: scala.collection.IndexedSeq[JsValue], idx: Int)
extends FromJsonContextValue {
override def value: JsValue = values(idx)
override val prefix: String = s"${parent.prefix}[$idx]"
override lazy val next: FromJsonContext = if (idx + 1 < values.length) {
FromJsonArray(parent, values, idx + 1)
} else {
parent.next
}
}
private case class FromJsonObject(parent: FromJsonContextValue, fields: IndexedSeq[(String, JsValue)], idx: Int)
extends FromJsonContextValue {
override def value: JsValue = fields(idx)._2
override val prefix: String = if (parent.prefix.isEmpty) {
fields(idx)._1
} else {
parent.prefix + "." + fields(idx)._1
}
override lazy val next: FromJsonContext = if (idx + 1 < fields.length) {
FromJsonObject(parent, fields, idx + 1)
} else {
parent.next
}
}
}

Expand Down Expand Up @@ -977,3 +1064,6 @@ trait ObjectMapping {
}
}
}

case class FormJsonExpansionTooLarge(limit: Long)
extends RuntimeException(s"Binding form from JSON exceeds form expansion limit of $limit")
123 changes: 123 additions & 0 deletions core/play/src/test/scala/play/api/data/FormUtilsSpec.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* Copyright (C) Lightbend Inc. <https://www.lightbend.com>
*/

package play.api.data

import org.specs2.mutable.Specification
import play.api.libs.json.JsNull
import play.api.libs.json.Json

class FormUtilsSpec extends Specification {

"FormUtils.fromJson" should {
"convert a complex json structure to a map" in {
val json = Json.obj(
"arr" -> Json.arr(
Json.obj(
"a" -> "an-a",
"b" -> true,
"c" -> JsNull,
"d" -> 10
),
"str",
20,
"blah"
),
"e" -> Json.obj(
"f" -> "an-f",
"g" -> false,
),
"h" -> "an-h",
"i" -> 30,
"j" -> Json.arr(
Json.arr(
40
)
)
)

val expected = Seq(
"arr[0].a" -> "an-a",
"arr[0].b" -> "true",
"arr[0].d" -> "10",
"arr[1]" -> "str",
"arr[2]" -> "20",
"arr[3]" -> "blah",
"e.f" -> "an-f",
"e.g" -> "false",
"h" -> "an-h",
"i" -> "30",
"j[0][0]" -> "40",
)

val map = FormUtils.fromJson(json, 1000)
map.toSeq must containTheSameElementsAs(expected)
}

"not stack overflow when converting heavily nested arrays" in {
try {
FormUtils.fromJson(Json.parse("{\"arr\":" + ("[" * 10000) + "1" + ("]" * 10000) + "}"), 1000000)
} catch {
case e: StackOverflowError =>
ko("StackOverflowError thrown")
}
ok
}

"parse normally when the input is small enough" in {
val keyLength = 10
val itemCount = 10
val maxChars = 500 * 1000 // a limit we're not reaching
try {
FormUtils.fromJson(
Json.obj("a" * keyLength -> Json.arr(0 to itemCount)),
maxChars
)
} catch {
case _: OutOfMemoryError =>
ko("OutOfMemoryError")
}
ok
}

"abort parsing when maximum memory is used" in {
// Even when the JSON is small, if the memory limit is exceed the parsing must stop.
val keyLength = 10
val itemCount = 10
val maxChars = 3 // yeah, maxChars is only 3 chars. We want to hit the limit.
(try {
FormUtils.fromJson(
Json.obj("a" * keyLength -> Json.arr(0 to itemCount)),
maxChars
)
} catch {
case _: OutOfMemoryError =>
ko("OutOfMemoryError")
}) must throwA[FormJsonExpansionTooLarge]
}

"not run out of heap when converting arrays with very long keys" in {
// a similar scenario to the previous one but this would cause OOME if it weren't for the limit
val keyLength = 10000
val itemCount = 100000
val maxChars = keyLength // some value we're likely to exceed. We want this limit to kick in.
(try {
FormUtils.fromJson(
// A JSON object with a key of length 10000, pointing to a list with 100000 elements.
// In memory, this will consume at most a few MB of space. When expanded, will consume
// many GB of space.
Json.obj("a" * keyLength -> Json.arr(0 to itemCount)),
maxChars
)
} catch {
case _: OutOfMemoryError =>
// No guarantee we'll be the thread that gets this, or that our handling will be graceful,
// but at least try.
ko("OutOfMemoryError")
}) must throwA[FormJsonExpansionTooLarge]
}

}

}
4 changes: 3 additions & 1 deletion project/BuildSettings.scala
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,9 @@ object BuildSettings {
(organization.value %% moduleName.value % version).cross(cross)
}.toSet,
mimaBinaryIssueFilters ++= Seq(
// Add mima filters here
// Limit JSON parsing resources
ProblemFilters.exclude[DirectMissingMethodProblem]("play.api.data.FormUtils.fromJson$default$1"),
ProblemFilters.exclude[IncompatibleMethTypeProblem]("play.api.data.FormUtils.fromJson"), // is private
),
unmanagedSourceDirectories in Compile += {
val suffix = CrossVersion.partialVersion(scalaVersion.value) match {
Expand Down
12 changes: 11 additions & 1 deletion web/play-java-forms/src/main/java/play/data/DynamicForm.java
Original file line number Diff line number Diff line change
Expand Up @@ -276,13 +276,23 @@ public DynamicForm bindFromRequestData(
}

@Override
@Deprecated
public DynamicForm bind(Lang lang, TypedMap attrs, JsonNode data, String... allowedFields) {
logger.warn(
"Binding json field from form with a hardcoded max size of {} bytes. This is deprecated. Use bind(Lang, TypedMap, JsonNode, Int, String...) instead.",
FROM_JSON_MAX_CHARS);
return bind(lang, attrs, data, FROM_JSON_MAX_CHARS, allowedFields);
}

@Override
public DynamicForm bind(
Lang lang, TypedMap attrs, JsonNode data, long maxChars, String... allowedFields) {
return bind(
lang,
attrs,
play.libs.Scala.asJava(
play.api.data.FormUtils.fromJson(
"", play.api.libs.json.Json.parse(play.libs.Json.stringify(data)))),
play.api.libs.json.Json.parse(play.libs.Json.stringify(data)), maxChars)),
allowedFields);
}

Expand Down
Loading

0 comments on commit 5e3ecbf

Please sign in to comment.