Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add basic redaction via new vendor extension 'x-data-redaction' #316

Merged
merged 1 commit into from
Jun 5, 2019
Merged

Add basic redaction via new vendor extension 'x-data-redaction' #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ val exampleCases: List[(java.io.File, String, Boolean, List[String])] = List(
(sampleResource("polymorphism.yaml"), "polymorphism", false, List.empty),
(sampleResource("polymorphism-mapped.yaml"), "polymorphismMapped", false, List.empty),
(sampleResource("raw-response.yaml"), "raw", false, List.empty),
(sampleResource("redaction.yaml"), "redaction", false, List.empty),
(sampleResource("server1.yaml"), "tracer", true, List.empty),
(sampleResource("server2.yaml"), "tracer", true, List.empty),
(sampleResource("pathological-parameters.yaml"), "pathological", false, List.empty)
Expand Down
5 changes: 5 additions & 0 deletions modules/codegen/src/main/scala/com/twilio/guardrail/ProtocolGenerator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,16 @@ sealed trait EmptyToNullBehaviour
case object EmptyIsNull extends EmptyToNullBehaviour
case object EmptyIsEmpty extends EmptyToNullBehaviour

sealed trait RedactionBehaviour
case object DataVisible extends RedactionBehaviour
case object DataRedacted extends RedactionBehaviour

case class ProtocolParameter[L <: LA](term: L#MethodParameter,
name: String,
dep: Option[L#TermName],
readOnlyKey: Option[String],
emptyToNull: EmptyToNullBehaviour,
dataRedaction: RedactionBehaviour,
defaultValue: Option[L#Term])

case class Discriminator[L <: LA](propertyName: String, mapping: Map[String, ProtocolElems[L]])
Expand Down
7 changes: 6 additions & 1 deletion modules/codegen/src/main/scala/com/twilio/guardrail/extract/Extractable.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.twilio.guardrail.extract

import com.twilio.guardrail.{ EmptyIsEmpty, EmptyIsNull, EmptyToNullBehaviour }
import com.twilio.guardrail.{ DataRedacted, DataVisible, EmptyIsEmpty, EmptyIsNull, EmptyToNullBehaviour, RedactionBehaviour }
import scala.util.{ Success, Try }
import scala.reflect.ClassTag
import scala.collection.JavaConverters._
Expand Down Expand Up @@ -43,6 +43,11 @@ object Extractable {
case x: Boolean if x => EmptyIsNull
case x: Boolean if !x => EmptyIsEmpty
})
implicit val defaultExtractableRedactionBehaviour: Extractable[RedactionBehaviour] =
build[RedactionBehaviour]({
case x: Boolean if x => DataRedacted
case x: Boolean if !x => DataVisible
})

implicit def defaultExtractableList[T: Extractable: ClassTag]: Extractable[List[T]] =
buildExceptionally[List[T]](validateSeq.andThen(xs => Try(validateListItems[T].apply(xs))))
Expand Down
3 changes: 3 additions & 0 deletions modules/codegen/src/main/scala/com/twilio/guardrail/extract/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,7 @@ package object extract {

def SecurityOptional[F: VendorExtension.VendorExtensible](v: F): List[String] =
VendorExtension(v).extract[List[String]]("x-security-optional").toList.flatten

def DataRedaction[F: VendorExtension.VendorExtensible](v: F): Option[RedactionBehaviour] =
VendorExtension(v).extract[RedactionBehaviour]("x-data-redaction")
}
34 changes: 28 additions & 6 deletions modules/codegen/src/main/scala/com/twilio/guardrail/generators/CirceProtocolGenerator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ package com.twilio.guardrail
package generators

import _root_.io.swagger.v3.oas.models.media._
import cats.data.NonEmptyList
import cats.implicits._
import cats.~>
import com.twilio.guardrail.extract.{ Default, EmptyValueIsNull }
import com.twilio.guardrail.extract.{ DataRedaction, Default, EmptyValueIsNull }
import com.twilio.guardrail.generators.syntax.RichString
import com.twilio.guardrail.languages.ScalaLanguage
import com.twilio.guardrail.protocol.terms.protocol._
Expand Down Expand Up @@ -141,6 +142,7 @@ object CirceProtocolGenerator {
case s: StringSchema => EmptyValueIsNull(s)
case _ => None
}).getOrElse(EmptyIsEmpty)
dataRedaction = DataRedaction(property).getOrElse(DataVisible)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this feature will eventually need to be controllable via core interpreter, defining the default based on call site (possibly plugin configuration) such that for secure environments everything should be redacted by default

Copy link
Member Author

@kelnos kelnos Jun 5, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that makes sense. Do you think we'd want to continue parse it out here, and then just give the core interp a chance to override it?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah -- I'm thinking a single Term that just Target.pure(DataVisible) defining the default. Eventually, it may consume the config parameters and use plugin configuration to differentiate, but I haven't thought that far ahead, mainly just interested in separating static defaults.


(tpe, classDep) = meta match {
case SwaggerUtil.Resolved(declType, classDep, _) =>
Expand All @@ -164,21 +166,41 @@ object CirceProtocolGenerator {
)(Function.const((tpe, defaultValue)) _)
term = param"${Term.Name(argName)}: ${finalDeclType}".copy(default = finalDefaultValue)
dep = classDep.filterNot(_.value == clsName) // Filter out our own class name
} yield ProtocolParameter[ScalaLanguage](term, name, dep, readOnlyKey, emptyToNull, finalDefaultValue)
} yield ProtocolParameter[ScalaLanguage](term, name, dep, readOnlyKey, emptyToNull, dataRedaction, finalDefaultValue)

case RenderDTOClass(clsName, selfParams, parents) =>
val discriminators = parents.flatMap(_.discriminators)
val discriminatorNames = discriminators.map(_.propertyName).toSet
val parentOpt = if (parents.exists(s => s.discriminators.nonEmpty)) { parents.headOption } else { None }
val terms = (parents.reverse.flatMap(_.params.map(_.term)) ++ selfParams.map(_.term)).filterNot(
param => discriminatorNames.contains(param.name.value)
val params = (parents.reverse.flatMap(_.params) ++ selfParams).filterNot(
param => discriminatorNames.contains(param.term.name.value)
)
val terms = params.map(_.term)

val toStringMethod = if (params.exists(_.dataRedaction != DataVisible)) {
def mkToStringTerm(param: ProtocolParameter[ScalaLanguage]): Term = param match {
case param if param.dataRedaction == DataVisible => Term.Name(param.term.name.value)
case _ => Lit.String("[redacted]")
}

val toStringTerms = NonEmptyList
.fromList(params)
.fold(List.empty[Term])(list => mkToStringTerm(list.head) +: list.tail.map(param => q"${Lit.String(",")} + ${mkToStringTerm(param)}"))

List[Defn.Def](
q"override def toString: String = ${toStringTerms.foldLeft[Term](Lit.String(s"${clsName}("))(
(accum, term) => q"$accum + $term"
)} + ${Lit.String(")")}"
)
} else {
List.empty[Defn.Def]
}

val code = parentOpt
.fold(q"""case class ${Type.Name(clsName)}(..${terms})""")(
.fold(q"""case class ${Type.Name(clsName)}(..${terms}) { ..$toStringMethod }""")(
parent =>
q"""case class ${Type.Name(clsName)}(..${terms}) extends ${template"..${init"${Type.Name(parent.clsName)}(...$Nil)" :: parent.interfaces
.map(a => init"${Type.Name(a)}(...$Nil)")}"}"""
.map(a => init"${Type.Name(a)}(...$Nil)")} { ..$toStringMethod }"}"""
)

Target.pure(code)
Expand Down
41 changes: 26 additions & 15 deletions modules/codegen/src/main/scala/com/twilio/guardrail/generators/Java/JacksonGenerator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import cats.implicits._
import cats.~>
import com.github.javaparser.ast.`type`.{ PrimitiveType, Type }
import com.twilio.guardrail.Discriminator
import com.twilio.guardrail.extract.{ Default, EmptyValueIsNull }
import com.twilio.guardrail.extract.{ DataRedaction, Default, EmptyValueIsNull }
import com.twilio.guardrail.generators.syntax.Java._
import com.twilio.guardrail.generators.syntax.RichString
import com.twilio.guardrail.languages.JavaLanguage
Expand All @@ -27,7 +27,12 @@ import scala.util.Try
object JacksonGenerator {
private val BUILDER_TYPE = JavaParser.parseClassOrInterfaceType("Builder")

private case class ParameterTerm(propertyName: String, parameterName: String, fieldType: Type, parameterType: Type, defaultValue: Option[Expression])
private case class ParameterTerm(propertyName: String,
parameterName: String,
fieldType: Type,
parameterType: Type,
defaultValue: Option[Expression],
dataRedacted: RedactionBehaviour)

// returns a tuple of (requiredTerms, optionalTerms)
// note that required terms _that have a default value_ are conceptually optional.
Expand All @@ -39,15 +44,15 @@ object JacksonGenerator {

params
.map({
case ProtocolParameter(term, name, _, _, _, selfDefaultValue) =>
case ProtocolParameter(term, name, _, _, _, dataRedaction, selfDefaultValue) =>
val parameterType = if (term.getType.isOptional) {
term.getType.containedType.unbox
} else {
term.getType.unbox
}
val defaultValue = defaultValueToExpression(selfDefaultValue)

ParameterTerm(name, term.getNameAsString, term.getType.unbox, parameterType, defaultValue)
ParameterTerm(name, term.getNameAsString, term.getType.unbox, parameterType, defaultValue, dataRedaction)
})
.partition(
pt => !pt.fieldType.isOptional && pt.defaultValue.isEmpty
Expand Down Expand Up @@ -323,7 +328,7 @@ object JacksonGenerator {
terms.filterNot(term => discriminatorNames.contains(term.propertyName))

terms.foreach({
case ParameterTerm(propertyName, parameterName, fieldType, _, _) =>
case ParameterTerm(propertyName, parameterName, fieldType, _, _, _) =>
val field: FieldDeclaration = dtoClass.addField(fieldType, parameterName, PRIVATE, FINAL)
field.addSingleMemberAnnotation("JsonProperty", new StringLiteralExpr(propertyName))
})
Expand All @@ -333,7 +338,7 @@ object JacksonGenerator {
primaryConstructor.setParameters(
new NodeList(
withoutDiscriminators(parentTerms ++ terms).map({
case ParameterTerm(propertyName, parameterName, fieldType, _, _) =>
case ParameterTerm(propertyName, parameterName, fieldType, _, _, _) =>
new Parameter(util.EnumSet.of(FINAL), fieldType, new SimpleName(parameterName))
.addAnnotation(new SingleMemberAnnotationExpr(new Name("JsonProperty"), new StringLiteralExpr(propertyName)))
}): _*
Expand Down Expand Up @@ -363,17 +368,22 @@ object JacksonGenerator {
scope.fold(new MethodCallExpr(methodName))(s => new MethodCallExpr(new NameExpr(s), methodName))
}

def parameterToStringExpr(term: ParameterTerm, scope: Option[String] = None): Expression = term.dataRedacted match {
case DataVisible => parameterGetterCall(term, scope)
case DataRedacted => new StringLiteralExpr("[redacted]")
}

val toStringFieldExprs = NonEmptyList
.fromList(parentTerms ++ terms)
.toList
.flatMap(
l =>
(new StringLiteralExpr(s"${l.head.parameterName}="), parameterGetterCall(l.head)) +:
(new StringLiteralExpr(s"${l.head.parameterName}="), parameterToStringExpr(l.head)) +:
l.tail.map(
term =>
(
new StringLiteralExpr(s", ${term.parameterName}="),
parameterGetterCall(term)
parameterToStringExpr(term)
)
)
)
Expand Down Expand Up @@ -485,11 +495,11 @@ object JacksonGenerator {
val builderClass = new ClassOrInterfaceDeclaration(util.EnumSet.of(PUBLIC, STATIC), false, "Builder")

withoutDiscriminators(parentRequiredTerms ++ requiredTerms).foreach({
case ParameterTerm(_, parameterName, fieldType, _, _) =>
case ParameterTerm(_, parameterName, fieldType, _, _, _) =>
builderClass.addField(fieldType, parameterName, PRIVATE, FINAL)
})
withoutDiscriminators(parentOptionalTerms ++ optionalTerms).foreach({
case ParameterTerm(_, parameterName, fieldType, _, defaultValue) =>
case ParameterTerm(_, parameterName, fieldType, _, defaultValue, _) =>
val initializer = defaultValue.fold[Expression](
new MethodCallExpr(new NameExpr("Optional"), "empty")
)(
Expand All @@ -507,7 +517,7 @@ object JacksonGenerator {
builderConstructor.setParameters(
new NodeList(
withoutDiscriminators(parentRequiredTerms ++ requiredTerms).map({
case ParameterTerm(_, parameterName, _, parameterType, _) =>
case ParameterTerm(_, parameterName, _, parameterType, _, _) =>
new Parameter(util.EnumSet.of(FINAL), parameterType, new SimpleName(parameterName))
}): _*
)
Expand All @@ -516,7 +526,7 @@ object JacksonGenerator {
new BlockStmt(
new NodeList(
withoutDiscriminators(parentRequiredTerms ++ requiredTerms).map({
case ParameterTerm(_, parameterName, fieldType, _, _) =>
case ParameterTerm(_, parameterName, fieldType, _, _, _) =>
new ExpressionStmt(
new AssignExpr(
new FieldAccessExpr(new ThisExpr, parameterName),
Expand All @@ -539,7 +549,7 @@ object JacksonGenerator {
new BlockStmt(
withoutDiscriminators(parentTerms ++ terms)
.map({
case term @ ParameterTerm(_, parameterName, _, _, _) =>
case term @ ParameterTerm(_, parameterName, _, _, _, _) =>
new ExpressionStmt(
new AssignExpr(
new FieldAccessExpr(new ThisExpr, parameterName),
Expand All @@ -554,7 +564,7 @@ object JacksonGenerator {

// TODO: leave out with${name}() if readOnlyKey?
withoutDiscriminators(parentOptionalTerms ++ optionalTerms).foreach({
case ParameterTerm(_, parameterName, fieldType, parameterType, _) =>
case ParameterTerm(_, parameterName, fieldType, parameterType, _, _) =>
builderClass
.addMethod(s"with${parameterName.unescapeIdentifier.capitalize}", PUBLIC)
.setType(BUILDER_TYPE)
Expand Down Expand Up @@ -669,6 +679,7 @@ object JacksonGenerator {
case s: StringSchema => EmptyValueIsNull(s)
case _ => None
}).getOrElse(EmptyIsEmpty)
dataRedaction = DataRedaction(property).getOrElse(DataVisible)

tpeClassDep <- meta match {
case SwaggerUtil.Resolved(declType, classDep, _) =>
Expand Down Expand Up @@ -711,7 +722,7 @@ object JacksonGenerator {
(finalDeclType, finalDefaultValue) = _declDefaultPair
term <- safeParseParameter(s"final ${finalDeclType} ${argName.escapeIdentifier}")
dep = classDep.filterNot(_.value == clsName) // Filter out our own class name
} yield ProtocolParameter[JavaLanguage](term, name, dep, readOnlyKey, emptyToNull, defaultValue)
} yield ProtocolParameter[JavaLanguage](term, name, dep, readOnlyKey, emptyToNull, dataRedaction, defaultValue)
}

case RenderDTOClass(clsName, selfParams, parents) =>
Expand Down
11 changes: 11 additions & 0 deletions modules/sample-akkaHttp/src/test/scala/generators/Circe/CirceRedactionTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package generators.Circe

import org.scalatest.{ FreeSpec, Matchers }
import redaction.client.akkaHttp.definitions.Redaction

class CirceRedactionTest extends FreeSpec with Matchers {
"Redacted fields should get replaced with '[redacted]'" in {
val redaction = Redaction("a", "b", Some("c"), Some("d"))
redaction.toString shouldBe "Redaction(a,[redacted],Some(c),[redacted])"
}
}
15 changes: 15 additions & 0 deletions modules/sample-dropwizard/src/test/scala/core/Jackson/JacksonRedactionTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package core.Jackson

import org.scalatest.{FreeSpec, Matchers}
import redaction.client.dropwizard.definitions.Redaction

class JacksonRedactionTest extends FreeSpec with Matchers {
"Redacted fields should get replaced with '[redacted]'" in {
val redaction = new Redaction.Builder("a", "b")
.withVisibleOptional("c")
.withRedactedOptional("d")
.build()

redaction.toString shouldBe "Redaction{visibleRequired=a, redactedRequired=[redacted], visibleOptional=Optional[c], redactedOptional=[redacted]}"
}
}
22 changes: 22 additions & 0 deletions modules/sample/src/main/resources/redaction.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
openapi: 3.0.2
info:
version: 1.0.0
paths: {}
components:
schemas:
Redaction:
type: object
required:
- visible_required
- redacted_required
properties:
visible_required:
type: string
redacted_required:
type: string
x-data-redaction: true
visible_optional:
type: string
redacted_optional:
type: string
x-data-redaction: true