RORDEV-10 Field level security: nested fields support

core/build.gradle

@@ -93,6 +93,7 @@ dependencies {
     compile group: 'org.yaml',                              name: 'snakeyaml',                              version: '1.17'
     compile group: 'org.typelevel',                         name: 'squants_2.12',                           version: '1.4.0'
     compile group: 'com.unboundid',                         name: 'unboundid-ldapsdk',                      version: '4.0.9'
+    compile group: 'com.lihaoyi',                           name: 'upickle_2.12',                           version: '0.7.1'
 
     testCompile project(':tests-utils')
     testCompile group: 'org.apache.logging.log4j',          name: 'log4j-core',                             version: '2.11.2'

core/src/main/java/tech/beshu/ror/Constants.java

@@ -34,7 +34,7 @@ public class Constants {
   public final static String CURRENT_USER_METADATA_PATH = "/_readonlyrest/metadata/current_user";
   public final static String FILTER_TRANSIENT = "_filter";
   public final static String FIELDS_TRANSIENT = "_fields";
-  public final static Set<String> FIELDS_ALWAYS_ALLOW = Sets.newHashSet("_id", "_uid", "_type", "_parent", "_routing", "_timestamp", "_ttl", "_size", "_index");
+  public final static Set<String> FIELDS_ALWAYS_ALLOW = Sets.newHashSet("_id", "_uid", "_type", "_version", "_seq_no", "_primary_term", "_parent", "_routing", "_timestamp", "_ttl", "_size", "_index");
 
   public static final String AUDIT_LOG_DEFAULT_INDEX_TEMPLATE = "'readonlyrest_audit-'yyyy-MM-dd";
 

core/src/main/scala/tech/beshu/ror/accesscontrol/blocks/rules/FieldsRule.scala

@@ -16,18 +16,19 @@
  */
 package tech.beshu.ror.accesscontrol.blocks.rules
 
-import cats.implicits._
 import cats.data.NonEmptySet
-import eu.timepit.refined.types.string.NonEmptyString
+import cats.implicits._
 import monix.eval.Task
 import tech.beshu.ror.accesscontrol.blocks.BlockContext
 import tech.beshu.ror.accesscontrol.blocks.rules.FieldsRule.Settings
 import tech.beshu.ror.accesscontrol.blocks.rules.Rule.{RegularRule, RuleResult}
-import tech.beshu.ror.accesscontrol.request.RequestContext
 import tech.beshu.ror.accesscontrol.domain.DocumentField.{ADocumentField, NegatedDocumentField}
 import tech.beshu.ror.accesscontrol.domain.Header.Name
 import tech.beshu.ror.accesscontrol.domain.{DocumentField, Header}
-import tech.beshu.ror.accesscontrol.show.logs._
+import tech.beshu.ror.accesscontrol.headerValues.transientFieldsToHeaderValue
+import tech.beshu.ror.accesscontrol.orders._
+import tech.beshu.ror.accesscontrol.request.RequestContext
+import tech.beshu.ror.utils.ScalaOps._
 
 class FieldsRule(val settings: Settings)
   extends RegularRule {
@@ -42,18 +43,16 @@ class FieldsRule(val settings: Settings)
 
   private val transientFieldsHeader = new Header(
     Name.transientFields,
-    NonEmptyString.unsafeFrom(settings.fields.map(_.show).mkString(","))
+    transientFieldsToHeaderValue.toRawValue(settings.fields)
   )
 }
 
 object FieldsRule {
   val name = Rule.Name("fields")
 
-  final case class Settings private(fields: Set[DocumentField])
+  final case class Settings private(fields: NonEmptySet[DocumentField])
   object Settings {
-    def ofFields(fields: NonEmptySet[ADocumentField], notAll: Option[NegatedDocumentField]): Settings =
-      Settings(fields.toSortedSet ++ notAll.toSet)
-    def ofNegatedFields(fields: NonEmptySet[NegatedDocumentField], notAll: Option[NegatedDocumentField]): Settings =
-      Settings(fields.toSortedSet ++ notAll.toSet)
+    def ofFields(fields: NonEmptySet[ADocumentField]): Settings = Settings(fields.widen[DocumentField])
+    def ofNegatedFields(fields: NonEmptySet[NegatedDocumentField]): Settings = Settings(fields.widen[DocumentField])
   }
 }

core/src/main/scala/tech/beshu/ror/accesscontrol/domain.scala

@@ -203,13 +203,10 @@ object domain {
 
   final case class UserOrigin(value: NonEmptyString)
 
-  sealed abstract class DocumentField(val value: String)
+  sealed abstract class DocumentField(val value: NonEmptyString)
   object DocumentField {
-    final case class ADocumentField(override val value: String) extends DocumentField(value)
-    final case class NegatedDocumentField(override val value: String) extends DocumentField(value)
-
-    val all = ADocumentField("_all")
-    val notAll = NegatedDocumentField("_all")
+    final case class ADocumentField(override val value: NonEmptyString) extends DocumentField(value)
+    final case class NegatedDocumentField(override val value: NonEmptyString) extends DocumentField(value)
   }
 
   final case class Type(value: String) extends AnyVal

core/src/main/scala/tech/beshu/ror/accesscontrol/factory/decoders/rules/FieldsRuleDecoder.scala

@@ -18,6 +18,7 @@ package tech.beshu.ror.accesscontrol.factory.decoders.rules
 
 import cats.data.NonEmptySet
 import cats.implicits._
+import eu.timepit.refined.types.string.NonEmptyString
 import tech.beshu.ror.accesscontrol.blocks.rules.FieldsRule
 import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.AclCreationError.Reason.Message
 import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.AclCreationError.RulesLevelCreationError
@@ -34,7 +35,7 @@ import scala.collection.SortedSet
 
 object FieldsRuleDecoder extends RuleDecoderWithoutAssociatedFields(
   DecoderHelpers
-    .decodeStringLikeOrNonEmptySet(toDocumentField)
+    .decodeStringLikeOrNonEmptySetE(toDocumentField)
     .toSyncDecoder
     .emapE { fields =>
       val (negatedFields, nonNegatedFields) = fields.toList.partitionEither {
@@ -46,15 +47,14 @@ object FieldsRuleDecoder extends RuleDecoderWithoutAssociatedFields(
       } else if (containsAlwaysAllowedFields(fields)) {
         Left(RulesLevelCreationError(Message(s"The fields rule cannot contain always-allowed fields: ${Constants.FIELDS_ALWAYS_ALLOW.asScala.mkString(",")}")))
       } else {
-        val notAllField = if (fields.exists(_ == DocumentField.all)) None else Some(DocumentField.notAll)
         val settings = NonEmptySet.fromSet(SortedSet.empty[ADocumentField] ++ nonNegatedFields.toSet) match {
           case Some(f) =>
-            FieldsRule.Settings.ofFields(f, notAllField)
+            FieldsRule.Settings.ofFields(f)
           case None =>
             val negatedFieldsNes = NonEmptySet
               .fromSet(SortedSet.empty[NegatedDocumentField] ++ negatedFields.toSet)
                 .getOrElse(throw new IllegalStateException("Should contain all negated fields"))
-            FieldsRule.Settings.ofNegatedFields(negatedFieldsNes, notAllField)
+            FieldsRule.Settings.ofNegatedFields(negatedFieldsNes)
         }
         Right(settings)
       }
@@ -64,12 +64,21 @@ object FieldsRuleDecoder extends RuleDecoderWithoutAssociatedFields(
 )
 
 private object FieldsRuleDecoderHelper {
-  def toDocumentField(value: String): DocumentField = {
-    if (value.startsWith("~")) NegatedDocumentField(value.substring(1))
-    else ADocumentField(value)
+  def toDocumentField(value: String): Either[String, DocumentField] = {
+    if (value.startsWith("~")) {
+      NonEmptyString.from(value.substring(1)) match {
+        case Right(nes) => Right(NegatedDocumentField(nes))
+        case Left(_) => Left("There was no name passed for blacklist field (~ only is forbidden)")
+      }
+    } else {
+      NonEmptyString.from(value) match {
+        case Right(nes) => Right(ADocumentField(nes))
+        case Left(_) => Left("Field cannot be empty string")
+      }
+    }
   }
 
   def containsAlwaysAllowedFields(fields: NonEmptySet[DocumentField]): Boolean = {
-    fields.toSortedSet.map(_.value).intersect(Constants.FIELDS_ALWAYS_ALLOW.asScala).nonEmpty
+    fields.toSortedSet.map(_.value).intersect(Constants.FIELDS_ALWAYS_ALLOW.asScala.map(NonEmptyString.unsafeFrom)).nonEmpty
   }
 }

core/src/main/scala/tech/beshu/ror/accesscontrol/ops.scala

@@ -16,9 +16,10 @@
  */
 package tech.beshu.ror.accesscontrol
 
+import java.util.Base64
 import java.util.regex.Pattern
 
-import cats.data.NonEmptyList
+import cats.data.{NonEmptyList, NonEmptySet}
 import cats.implicits._
 import cats.{Order, Show}
 import com.softwaremill.sttp.{Method, Uri}
@@ -39,14 +40,17 @@ import tech.beshu.ror.accesscontrol.blocks.{Block, BlockContext, RuleOrdering, U
 import tech.beshu.ror.accesscontrol.domain.DocumentField.{ADocumentField, NegatedDocumentField}
 import tech.beshu.ror.accesscontrol.domain._
 import tech.beshu.ror.accesscontrol.factory.RulesValidator.ValidationError
-import tech.beshu.ror.accesscontrol.header.ToHeaderValue
+import tech.beshu.ror.accesscontrol.header.{FromHeaderValue, ToHeaderValue}
 import tech.beshu.ror.com.jayway.jsonpath.JsonPath
 import tech.beshu.ror.providers.EnvVarProvider.EnvVarName
 import tech.beshu.ror.providers.PropertiesProvider.PropName
 import tech.beshu.ror.utils.FilterTransient
+import upickle.default
 
+import scala.collection.SortedSet
 import scala.concurrent.duration.FiniteDuration
 import scala.language.{implicitConversions, postfixOps}
+import scala.util.Try
 
 object header {
 
@@ -70,6 +74,10 @@ object header {
   object ToHeaderValue {
     def apply[T](func: T => NonEmptyString): ToHeaderValue[T] = (t: T) => func(t)
   }
+
+  trait FromHeaderValue[T] {
+    def fromRawValue(value: NonEmptyString): Try[T]
+  }
 }
 
 object orders {
@@ -117,8 +125,8 @@ object show {
     implicit val uriShow: Show[Uri] = Show.show(_.toJavaUri.toString())
     implicit val headerNameShow: Show[Header.Name] = Show.show(_.value.value)
     implicit val documentFieldShow: Show[DocumentField] = Show.show {
-      case f: ADocumentField => f.value
-      case f: NegatedDocumentField => s"~${f.value}"
+      case f: ADocumentField => f.value.value
+      case f: NegatedDocumentField => s"~${f.value.value}"
     }
     implicit val kibanaAppShow: Show[KibanaApp] = Show.show(_.value.value)
     implicit val proxyAuthNameShow: Show[ProxyAuth.Name] = Show.show(_.value)
@@ -238,5 +246,32 @@ object headerValues {
   implicit val transientFilterHeaderValue: ToHeaderValue[Filter] = ToHeaderValue { filter =>
     NonEmptyString.unsafeFrom(FilterTransient.createFromFilter(filter.value.value).serialize())
   }
+  implicit val transientFieldsToHeaderValue: ToHeaderValue[NonEmptySet[DocumentField]] = ToHeaderValue { filters =>
+    implicit val nesW: default.Writer[NonEmptyString] = default.StringWriter.comap(_.value)
+    implicit val documentFieldW: default.Writer[DocumentField] = default.Writer.merge(
+      upickle.default.macroW[DocumentField.ADocumentField],
+      upickle.default.macroW[DocumentField.NegatedDocumentField]
+    )
+    implicit val setR: default.Writer[NonEmptySet[DocumentField]] =
+      default.SeqLikeWriter[Set, DocumentField].comap(_.toSortedSet)
+    val filtersJsonString = upickle.default.write(filters)
+    NonEmptyString.unsafeFrom(
+      Base64.getEncoder.encodeToString(filtersJsonString.getBytes("UTF-8"))
+    )
+  }
+  implicit val transientFieldsFromHeaderValue: FromHeaderValue[NonEmptySet[DocumentField]] = (value: NonEmptyString) => {
+    implicit val nesR: default.Reader[NonEmptyString] = default.StringReader.map(NonEmptyString.unsafeFrom)
+    implicit val documentFieldR: default.Reader[DocumentField] = default.Reader.merge(
+      upickle.default.macroR[DocumentField.ADocumentField],
+      upickle.default.macroR[DocumentField.NegatedDocumentField]
+    )
+    import tech.beshu.ror.accesscontrol.orders._
+    implicit val setR: default.Reader[NonEmptySet[DocumentField]] =
+      default.SeqLikeReader[Set, DocumentField]
+        .map(set => NonEmptySet.fromSetUnsafe(SortedSet.empty[DocumentField] ++ set))
+    Try(upickle.default.read[NonEmptySet[DocumentField]](
+      new String(Base64.getDecoder.decode(value.value), "UTF-8")
+    ))
+  }
   implicit val groupHeaderValue: ToHeaderValue[Group] = ToHeaderValue(_.value)
 }

core/src/main/scala/tech/beshu/ror/fls/FieldsPolicy.scala

@@ -0,0 +1,85 @@
+/*
+ *    This file is part of ReadonlyREST.
+ *
+ *    ReadonlyREST is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation, either version 3 of the License, or
+ *    (at your option) any later version.
+ *
+ *    ReadonlyREST is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with ReadonlyREST.  If not, see http://www.gnu.org/licenses/
+ */
+package tech.beshu.ror.fls
+
+import java.util.regex.Pattern
+
+import cats.data.NonEmptySet
+import cats.implicits._
+import tech.beshu.ror.Constants
+import tech.beshu.ror.accesscontrol.domain.DocumentField
+import tech.beshu.ror.accesscontrol.domain.DocumentField.NegatedDocumentField
+
+class FieldsPolicy(fields: NonEmptySet[DocumentField]) {
+
+  private val enhancedFields = fields.toList.map(new FieldsPolicy.EnhancedDocumentField(_))
+
+  def canKeep(field: String): Boolean = {
+    Constants.FIELDS_ALWAYS_ALLOW.contains(field) || {
+      if (enhancedFields.head.isNegated) {
+        !enhancedFields.exists(f => blacklistMatch(f, field))
+      } else {
+        enhancedFields.exists(f => whitelistMatch(f, field))
+      }
+    }
+  }
+
+  private def whitelistMatch(enhancedField: FieldsPolicy.EnhancedDocumentField, field: String): Boolean = {
+    val fieldParts = field.split("\\.").toList
+    if(enhancedField.fieldPartPatterns.length < fieldParts.length) false
+    else {
+      val foundMismatch = fieldParts.zip(enhancedField.fieldPartPatterns)
+        .exists { case (fieldPart, patternPart) =>
+          if (fieldPart == patternPart.pattern()) false
+          else !wildcardedPatternMatch(patternPart, fieldPart)
+        }
+      !foundMismatch
+    }
+  }
+
+  private def blacklistMatch(enhancedField: FieldsPolicy.EnhancedDocumentField, field: String): Boolean = {
+    val fieldParts = field.split("\\.").toList
+    if(enhancedField.fieldPartPatterns.length  > fieldParts.length) false
+    else {
+      val foundMismatch = enhancedField.fieldPartPatterns.zip(fieldParts)
+        .forall { case (patternPart, fieldPart) =>
+          if (patternPart.pattern() == fieldPart) true
+          else wildcardedPatternMatch(patternPart, fieldPart)
+        }
+      foundMismatch
+    }
+  }
+
+  private def wildcardedPatternMatch(pattern: Pattern, value: String): Boolean = {
+    pattern.matcher(value).find()
+  }
+
+}
+
+object FieldsPolicy {
+  private class EnhancedDocumentField(field: DocumentField) {
+    val fieldPartPatterns: List[Pattern] =
+      field.value.value
+        .split("\\.").toList
+        .map { part =>
+          Pattern.compile(s"^${part.replace("*", ".*")}$$")
+        }
+
+    val isNegated: Boolean = field.isInstanceOf[NegatedDocumentField]
+
+  }
+}