Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
wip: ror stating algorithm
  • Loading branch information
coutoPL committed May 20, 2019
1 parent a4ee2f6 commit 7c47306
Show file tree
Hide file tree
Showing 31 changed files with 251 additions and 75 deletions.
Binary file added config/elasticsearch.keystore
Binary file not shown.
21 changes: 11 additions & 10 deletions core/build.gradle
Expand Up @@ -75,25 +75,25 @@ dependencies {
compile project(':audit')
compile project(path: ':ror-shadowed-libs', configuration: 'shadow')

compile group: 'com.comcast', name: 'ip4s-cats_2.12', version: '1.1.1'
compile group: 'com.google.code.gson', name: 'gson', version: '2.8.2'
compile group: 'com.softwaremill.sttp', name: 'async-http-client-backend-cats_2.12', version: '1.5.8'
compile group: 'com.softwaremill.sttp', name: 'core_2.12', version: '1.5.1'
compile group: 'com.unboundid', name: 'unboundid-ldapsdk', version: '4.0.9'
compile group: 'commons-codec', name: 'commons-codec', version: '1.10'
compile('eu.timepit:refined_2.12:0.9.5') {
exclude group: 'org.scala-lang', module: 'scala-compiler'
}
compile group: 'com.github.pathikrit', name: 'better-files_2.12', version: '3.8.0'
compile group: 'io.circe', name: 'circe-core_2.12', version: '0.10.1'
compile group: 'io.circe', name: 'circe-generic-extras_2.12', version: '0.10.1'
compile group: 'io.circe', name: 'circe-parser_2.12', version: '0.10.1'
compile group: 'io.circe', name: 'circe-yaml_2.12', version: '0.9.0'
compile group: 'commons-codec', name: 'commons-codec', version: '1.10'
compile group: 'com.softwaremill.sttp', name: 'core_2.12', version: '1.5.1'
compile group: 'com.comcast', name: 'ip4s-cats_2.12', version: '1.1.1'
compile group: 'io.jsonwebtoken', name: 'jjwt-api', version: '0.10.5'
compile group: 'io.jsonwebtoken', name: 'jjwt-impl', version: '0.10.5'
compile group: 'io.jsonwebtoken', name: 'jjwt-jackson', version: '0.10.5'
compile group: 'io.monix', name: 'monix_2.12', version: '3.0.0-RC2'
compile group: 'com.google.code.gson', name: 'gson', version: '2.8.2'
// todo: add MDC to logs
compile group: 'org.apache.logging.log4j', name: 'log4j-api-scala_2.12', version: '11.0'
compile group: 'io.monix', name: 'monix_2.12', version: '3.0.0-RC2'
compile('eu.timepit:refined_2.12:0.9.5') {
exclude group: 'org.scala-lang', module: 'scala-compiler'
}
compile('org.reflections:reflections:0.9.11') {
exclude group: 'org.slf4j', module: 'slf4j-api'
exclude group: 'org.slf4j', module: 'slf4j-simple'
Expand All @@ -104,7 +104,8 @@ dependencies {
compile group: 'org.scala-lang.modules', name: 'scala-java8-compat_2.12', version: '0.9.0'
compile group: 'io.lemonlabs', name: 'scala-uri_2.12', version: '1.4.1'
compile group: 'org.typelevel', name: 'squants_2.12', version: '1.4.0'

compile group: 'com.unboundid', name: 'unboundid-ldapsdk', version: '4.0.9'

testCompile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.11.2'
testRuntime group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3'
testRuntime group: 'org.pegdown', name: 'pegdown', version: '1.4.2'
Expand Down
39 changes: 37 additions & 2 deletions core/src/main/scala/tech/beshu/ror/RorEngine.scala
@@ -1,15 +1,50 @@
package tech.beshu.ror

import java.nio.file.Path

import cats.data.EitherT
import monix.eval.Task
import tech.beshu.ror.acl.factory.AsyncHttpClientsFactory
import tech.beshu.ror.acl.logging.AuditSink
import tech.beshu.ror.acl.{Acl, AclStaticContext}
import tech.beshu.ror.configuration.EsConfig.LoadEsConfigError
import tech.beshu.ror.configuration.{EsConfig, FileConfigLoader}
import tech.beshu.ror.es.{AuditSink, IndexContentProvider}
import tech.beshu.ror.utils.OsEnvVarsProvider

object RorEngine {

def start(auditSink: AuditSink): Task[Engine] = ???
private val envVarsProvider = OsEnvVarsProvider

def start(esConfigPath: Path,
auditSink: AuditSink,
indexContentProvider: IndexContentProvider): Task[Either[StartingFailure, Unit]] = {
(for {
fileConfigLoader <- createFileConfigLoader(esConfigPath)
esConfig <- loadEsConfig(esConfigPath)
//_ = if (esConfig.forceLoadRorFromFile) fileConfigLoader.load()
} yield ()).value
}

private def createFileConfigLoader(esConfigPath: Path) = {
EitherT.pure[Task, StartingFailure](new FileConfigLoader(esConfigPath, envVarsProvider))
}

private def loadEsConfig(esConfigPath: Path) = {
EitherT {
EsConfig
.from(esConfigPath)
.map(_.left.map {
case LoadEsConfigError.FileNotFound(file) =>
StartingFailure(s"Cannot find elasticsearch config file: [${file.pathAsString}]")
case LoadEsConfigError.MalformedContent(file, ex) =>
StartingFailure(s"Elasticsearch config file is malformed: [${file.pathAsString}]", Some(ex))
})
}
}
}

final case class StartingFailure(message: String, throwable: Option[Throwable] = None)

final class Engine(val acl: Acl, val context: AclStaticContext, httpClientsFactory: AsyncHttpClientsFactory) {
def shutdown(): Unit = {
httpClientsFactory.shutdown()
Expand Down
7 changes: 4 additions & 3 deletions core/src/main/scala/tech/beshu/ror/RorEngineFactory.scala
Expand Up @@ -5,17 +5,18 @@ import java.time.Clock
import monix.eval.Task
import tech.beshu.ror.acl.factory.CoreFactory.AclCreationError.Reason
import tech.beshu.ror.acl.factory.{AsyncHttpClientsFactory, CoreFactory}
import tech.beshu.ror.acl.logging.{AclLoggingDecorator, AuditSink, AuditingTool}
import tech.beshu.ror.acl.logging.{AclLoggingDecorator, AuditingTool}
import tech.beshu.ror.acl.utils.StaticVariablesResolver
import tech.beshu.ror.acl.{Acl, AclStaticContext}
import tech.beshu.ror.es.AuditSink
import tech.beshu.ror.settings.SettingsMalformedException
import tech.beshu.ror.utils.{JavaEnvVarsProvider, JavaUuidProvider, UuidProvider}
import tech.beshu.ror.utils.{JavaUuidProvider, OsEnvVarsProvider, UuidProvider}

object RorEngineFactory {

private implicit val clock: Clock = Clock.systemUTC()
private implicit val uuidProvider: UuidProvider = JavaUuidProvider
private implicit val resolver: StaticVariablesResolver = new StaticVariablesResolver(JavaEnvVarsProvider)
private implicit val resolver: StaticVariablesResolver = new StaticVariablesResolver(OsEnvVarsProvider)
private val aclFactory = new CoreFactory

def reload(auditSink: AuditSink,
Expand Down
Expand Up @@ -28,6 +28,7 @@ import tech.beshu.ror.acl.logging.AuditingTool.Settings
import tech.beshu.ror.acl.request.RequestContext
import tech.beshu.ror.audit.{AuditLogSerializer, AuditRequestContext, AuditResponseContext}
import tech.beshu.ror.acl.show.logs._
import tech.beshu.ror.es.AuditSink

class AuditingTool(settings: Settings,
auditSink: AuditSink)
Expand Down Expand Up @@ -113,7 +114,3 @@ object AuditingTool {
logSerializer: AuditLogSerializer)

}

trait AuditSink {
def submit(indexName: String, documentId: String, jsonRecord: String): Unit
}
@@ -1,8 +1,31 @@
package tech.beshu.ror.configuration

import io.circe.Json
import monix.eval.Task
import tech.beshu.ror.configuration.ConfigLoader.{ConfigLoaderError, RawRorConfig}
import tech.beshu.ror.configuration.ConfigLoader.ConfigLoaderError.{MoreThanOneRorSection, NoRorSection}

trait ConfigLoader {
trait ConfigLoader[SPECIALIZED_ERROR] {

def load(): Task[RawRorConfig]
def load(): Task[Either[ConfigLoaderError[SPECIALIZED_ERROR], RawRorConfig]]

protected def validateRorJson(json: Json): Either[ConfigLoaderError[SPECIALIZED_ERROR], Json] = {
json \\ "readonlyrest" match {
case Nil => Left(NoRorSection)
case one :: Nil => Right(one)
case _ => Left(MoreThanOneRorSection)
}
}
}

object ConfigLoader {

final case class RawRorConfig(rawConfig: Json) extends AnyVal

sealed trait ConfigLoaderError[+SPECIALIZED_ERROR]
object ConfigLoaderError {
case object NoRorSection extends ConfigLoaderError[Nothing]
case object MoreThanOneRorSection extends ConfigLoaderError[Nothing]
final case class SpecializedError[ERROR](error: ERROR) extends ConfigLoaderError[ERROR]
}
}
48 changes: 48 additions & 0 deletions core/src/main/scala/tech/beshu/ror/configuration/EsConfig.scala
@@ -0,0 +1,48 @@
package tech.beshu.ror.configuration

import java.nio.file.Path

import better.files.File
import monix.eval.Task
import org.yaml.snakeyaml.Yaml
import tech.beshu.ror.configuration.EsConfig.LoadEsConfigError.{FileNotFound, MalformedContent}

import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import scala.util.Try

final case class EsConfig(forceLoadRorFromFile: Boolean)

object EsConfig {

def from(path: Path): Task[Either[LoadEsConfigError, EsConfig]] = Task {
val config = File(s"${path.toAbsolutePath}/elasticsearch.yml")
for {
_ <- Either.cond(config.exists, (), FileNotFound(config))
content <- parseFileContent(config)
} yield EsConfig(
get(content, "readonlyrest.force_load_from_file", false)
)
}

private def parseFileContent(file: File) = {
val yaml = new Yaml()
Try(yaml.load[java.util.Map[String, Object]](file.inputStream.get()))
.toEither
.left.map(MalformedContent(file, _))
.map(_.asScala.toMap[String, Any])
}

private def get[T : ClassTag](config: Map[String, Any], key: String, default: T) = {
config
.get(key)
.collect { case value: T => value }
.getOrElse(default)
}

sealed trait LoadEsConfigError
object LoadEsConfigError {
final case class FileNotFound(file: File) extends LoadEsConfigError
final case class MalformedContent(file: File, throwable: Throwable) extends LoadEsConfigError
}
}
@@ -1,8 +1,50 @@
package tech.beshu.ror.configuration

import java.io.InputStreamReader
import java.nio.file.Path

import better.files.File
import io.circe.Json
import io.circe.yaml.parser
import monix.eval.Task
import tech.beshu.ror.Constants
import tech.beshu.ror.configuration.ConfigLoader.{ConfigLoaderError, RawRorConfig}
import tech.beshu.ror.configuration.ConfigLoader.ConfigLoaderError.SpecializedError
import tech.beshu.ror.configuration.FileConfigLoader.FileConfigError
import tech.beshu.ror.configuration.FileConfigLoader.FileConfigError.{FileNotExist, InvalidFileContent}
import tech.beshu.ror.utils.EnvVarsProvider

class FileConfigLoader(envVarsProvider: EnvVarsProvider) extends ConfigLoader {
override def load(): Task[RawRorConfig] = ???
class FileConfigLoader(esConfigFolderPath: Path,
envVarsProvider: EnvVarsProvider) extends ConfigLoader[FileConfigError] {

override def load(): Task[Either[ConfigLoaderError[FileConfigError], RawRorConfig]] = Task {
val configFile = envVarsProvider.getEnv(Constants.SETTINGS_YAML_FILE_PATH_PROPERTY) match {
case Some(customRorFilePath) => File(customRorFilePath)
case None => File(s"${esConfigFolderPath.toAbsolutePath}/readonlyrest.yml")
}
loadRorConfigFromFile(configFile)
}

private def loadRorConfigFromFile(file: File) = {
for {
_ <- Either.cond(file.exists, (), SpecializedError(FileNotExist(file)))
content <- parseFileContent(file)
} yield RawRorConfig(content)
}

private def parseFileContent(file: File): Either[ConfigLoaderError[FileConfigError], Json] = {
parser
.parse(new InputStreamReader(file.inputStream.get()))
.left.map(ex => SpecializedError(InvalidFileContent(file, ex)))
.flatMap { json => validateRorJson(json) }
}
}

object FileConfigLoader {

sealed trait FileConfigError
object FileConfigError {
final case class FileNotExist(file: File) extends FileConfigError
final case class InvalidFileContent(file: File, throwable: Throwable) extends FileConfigError
}
}
@@ -1,7 +1,42 @@
package tech.beshu.ror.configuration

import io.circe.yaml.parser
import monix.eval.Task
import tech.beshu.ror.configuration.ConfigLoader.{ConfigLoaderError, RawRorConfig}
import tech.beshu.ror.configuration.ConfigLoader.ConfigLoaderError.SpecializedError
import tech.beshu.ror.configuration.IndexConfigLoader.IndexConfigError
import tech.beshu.ror.configuration.IndexConfigLoader.IndexConfigError.{IndexConfigNotExist, InvalidIndexContent}
import tech.beshu.ror.es.IndexContentProvider

class IndexConfigLoader(indexContentProvider: IndexContentProvider)
extends ConfigLoader[IndexConfigError] {

override def load(): Task[Either[ConfigLoaderError[IndexConfigError], RawRorConfig]] =
indexContentProvider
.contentOf(".readonlyrest", "settings", "1")
.map {
case Right(content) =>
parseIndexContent(content).map(RawRorConfig.apply)
case Left(IndexContentProvider.Error.CannotReachContentSource) =>
Left(SpecializedError[IndexConfigError](IndexConfigNotExist))
case Left(IndexContentProvider.Error.ContentNotFound) =>
Left(SpecializedError[IndexConfigError](IndexConfigNotExist))
}

private def parseIndexContent(content: String) = {
parser
.parse(content)
.left.map(ex => SpecializedError(InvalidIndexContent(ex)))
.flatMap { json => validateRorJson(json) }
}
}

object IndexConfigLoader {

sealed trait IndexConfigError
object IndexConfigError {
case object IndexConfigNotExist extends IndexConfigError
final case class InvalidIndexContent(throwable: Throwable) extends IndexConfigError
}

class IndexConfigLoader extends ConfigLoader {
override def load(): Task[RawRorConfig] = ???
}

This file was deleted.

5 changes: 5 additions & 0 deletions core/src/main/scala/tech/beshu/ror/es/AuditSink.scala
@@ -0,0 +1,5 @@
package tech.beshu.ror.es

trait AuditSink {
def submit(indexName: String, documentId: String, jsonRecord: String): Unit
}
17 changes: 17 additions & 0 deletions core/src/main/scala/tech/beshu/ror/es/IndexContentProvider.scala
@@ -0,0 +1,17 @@
package tech.beshu.ror.es

import monix.eval.Task

trait IndexContentProvider {

def contentOf(index: String, `type`: String, id: String): Task[Either[IndexContentProvider.Error, String]]
}

object IndexContentProvider {

sealed trait Error
object Error {
case object ContentNotFound extends Error
case object CannotReachContentSource extends Error
}
}
Expand Up @@ -6,7 +6,7 @@ trait EnvVarsProvider {
def getEnv(name: String): Option[String]
}

object JavaEnvVarsProvider extends EnvVarsProvider {
object OsEnvVarsProvider extends EnvVarsProvider {
override def getEnv(name: String): Option[String] =
Try(Option(System.getenv(name))).toOption.flatten
}
Expand Up @@ -33,14 +33,14 @@ import tech.beshu.ror.acl.factory.{CoreFactory, CoreSettings}
import tech.beshu.ror.acl.utils.StaticVariablesResolver
import tech.beshu.ror.mocks.{MockHttpClientsFactory, MockRequestContext}
import tech.beshu.ror.utils.TestsUtils.{BlockContextAssertion, basicAuthHeader}
import tech.beshu.ror.utils.{JavaEnvVarsProvider, JavaUuidProvider, UuidProvider}
import tech.beshu.ror.utils.{OsEnvVarsProvider, JavaUuidProvider, UuidProvider}

class AuthKeyYamlLoadedAclTests extends WordSpec with MockFactory with Inside with BlockContextAssertion {

private val factory = {
implicit val clock: Clock = Clock.systemUTC()
implicit val uuidProvider: UuidProvider = JavaUuidProvider
implicit val resolver: StaticVariablesResolver = new StaticVariablesResolver(JavaEnvVarsProvider)
implicit val resolver: StaticVariablesResolver = new StaticVariablesResolver(OsEnvVarsProvider)
new CoreFactory
}
private val acl: Acl = factory
Expand Down

0 comments on commit 7c47306

Please sign in to comment.