From e1d55fb2845c4fccc697821dc8edcce5d4091123 Mon Sep 17 00:00:00 2001 From: Timo Rantalaiho Date: Wed, 20 Jun 2018 14:52:40 +0300 Subject: [PATCH 1/7] OY-227 : Better names No functional changes. --- .../oppijanumerorekisteri.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala index 6be1f641a..5a0840700 100644 --- a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala +++ b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala @@ -43,7 +43,7 @@ object Henkilo { trait OppijanumerorekisteriService { def henkilo(personOid: String): Henkilo - def fetchAllDuplicateOids(pOid: String): List[String] + def fetchAllDuplicateOids(oppijanumero: String): List[String] } trait OppijanumerorekisteriComponent { @@ -51,7 +51,7 @@ trait OppijanumerorekisteriComponent { class StubbedOppijanumerorekisteriService extends OppijanumerorekisteriService { override def henkilo(personOid: String): Henkilo = ??? - override def fetchAllDuplicateOids(pOid: String): List[String] = List(pOid) + override def fetchAllDuplicateOids(oppijanumero: String): List[String] = List(oppijanumero) } class RemoteOppijanumerorekisteriService(config: AppConfig) extends OppijanumerorekisteriService with JsonFormats with Logging { @@ -91,12 +91,12 @@ trait OppijanumerorekisteriComponent { httpClient.fetch(request)(r => r.as[String].map(body => decoder(r.status.code, body, request))) } - override def fetchAllDuplicateOids(pOid: String): List[String] = { + override def fetchAllDuplicateOids(oppijanumero: String): List[String] = { implicit val formats = DefaultFormats val timeout = 1000*30L val masterRequest: Request = Request( - uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-master", pOid)), + uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-master", oppijanumero)), headers = Headers(callerIdHeader)) val masterOid: String = runHttp[Option[String]](masterRequest) { @@ -105,9 +105,9 @@ trait OppijanumerorekisteriComponent { val oid = f \ "oidHenkilo" Some(oid.extract[String]) case (code, responseString, _) => - logger.error("Failed to fetch master oid for user oid {}, response was {}, {}", pOid, Integer.toString(code), responseString) + logger.error("Failed to fetch master oid for user oid {}, response was {}, {}", oppijanumero, Integer.toString(code), responseString) None - }.runFor(timeoutInMillis = timeout).getOrElse(pOid) + }.runFor(timeoutInMillis = timeout).getOrElse(oppijanumero) val slaveRequest: Request = Request( uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-slaves", masterOid)), From 597ac87303cf46c9366e4178e9203d3bd35b5038 Mon Sep 17 00:00:00 2001 From: Timo Rantalaiho Date: Wed, 20 Jun 2018 14:53:18 +0300 Subject: [PATCH 2/7] OY-227 : Better name No functional changes. --- .../fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala b/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala index 5426563e9..eb4dbe40d 100644 --- a/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala +++ b/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala @@ -63,9 +63,9 @@ trait ApplicationsServletContainer { get("/") { implicit val formats = DefaultFormats val timeout = 1000*30L - val pOid: String = personOid() + val oppijanumero: String = personOid() - val allOids: List[String] = oppijanumerorekisteriService.fetchAllDuplicateOids(pOid) + val allOids: List[String] = oppijanumerorekisteriService.fetchAllDuplicateOids(oppijanumero) var allSuccess = true val allHakemukset = allOids.flatMap(oid => { From b219577139205f12508328c568d6ef6e93a00d5e Mon Sep 17 00:00:00 2001 From: Timo Rantalaiho Date: Wed, 20 Jun 2018 14:54:31 +0300 Subject: [PATCH 3/7] OY-227 : Add type No functional changes --- .../oppijanumerorekisteri/oppijanumerorekisteri.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala index 5a0840700..e6e85238c 100644 --- a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala +++ b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala @@ -93,7 +93,7 @@ trait OppijanumerorekisteriComponent { override def fetchAllDuplicateOids(oppijanumero: String): List[String] = { implicit val formats = DefaultFormats - val timeout = 1000*30L + val timeout = Duration(30, TimeUnit.SECONDS) val masterRequest: Request = Request( uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-master", oppijanumero)), @@ -107,7 +107,7 @@ trait OppijanumerorekisteriComponent { case (code, responseString, _) => logger.error("Failed to fetch master oid for user oid {}, response was {}, {}", oppijanumero, Integer.toString(code), responseString) None - }.runFor(timeoutInMillis = timeout).getOrElse(oppijanumero) + }.runFor(timeoutInMillis = timeout.toMillis).getOrElse(oppijanumero) val slaveRequest: Request = Request( uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-slaves", masterOid)), @@ -124,7 +124,7 @@ trait OppijanumerorekisteriComponent { case (code, responseString, _) => logger.error("Failed to fetch slave OIDs for user oid {}, response was {}, {}", masterOid, Integer.toString(code), responseString) List(masterOid) - }.runFor(timeoutInMillis = timeout) + }.runFor(timeoutInMillis = timeout.toMillis) return allOids } From b095912615582d543c23f2bf6e68f4ba1484fa53 Mon Sep 17 00:00:00 2001 From: Timo Rantalaiho Date: Wed, 20 Jun 2018 14:59:08 +0300 Subject: [PATCH 4/7] OY-227 : Format No functional changes --- .../oppijanumerorekisteri.scala | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala index e6e85238c..a3b66285b 100644 --- a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala +++ b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala @@ -92,39 +92,39 @@ trait OppijanumerorekisteriComponent { } override def fetchAllDuplicateOids(oppijanumero: String): List[String] = { - implicit val formats = DefaultFormats - val timeout = Duration(30, TimeUnit.SECONDS) - - val masterRequest: Request = Request( - uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-master", oppijanumero)), - headers = Headers(callerIdHeader)) - - val masterOid: String = runHttp[Option[String]](masterRequest) { - case (200, resultString, _) => - val f: json4s.JValue = parse(resultString).asInstanceOf[JObject] - val oid = f \ "oidHenkilo" - Some(oid.extract[String]) - case (code, responseString, _) => - logger.error("Failed to fetch master oid for user oid {}, response was {}, {}", oppijanumero, Integer.toString(code), responseString) - None - }.runFor(timeoutInMillis = timeout.toMillis).getOrElse(oppijanumero) - - val slaveRequest: Request = Request( - uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-slaves", masterOid)), - headers = Headers(callerIdHeader)) - - val allOids: List[String] = runHttp(slaveRequest) { - case (200, resultString, _) => - val slaveOids: Seq[String] = parse(resultString).extract[List[JObject]] - .map(obj => { - val oidObj = obj \ "oidHenkilo" - oidObj.extract[String] - }) - List(masterOid) ++ slaveOids - case (code, responseString, _) => - logger.error("Failed to fetch slave OIDs for user oid {}, response was {}, {}", masterOid, Integer.toString(code), responseString) - List(masterOid) - }.runFor(timeoutInMillis = timeout.toMillis) + implicit val formats = DefaultFormats + val timeout = Duration(30, TimeUnit.SECONDS) + + val masterRequest: Request = Request( + uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-master", oppijanumero)), + headers = Headers(callerIdHeader)) + + val masterOid: String = runHttp[Option[String]](masterRequest) { + case (200, resultString, _) => + val f: json4s.JValue = parse(resultString).asInstanceOf[JObject] + val oid = f \ "oidHenkilo" + Some(oid.extract[String]) + case (code, responseString, _) => + logger.error("Failed to fetch master oid for user oid {}, response was {}, {}", oppijanumero, Integer.toString(code), responseString) + None + }.runFor(timeoutInMillis = timeout.toMillis).getOrElse(oppijanumero) + + val slaveRequest: Request = Request( + uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-slaves", masterOid)), + headers = Headers(callerIdHeader)) + + val allOids: List[String] = runHttp(slaveRequest) { + case (200, resultString, _) => + val slaveOids: Seq[String] = parse(resultString).extract[List[JObject]] + .map(obj => { + val oidObj = obj \ "oidHenkilo" + oidObj.extract[String] + }) + List(masterOid) ++ slaveOids + case (code, responseString, _) => + logger.error("Failed to fetch slave OIDs for user oid {}, response was {}, {}", masterOid, Integer.toString(code), responseString) + List(masterOid) + }.runFor(timeoutInMillis = timeout.toMillis) return allOids } From a3cc741552bd2d9fc7825525090e67fc4011bec6 Mon Sep 17 00:00:00 2001 From: Timo Rantalaiho Date: Thu, 21 Jun 2018 13:17:37 +0300 Subject: [PATCH 5/7] OY-227 : fetch linked persons with single call --- src/main/resources/omatsivut-oph.properties | 3 +- .../oppijanumerorekisteri.scala | 82 +++++++++---------- .../servlet/ApplicationsServlet.scala | 2 +- 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/src/main/resources/omatsivut-oph.properties b/src/main/resources/omatsivut-oph.properties index 6fc11e99c..671615920 100644 --- a/src/main/resources/omatsivut-oph.properties +++ b/src/main/resources/omatsivut-oph.properties @@ -32,6 +32,5 @@ ataru.applications.modify=${url-oppija}/hakemus?modify=$1 url-oppijanumerorekisteri-service=${url-virkailija}/oppijanumerorekisteri-service oppijanumerorekisteri-service.henkiloByOid=${url-oppijanumerorekisteri-service}/henkilo/$1 -oppijanumerorekisteri-service.henkilo-slaves=${url-oppijanumerorekisteri-service}/henkilo/$1/slaves -oppijanumerorekisteri-service.henkilo-master=${url-oppijanumerorekisteri-service}/henkilo/$1/master +oppijanumerorekisteri-service.duplicatesByPersonOids=${url-oppijanumerorekisteri-service}/s2s/duplicateHenkilos oppijanumerorekisteri-service.henkiloByHetu=${url-oppijanumerorekisteri-service}/henkilo/hetu=$1 diff --git a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala index a3b66285b..e847cf766 100644 --- a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala +++ b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala @@ -7,17 +7,18 @@ import fi.vm.sade.omatsivut.OphUrlProperties import fi.vm.sade.omatsivut.config.AppConfig.AppConfig import fi.vm.sade.utils.cas.{CasAuthenticatingClient, CasClient, CasParams} import fi.vm.sade.utils.slf4j.Logging +import org.http4s.MediaType.`application/json` import org.http4s.Method.GET +import org.http4s._ import org.http4s.client.blaze -import org.http4s.{Header, Headers, Request, Uri} -import org.json4s -import org.json4s._ +import org.http4s.headers.{Accept, `Content-Type`} import org.json4s.JsonAST.{JNull, JObject, JString, JValue} import org.json4s.jackson.JsonMethods -import org.json4s.{DefaultFormats, Reader, Writer} +import org.json4s.jackson.Serialization.write +import org.json4s.{DefaultFormats, Reader, Writer, _} +import scalaz.concurrent.Task import scala.concurrent.duration.Duration -import scalaz.concurrent.Task case class Henkilo(oid: String, hetu: Option[String]) @@ -43,7 +44,7 @@ object Henkilo { trait OppijanumerorekisteriService { def henkilo(personOid: String): Henkilo - def fetchAllDuplicateOids(oppijanumero: String): List[String] + def fetchAllDuplicateOids(oppijanumero: String): Set[String] } trait OppijanumerorekisteriComponent { @@ -51,11 +52,11 @@ trait OppijanumerorekisteriComponent { class StubbedOppijanumerorekisteriService extends OppijanumerorekisteriService { override def henkilo(personOid: String): Henkilo = ??? - override def fetchAllDuplicateOids(oppijanumero: String): List[String] = List(oppijanumero) + + override def fetchAllDuplicateOids(oppijanumero: String): Set[String] = Set(oppijanumero) } class RemoteOppijanumerorekisteriService(config: AppConfig) extends OppijanumerorekisteriService with JsonFormats with Logging { - import org.json4s.jackson.JsonMethods._ private val blazeHttpClient = blaze.defaultClient private val casClient = new CasClient(config.settings.securitySettings.casUrl, blazeHttpClient) private val casParams = CasParams( @@ -91,42 +92,39 @@ trait OppijanumerorekisteriComponent { httpClient.fetch(request)(r => r.as[String].map(body => decoder(r.status.code, body, request))) } - override def fetchAllDuplicateOids(oppijanumero: String): List[String] = { - implicit val formats = DefaultFormats + override def fetchAllDuplicateOids(oppijanumero: String): Set[String] = { val timeout = Duration(30, TimeUnit.SECONDS) - val masterRequest: Request = Request( - uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-master", oppijanumero)), - headers = Headers(callerIdHeader)) - - val masterOid: String = runHttp[Option[String]](masterRequest) { - case (200, resultString, _) => - val f: json4s.JValue = parse(resultString).asInstanceOf[JObject] - val oid = f \ "oidHenkilo" - Some(oid.extract[String]) - case (code, responseString, _) => - logger.error("Failed to fetch master oid for user oid {}, response was {}, {}", oppijanumero, Integer.toString(code), responseString) - None - }.runFor(timeoutInMillis = timeout.toMillis).getOrElse(oppijanumero) - - val slaveRequest: Request = Request( - uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.henkilo-slaves", masterOid)), - headers = Headers(callerIdHeader)) - - val allOids: List[String] = runHttp(slaveRequest) { - case (200, resultString, _) => - val slaveOids: Seq[String] = parse(resultString).extract[List[JObject]] - .map(obj => { - val oidObj = obj \ "oidHenkilo" - oidObj.extract[String] - }) - List(masterOid) ++ slaveOids - case (code, responseString, _) => - logger.error("Failed to fetch slave OIDs for user oid {}, response was {}, {}", masterOid, Integer.toString(code), responseString) - List(masterOid) - }.runFor(timeoutInMillis = timeout.toMillis) - - return allOids + val duplicateHenkilosRequest = Request( + method = Method.POST, + uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.duplicatesByPersonOids")), + headers = Headers(callerIdHeader, `Content-Type`(`application/json`), `Accept`(`application/json`)) + ).withBody(write(Map("henkiloOids" -> List(oppijanumero)))) + + val henkiloviitteet: Seq[Henkiloviite] = httpClient.fetch(duplicateHenkilosRequest)((r: Response) => + if (r.status == Status.Ok) { + r.as[String].map(parseHenkiloviiteResponse(_, oppijanumero)) + } else { + logger.error("Failed to fetch henkiloviite data for user oid {}, response was {}, {}", oppijanumero, r.status, r.body) + Task.now(Nil) + }).runFor(timeout) + val allHenkiloOids: Set[String] = henkiloviitteet.flatMap(viite => Set(viite.henkiloOid, viite.masterOid)).++(Seq(oppijanumero)).toSet + if (allHenkiloOids.size > 1) { + logger.info(s"Got ${allHenkiloOids.size} in total for oppijanumero $oppijanumero : $allHenkiloOids") + } + allHenkiloOids + } + + private def parseHenkiloviiteResponse(responseBody: String, oppijanumero: String): Seq[Henkiloviite] = { + try { + JsonMethods.parse(responseBody).extract[Seq[Henkiloviite]] + } catch { + case e: Exception => + logger.error(s"Problem when parsing Henkiloviite list for $oppijanumero from response '$responseBody'", e) + throw e + } } } } + +case class Henkiloviite(henkiloOid: String, masterOid: String) diff --git a/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala b/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala index eb4dbe40d..c6c1a2960 100644 --- a/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala +++ b/src/main/scala/fi/vm/sade/omatsivut/servlet/ApplicationsServlet.scala @@ -65,7 +65,7 @@ trait ApplicationsServletContainer { val timeout = 1000*30L val oppijanumero: String = personOid() - val allOids: List[String] = oppijanumerorekisteriService.fetchAllDuplicateOids(oppijanumero) + val allOids: Iterable[String] = oppijanumerorekisteriService.fetchAllDuplicateOids(oppijanumero) var allSuccess = true val allHakemukset = allOids.flatMap(oid => { From af042a1a3e3f0b7f5020b4001f85ab8706b7be07 Mon Sep 17 00:00:00 2001 From: Timo Rantalaiho Date: Fri, 27 Jul 2018 12:48:20 +0300 Subject: [PATCH 6/7] Fix Scala style No functional changes. --- src/test/scala/fi/vm/sade/omatsivut/TestProfile.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/scala/fi/vm/sade/omatsivut/TestProfile.scala b/src/test/scala/fi/vm/sade/omatsivut/TestProfile.scala index 9753d3648..c57cbbf63 100644 --- a/src/test/scala/fi/vm/sade/omatsivut/TestProfile.scala +++ b/src/test/scala/fi/vm/sade/omatsivut/TestProfile.scala @@ -4,9 +4,9 @@ import fi.vm.sade.omatsivut.config.{ComponentRegistry, AppConfig} object SharedAppConfig { lazy final val appConfig = new AppConfig.IT - lazy val componentRegistry = { + lazy val componentRegistry: ComponentRegistry = { val registry = new ComponentRegistry(appConfig) - registry.start + registry.start() registry } } From cdd135b32dc976e608b22f92d164e6671e0adcb6 Mon Sep 17 00:00:00 2001 From: Timo Rantalaiho Date: Fri, 27 Jul 2018 14:54:32 +0300 Subject: [PATCH 7/7] OY-227 : Fix request content-type Calling withBody with String seems to always produce text/plain. --- .../oppijanumerorekisteri/oppijanumerorekisteri.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala index e847cf766..da7e1ce60 100644 --- a/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala +++ b/src/main/scala/fi/vm/sade/hakemuseditori/oppijanumerorekisteri/oppijanumerorekisteri.scala @@ -2,6 +2,7 @@ package fi.vm.sade.hakemuseditori.oppijanumerorekisteri import java.util.concurrent.TimeUnit +import fi.vm.sade.groupemailer.Json4sHttp4s import fi.vm.sade.hakemuseditori.json.JsonFormats import fi.vm.sade.omatsivut.OphUrlProperties import fi.vm.sade.omatsivut.config.AppConfig.AppConfig @@ -11,10 +12,10 @@ import org.http4s.MediaType.`application/json` import org.http4s.Method.GET import org.http4s._ import org.http4s.client.blaze -import org.http4s.headers.{Accept, `Content-Type`} +import org.http4s.headers.Accept +import org.json4s import org.json4s.JsonAST.{JNull, JObject, JString, JValue} import org.json4s.jackson.JsonMethods -import org.json4s.jackson.Serialization.write import org.json4s.{DefaultFormats, Reader, Writer, _} import scalaz.concurrent.Task @@ -95,11 +96,12 @@ trait OppijanumerorekisteriComponent { override def fetchAllDuplicateOids(oppijanumero: String): Set[String] = { val timeout = Duration(30, TimeUnit.SECONDS) + val body: json4s.JValue = Extraction.decompose(Map("henkiloOids" -> List(oppijanumero))) val duplicateHenkilosRequest = Request( method = Method.POST, uri = uriFromString(OphUrlProperties.url("oppijanumerorekisteri-service.duplicatesByPersonOids")), - headers = Headers(callerIdHeader, `Content-Type`(`application/json`), `Accept`(`application/json`)) - ).withBody(write(Map("henkiloOids" -> List(oppijanumero)))) + headers = Headers(callerIdHeader, `Accept`(`application/json`)) + ).withBody(body)(Json4sHttp4s.json4sEncoderOf) val henkiloviitteet: Seq[Henkiloviite] = httpClient.fetch(duplicateHenkilosRequest)((r: Response) => if (r.status == Status.Ok) {