Merge pull request #351 from guardian/kc-backfill

Implement holiday-stop backfill with new detail model
guardian · Jul 25, 2019 · 96d980b · 96d980b
2 parents 80228b0 + 5bdd0bf
commit 96d980b
Show file tree

Hide file tree

Showing 10 changed files with 74 additions and 87 deletions.
diff --git a/...ers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/BackfillResult.scala b/...ers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/BackfillResult.scala
@@ -1,6 +1,9 @@
 package com.gu.holidaystopbackfill
 
 import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest.NewHolidayStopRequest
-import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.HolidayStopRequestsDetailPending
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.ActionedHolidayStopRequestsDetailToBackfill
 
-case class BackfillResult(requests: Seq[NewHolidayStopRequest], zuoraRefs: Seq[HolidayStopRequestsDetailPending])
+case class BackfillResult(
+  requests: Seq[NewHolidayStopRequest],
+  zuoraRefs: Seq[ActionedHolidayStopRequestsDetailToBackfill]
+)
diff --git a/handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Backfiller.scala b/handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Backfiller.scala
@@ -1,7 +1,6 @@
 package com.gu.holidaystopbackfill
 
 import java.io.File
-import java.time.LocalDate
 
 import com.gu.holidaystopbackfill.SalesforceHolidayStop._
 import com.gu.holidaystopbackfill.ZuoraHolidayStop.holidayStopsAlreadyInZuora
@@ -13,15 +12,15 @@ object Backfiller {
    * If any call fails it should leave Salesforce in a consistent state.
    * First, the holiday request table is updated, and then the zuora refs child table.
    */
-  def backfill(src: File, startThreshold: LocalDate, endThreshold: Option[LocalDate], dryRun: Boolean, stage: String): Either[BackfillFailure, BackfillResult] = {
+  def backfill(src: File, dryRun: Boolean, stage: String): Either[BackfillFailure, BackfillResult] = {
     for {
       config <- Config(stage)
       stopsInZuora <- Right(holidayStopsAlreadyInZuora(src))
-      requestsInSf <- holidayStopRequestsAlreadyInSalesforce(config)(startThreshold, endThreshold)
-      requestsToAddToSf = holidayStopRequestsToBeBackfilled(stopsInZuora, requestsInSf)
+      requestsInSf1 <- holidayStopRequestsAlreadyInSalesforce(config)
+      requestsToAddToSf = holidayStopRequestsToBeBackfilled(stopsInZuora, requestsInSf1)
       _ <- holidayStopRequestsAddedToSalesforce(config, dryRun)(requestsToAddToSf)
-      detailsInSf <- detailsAlreadyInSalesforce(config)(startThreshold, endThreshold)
-      detailsToAddToSf = detailsToBeBackfilled(stopsInZuora, detailsInSf)
+      requestsInSf2 <- holidayStopRequestsAlreadyInSalesforce(config)
+      detailsToAddToSf = detailsToBeBackfilled(stopsInZuora, requestsInSf2)
       _ <- detailsAddedToSalesforce(config, dryRun)(detailsToAddToSf)
     } yield BackfillResult(requestsToAddToSf, detailsToAddToSf)
   }

diff --git a/...ers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/BackfillingApp.scala b/...ers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/BackfillingApp.scala
@@ -1,7 +1,6 @@
 package com.gu.holidaystopbackfill
 
 import java.io.File
-import java.time.LocalDate
 
 /**
  * <p>Backfill app, to be run from a dev machine.</p>
@@ -16,8 +15,6 @@ import java.time.LocalDate
  *     <li>Holiday start date.</li>
  *     <li>Holiday end date.</li>
  *     <li>Credit price.</ol></li>
- *   <li>Earliest date of start of holiday-stops to find.</li>
- *   <li>Latest date of end of holiday-stops to find.</li>
  *   <li>Salesforce stage to backfill.  Defaults to DEV.</li>
  * </ol>
  * </p>
@@ -26,11 +23,9 @@ object BackfillingApp extends App {
 
   val dryRun = args(0).toBoolean
   val src = new File(args(1))
-  val start = LocalDate.parse(args(2))
-  val end = args.lift(3).map(LocalDate.parse)
-  val stage = args.lift(4).getOrElse("DEV")
+  val stage = args.lift(2).getOrElse("DEV")
 
-  Backfiller.backfill(src, start, end, dryRun, stage) match {
+  Backfiller.backfill(src, dryRun, stage) match {
     case Left(failure) => println(s"Failed: $failure")
     case Right(result) =>
       println("Success!")

diff --git a/.../holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/HolidayStopDetail.scala b/.../holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/HolidayStopDetail.scala
@@ -6,7 +6,7 @@ import java.time.{DayOfWeek, LocalDate}
 
 import com.gu.salesforce.SalesforceAuthenticate.SFAuthConfig
 import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest.{HolidayStopRequest, HolidayStopRequestEndDate, HolidayStopRequestStartDate, NewHolidayStopRequest, SubscriptionNameLookup}
-import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{HolidayStopRequestId, HolidayStopRequestsDetail, HolidayStopRequestsDetailChargeCode, HolidayStopRequestsDetailChargePrice, HolidayStopRequestsDetailPending, ProductName, StoppedPublicationDate, SubscriptionName}
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail._
 import com.gu.util.Time
 
 import scala.io.Source
@@ -42,15 +42,8 @@ object ZuoraHolidayStop {
 
 object SalesforceHolidayStop {
 
-  def holidayStopRequestsAlreadyInSalesforce(sfCredentials: SFAuthConfig)(
-    start: LocalDate,
-    end: Option[LocalDate]
-  ): Either[SalesforceFetchFailure, Seq[HolidayStopRequest]] = {
-    Salesforce.holidayStopRequestsByProductAndDateRange(sfCredentials)(ProductName("Guardian Weekly"), start, end.getOrElse(LocalDate.MAX))
-  }
-
-  def detailsAlreadyInSalesforce(sfCredentials: SFAuthConfig)(start: LocalDate, end: Option[LocalDate]): Either[SalesforceFetchFailure, Seq[HolidayStopRequestsDetail]] = {
-    Salesforce.holidayStopRequestDetails(sfCredentials)(ProductName("Guardian Weekly"), start, end.getOrElse(LocalDate.MAX))
+  def holidayStopRequestsAlreadyInSalesforce(sfCredentials: SFAuthConfig): Either[SalesforceFetchFailure, Seq[HolidayStopRequest]] = {
+    Salesforce.holidayStopRequestsByProduct(sfCredentials)(ProductName("Guardian Weekly"))
   }
 
   def holidayStopRequestsToBeBackfilled(inZuora: Seq[ZuoraHolidayStop], inSalesforce: Seq[HolidayStopRequest]): Seq[NewHolidayStopRequest] = {
@@ -85,7 +78,7 @@ object SalesforceHolidayStop {
     }
   }
 
-  def detailsToBeBackfilled(inZuora: Seq[ZuoraHolidayStop], inSalesforce: Seq[HolidayStopRequestsDetail]): Seq[HolidayStopRequestsDetailPending] = {
+  def detailsToBeBackfilled(inZuora: Seq[ZuoraHolidayStop], inSalesforce: Seq[HolidayStopRequest]): Seq[ActionedHolidayStopRequestsDetailToBackfill] = {
 
     /*
      * We take legacy holiday stops that have a range of dates
@@ -115,16 +108,35 @@ object SalesforceHolidayStop {
      * There should be a request ID available for each subscription and stopped publication date
      * as in the first pass the parent holiday requests will have been populated.
      */
-    def correspondingRequestId(zStop: ZuoraHolidayStop): Option[HolidayStopRequestId] = None
+    def correspondingRequest(zStop: ZuoraHolidayStop): Option[HolidayStopRequest] = {
+      inSalesforce find { sfStop =>
+        val startDate = Time.toJavaDate(sfStop.Start_Date__c.value)
+        val endDate = Time.toJavaDate(sfStop.End_Date__c.value)
+        sfStop.Subscription_Name__c == zStop.subscriptionName &&
+          (startDate.isBefore(zStop.startDate) || startDate.isEqual(zStop.startDate)) &&
+          (endDate.isEqual(zStop.endDate) || endDate.isAfter(zStop.endDate))
+      }
+    }
+
+    def alreadyBackfilled(zuoraStop: ZuoraHolidayStop): Boolean =
+      inSalesforce.exists {
+        _.Holiday_Stop_Request_Detail__r.exists {
+          _.records.exists { sfDetail =>
+            isSame(zuoraStop, sfDetail)
+          }
+        }
+      }
 
     val details = for {
-      stop <- stoppedPublications.filterNot(zuoraStop => inSalesforce.exists(sfStop => isSame(zuoraStop, sfStop)))
-      sfRequestId <- correspondingRequestId(stop)
+      zStop <- stoppedPublications.filterNot(alreadyBackfilled)
+      sfRequest <- correspondingRequest(zStop)
     } yield {
-      HolidayStopRequestsDetailPending(
-        sfRequestId,
-        StoppedPublicationDate(stop.startDate),
-        Some(HolidayStopRequestsDetailChargePrice(stop.creditPrice))
+      ActionedHolidayStopRequestsDetailToBackfill(
+        sfRequest.Id,
+        StoppedPublicationDate(zStop.startDate),
+        Some(HolidayStopRequestsDetailChargePrice(zStop.creditPrice)),
+        Some(zStop.chargeNumber),
+        Some(HolidayStopRequestsDetailChargePrice(zStop.creditPrice))
       )
     }
 
@@ -139,7 +151,7 @@ object SalesforceHolidayStop {
       Right(())
     } else Salesforce.holidayStopCreateResponse(sfCredentials)(requests)
 
-  def detailsAddedToSalesforce(sfCredentials: SFAuthConfig, dryRun: Boolean)(details: Seq[HolidayStopRequestsDetailPending]): Either[SalesforceUpdateFailure, Unit] =
+  def detailsAddedToSalesforce(sfCredentials: SFAuthConfig, dryRun: Boolean)(details: Seq[ActionedHolidayStopRequestsDetailToBackfill]): Either[SalesforceUpdateFailure, Unit] =
     if (dryRun) {
       println("-----------------------------")
       details.foreach(println)

diff --git a/handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Salesforce.scala b/handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Salesforce.scala
@@ -1,43 +1,30 @@
 package com.gu.holidaystopbackfill
 
-import java.time.LocalDate
-
 import com.gu.effects.RawEffects
 import com.gu.salesforce.SalesforceAuthenticate.SFAuthConfig
 import com.gu.salesforce.SalesforceClient
-import com.gu.salesforce.holiday_stops.{SalesforceHolidayStopRequest, SalesforceHolidayStopRequestsDetail}
 import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest.{CreateHolidayStopRequest, HolidayStopRequest, NewHolidayStopRequest}
-import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{HolidayStopRequestsDetail, HolidayStopRequestsDetailPending, ProductName}
-import com.gu.util.Time
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{ActionedHolidayStopRequestsDetailToBackfill, ProductName}
+import com.gu.salesforce.holiday_stops.{SalesforceHolidayStopRequest, SalesforceHolidayStopRequestsDetail}
 import com.gu.util.resthttp.JsonHttp
 import scalaz.{-\/, \/-}
 
 object Salesforce {
 
-  def holidayStopRequestsByProductAndDateRange(sfCredentials: SFAuthConfig)(productNamePrefix: ProductName, start: LocalDate, end: LocalDate): Either[SalesforceFetchFailure, Seq[HolidayStopRequest]] =
+  def holidayStopRequestsByProduct(sfCredentials: SFAuthConfig)(productNamePrefix: ProductName): Either[SalesforceFetchFailure, Seq[HolidayStopRequest]] =
     SalesforceClient(RawEffects.response, sfCredentials).value.flatMap { sfAuth =>
       val sfGet = sfAuth.wrapWith(JsonHttp.getWithParams)
-      val fetchOp = SalesforceHolidayStopRequest.LookupByDateRangeAndProductNamePrefix(sfGet)
-      fetchOp(Time.toJodaDate(start), Time.toJodaDate(end), productNamePrefix)
+      val fetchOp = SalesforceHolidayStopRequest.LookupByProductNamePrefix(sfGet)
+      fetchOp(productNamePrefix)
     }.toDisjunction match {
       case -\/(failure) => Left(SalesforceFetchFailure(failure.toString))
       case \/-(requests) => Right(requests)
     }
 
-  def holidayStopRequestDetails(sfCredentials: SFAuthConfig)(productNamePrefix: ProductName, startThreshold: LocalDate, endThreshold: LocalDate): Either[SalesforceFetchFailure, Seq[HolidayStopRequestsDetail]] =
-    SalesforceClient(RawEffects.response, sfCredentials).value.flatMap { sfAuth =>
-      val sfGet = sfAuth.wrapWith(JsonHttp.getWithParams)
-      val fetchOp = SalesforceHolidayStopRequestsDetail.LookupActionedByProductNamePrefixAndDateRange(sfGet)
-      fetchOp(productNamePrefix, startThreshold, endThreshold)
-    }.toDisjunction match {
-      case -\/(failure) => Left(SalesforceFetchFailure(failure.toString))
-      case \/-(details) => Right(details)
-    }
-
-  def holidayStopDetailsCreateResponse(sfCredentials: SFAuthConfig)(details: Seq[HolidayStopRequestsDetailPending]): Either[SalesforceUpdateFailure, Unit] =
+  def holidayStopDetailsCreateResponse(sfCredentials: SFAuthConfig)(details: Seq[ActionedHolidayStopRequestsDetailToBackfill]): Either[SalesforceUpdateFailure, Unit] =
     SalesforceClient(RawEffects.response, sfCredentials).value.map { sfAuth =>
       val sfPost = sfAuth.wrapWith(JsonHttp.post)
-      val sendOp = SalesforceHolidayStopRequestsDetail.CreatePendingSalesforceHolidayStopRequestsDetail(sfPost)
+      val sendOp = SalesforceHolidayStopRequestsDetail.BackfillActionedSalesforceHolidayStopRequestsDetail(sfPost)
       details.map(sendOp).find(_.isFailure)
     }.toDisjunction match {
       case -\/(failure) => Left(SalesforceUpdateFailure(failure.toString))

diff --git a/handlers/holiday-stop-processor/src/test/scala/com/gu/holidaystopprocessor/Fixtures.scala b/handlers/holiday-stop-processor/src/test/scala/com/gu/holidaystopprocessor/Fixtures.scala
@@ -3,8 +3,8 @@ package com.gu.holidaystopprocessor
 import java.time.LocalDate
 
 import com.gu.salesforce.SalesforceAuthenticate.SFAuthConfig
-import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest.{HolidayStopRequest, HolidayStopRequestActionedCount, HolidayStopRequestEndDate, HolidayStopRequestId, HolidayStopRequestStartDate}
-import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{HolidayStopRequestsDetail, HolidayStopRequestsDetailChargeCode, HolidayStopRequestsDetailId, ProductName, StoppedPublicationDate, SubscriptionName}
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest.{HolidayStopRequest, HolidayStopRequestActionedCount, HolidayStopRequestEndDate, HolidayStopRequestStartDate}
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{HolidayStopRequestId, HolidayStopRequestsDetail, HolidayStopRequestsDetailChargeCode, HolidayStopRequestsDetailId, ProductName, StoppedPublicationDate, SubscriptionName}
 import com.gu.util.Time
 
 object Fixtures {

diff --git a/...y-stops/src/main/scala/com/gu/salesforce/holiday_stops/SalesforceHolidayStopRequest.scala b/...y-stops/src/main/scala/com/gu/salesforce/holiday_stops/SalesforceHolidayStopRequest.scala
@@ -2,7 +2,7 @@ package com.gu.salesforce.holiday_stops
 
 import ai.x.play.json.Jsonx
 import com.gu.salesforce.SalesforceConstants._
-import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{HolidayStopRequestsDetailSearchQueryResponse, ProductName, SubscriptionName}
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{HolidayStopRequestId, HolidayStopRequestsDetailSearchQueryResponse, ProductName, SubscriptionName}
 import com.gu.util.Logging
 import com.gu.util.resthttp.RestOp._
 import com.gu.util.resthttp.RestRequestMaker._
@@ -26,9 +26,6 @@ object SalesforceHolidayStopRequest extends Logging {
     override def writes(date: LocalDate): JsValue = JsString(date.toString(SALESFORCE_DATE_FORMAT))
   }
 
-  case class HolidayStopRequestId(value: String) extends AnyVal
-  implicit val formatHolidayStopRequestId = Jsonx.formatInline[HolidayStopRequestId]
-
   case class HolidayStopRequestStartDate(value: LocalDate) extends AnyVal
   implicit val formatHolidayStopRequestStartDate = Jsonx.formatInline[HolidayStopRequestStartDate]
 
@@ -81,17 +78,13 @@ object SalesforceHolidayStopRequest extends Logging {
 
   }
 
-  object LookupByDateRangeAndProductNamePrefix {
+  object LookupByProductNamePrefix {
 
-    def apply(sfGet: HttpOp[RestRequestMaker.GetRequestWithParams, JsValue]): (LocalDate, LocalDate, ProductName) => ClientFailableOp[List[HolidayStopRequest]] =
-      sfGet.setupRequestMultiArg(toRequest _).parse[HolidayStopRequestSearchQueryResponse].map(_.records).runRequestMultiArg
+    def apply(sfGet: HttpOp[RestRequestMaker.GetRequestWithParams, JsValue]): ProductName => ClientFailableOp[List[HolidayStopRequest]] =
+      sfGet.setupRequest(toRequest).parse[HolidayStopRequestSearchQueryResponse].map(_.records).runRequest
 
-    def toRequest(start: LocalDate, end: LocalDate, productNamePrefix: ProductName) = {
-      val sfStart = start.toString(SALESFORCE_DATE_FORMAT)
-      val sfEnd = end.toString(SALESFORCE_DATE_FORMAT)
-      val soqlQuery = getHolidayStopRequestPrefixSOQL(productNamePrefix) +
-        s"AND Start_Date__c >= $sfStart " +
-        s"AND Start_Date__c <= $sfEnd "
+    def toRequest(productNamePrefix: ProductName) = {
+      val soqlQuery = getHolidayStopRequestPrefixSOQL(productNamePrefix)
       logger.info(s"using SF query : $soqlQuery")
       RestRequestMaker.GetRequestWithParams(RelativePath(soqlQueryBaseUrl), UrlParams(Map("q" -> soqlQuery)))
     }

diff --git a/.../src/main/scala/com/gu/salesforce/holiday_stops/SalesforceHolidayStopRequestsDetail.scala b/.../src/main/scala/com/gu/salesforce/holiday_stops/SalesforceHolidayStopRequestsDetail.scala
@@ -107,23 +107,21 @@ object SalesforceHolidayStopRequestsDetail extends Logging {
     }
   }
 
-  object LookupActionedByProductNamePrefixAndDateRange {
+  case class ActionedHolidayStopRequestsDetailToBackfill(
+    Holiday_Stop_Request__c: HolidayStopRequestId,
+    Stopped_Publication_Date__c: StoppedPublicationDate,
+    Estimated_Price__c: Option[HolidayStopRequestsDetailChargePrice],
+    Charge_Code__c: Option[HolidayStopRequestsDetailChargeCode],
+    Actual_Price__c: Option[HolidayStopRequestsDetailChargePrice]
+  )
+  implicit val writesHolidayStopRequestsDetail = Json.writes[ActionedHolidayStopRequestsDetailToBackfill]
 
-    def apply(sfGet: HttpOp[RestRequestMaker.GetRequestWithParams, JsValue]): (ProductName, LocalDate, LocalDate) => ClientFailableOp[List[HolidayStopRequestsDetail]] =
-      sfGet.setupRequestMultiArg(toRequest _).parse[HolidayStopRequestsDetailSearchQueryResponse].map(_.records).runRequestMultiArg
+  object BackfillActionedSalesforceHolidayStopRequestsDetail {
+
+    def apply(sfPost: HttpOp[RestRequestMaker.PostRequest, JsValue]): ActionedHolidayStopRequestsDetailToBackfill => ClientFailableOp[JsValue] =
+      sfPost.setupRequest[ActionedHolidayStopRequestsDetailToBackfill] { toCreate =>
+        PostRequest(toCreate, RelativePath(sfObjectsBaseUrl + holidayStopRequestsDetailSfObjectRef))
+      }.parse[JsValue].runRequest
 
-    def toRequest(productNamePrefix: ProductName, startThreshold: LocalDate, endThreshold: LocalDate): GetRequestWithParams = {
-      val soqlQuery = s"""
-          | $SOQL_SELECT_CLAUSE
-          | FROM $holidayStopRequestsDetailSfObjectRef
-          | WHERE Product_Name__c LIKE '${productNamePrefix.value}%'
-          | AND Stopped_Publication_Date__c >= ${startThreshold.toString}
-          | AND Stopped_Publication_Date__c <= ${endThreshold.toString}
-          | AND Is_Actioned__c = true
-          | $SOQL_ORDER_BY_CLAUSE
-          |""".stripMargin
-      logger.info(s"using SF query : $soqlQuery")
-      RestRequestMaker.GetRequestWithParams(RelativePath(soqlQueryBaseUrl), UrlParams(Map("q" -> soqlQuery)))
-    }
   }
 }
diff --git a/lib/holiday-stops/src/test/scala/com/gu/holiday_stops/ActionCalculatorTest.scala b/lib/holiday-stops/src/test/scala/com/gu/holiday_stops/ActionCalculatorTest.scala
@@ -1,7 +1,7 @@
 package com.gu.holiday_stops
 
 import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest._
-import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{ProductName, SubscriptionName}
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestsDetail.{HolidayStopRequestId, ProductName, SubscriptionName}
 import org.joda.time.{DateTimeConstants, LocalDate}
 import org.scalatest.{FlatSpec, Matchers}
 

diff --git a/...ala/com/gu/salesforce/holiday_stops/SalesforceHolidayStopRequestEndToEndEffectsTest.scala b/...ala/com/gu/salesforce/holiday_stops/SalesforceHolidayStopRequestEndToEndEffectsTest.scala
@@ -16,7 +16,7 @@ import scalaz.{-\/, \/-}
 class SalesforceHolidayStopRequestEndToEndEffectsTest extends FlatSpec with Matchers {
 
   case class EndToEndResults(
-    createResult: SalesforceHolidayStopRequest.HolidayStopRequestId,
+    createResult: HolidayStopRequestId,
     preProcessingFetchResult: List[SalesforceHolidayStopRequest.HolidayStopRequest],
     postProcessingFetchResult: List[SalesforceHolidayStopRequest.HolidayStopRequest],
     deleteResult: String