Backfill Salesforce with legacy holiday stops

Holiday stops found in Zuora and not already in Salesforce
are written to Salesforce.

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/AccessToken.scala

@@ -0,0 +1,16 @@
+package com.gu.holidaystopbackfill
+
+import com.softwaremill.sttp.Response
+import io.circe.generic.auto._
+import io.circe.parser.decode
+
+case class AccessToken(access_token: String)
+
+object AccessToken {
+
+  def fromZuoraResponse(response: Response[String]): Either[ZuoraFetchFailure, AccessToken] =
+    for {
+      body <- response.body.left.map(ZuoraFetchFailure)
+      token <- decode[AccessToken](body).left.map(e => ZuoraFetchFailure(e.getMessage))
+    } yield token
+}

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/BackfillFailure.scala

@@ -0,0 +1,13 @@
+package com.gu.holidaystopbackfill
+
+sealed trait BackfillFailure {
+  def reason: String
+}
+
+case class ConfigFailure(reason: String) extends BackfillFailure
+
+case class ZuoraFetchFailure(reason: String) extends BackfillFailure
+
+case class SalesforceFetchFailure(reason: String) extends BackfillFailure
+
+case class SalesforceUpdateFailure(reason: String) extends BackfillFailure

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Backfiller.scala

@@ -0,0 +1,30 @@
+package com.gu.holidaystopbackfill
+
+import java.time.LocalDate
+
+import com.gu.holidaystopbackfill.AccessToken.fromZuoraResponse
+import com.gu.holidaystopbackfill.SalesforceHolidayStop.holidayStopsAlreadyInSalesforce
+import com.gu.holidaystopbackfill.ZuoraHolidayStop.holidayStopsAlreadyInZuora
+
+object Backfiller {
+
+  /*
+   * This makes two passes to update Salesforce to ensure it's failsafe.
+   * If any call fails it should leave Salesforce in a consistent state.
+   * First, the holiday request table is updated, and then the zuora refs child table.
+   */
+  def backfill(startThreshold: LocalDate, endThreshold: Option[LocalDate], dryRun: Boolean): Either[BackfillFailure, Unit] = {
+    for {
+      config <- Config.build()
+      accessToken <- fromZuoraResponse(Zuora.accessTokenGetResponse(config.zuoraConfig))
+      stopsInZuora1 <- holidayStopsAlreadyInZuora(Zuora.queryGetResponse(config.zuoraConfig, accessToken))(startThreshold, endThreshold)
+      stopsInSf1 <- holidayStopsAlreadyInSalesforce(config.sfConfig)(startThreshold, endThreshold)
+      requestsToAddToSf = SalesforceHolidayStop.holidayStopRequestsToBeBackfilled(stopsInZuora1, stopsInSf1)
+      _ <- SalesforceHolidayStop.holidayStopRequestsAddedToSalesforce(config.sfConfig, dryRun)(requestsToAddToSf)
+      stopsInZuora2 <- holidayStopsAlreadyInZuora(Zuora.queryGetResponse(config.zuoraConfig, accessToken))(startThreshold, endThreshold)
+      stopsInSf2 <- holidayStopsAlreadyInSalesforce(config.sfConfig)(startThreshold, endThreshold)
+      zuoraRefsToAddToSf = SalesforceHolidayStop.zuoraRefsToBeBackfilled(stopsInZuora2, stopsInSf2)
+      zuoraRefsAddedToSf <- SalesforceHolidayStop.zuoraRefsAddedToSalesforce(config.sfConfig, dryRun)(zuoraRefsToAddToSf)
+    } yield zuoraRefsAddedToSf
+  }
+}

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Config.scala

@@ -0,0 +1,64 @@
+package com.gu.holidaystopbackfill
+
+import com.amazonaws.auth.profile.ProfileCredentialsProvider
+import com.amazonaws.regions.Regions.EU_WEST_1
+import com.amazonaws.services.s3.AmazonS3Client
+import com.gu.salesforce.SalesforceAuthenticate.SFAuthConfig
+import io.circe.Decoder
+import io.circe.generic.auto._
+import io.circe.parser.decode
+
+import scala.io.Source
+
+case class Config(
+  zuoraConfig: ZuoraConfig,
+  sfConfig: SFAuthConfig
+)
+
+case class ZuoraConfig(
+  baseUrl: String,
+  holidayStopProcessor: HolidayStopProcessor
+)
+
+case class HolidayStopProcessor(oauth: Oauth)
+
+case class Oauth(clientId: String, clientSecret: String)
+
+object Config {
+
+  private def zuoraCredentials(stage: String): Either[ConfigFailure, ZuoraConfig] =
+    credentials[ZuoraConfig](stage, "zuoraRest")
+
+  private def salesforceCredentials(stage: String): Either[ConfigFailure, SFAuthConfig] =
+    credentials[SFAuthConfig](stage, "sfAuth")
+
+  private def credentials[T](stage: String, filePrefix: String)(implicit evidence: Decoder[T]): Either[ConfigFailure, T] = {
+    val profileName = "membership"
+    val bucketName = "gu-reader-revenue-private"
+    val key =
+      if (stage == "DEV")
+        s"membership/support-service-lambdas/$stage/$filePrefix-$stage.json"
+      else
+        s"membership/support-service-lambdas/$stage/$filePrefix-$stage.v1.json"
+    val builder =
+      if (stage == "DEV")
+        AmazonS3Client.builder
+          .withCredentials(new ProfileCredentialsProvider(profileName))
+          .withRegion(EU_WEST_1)
+      else AmazonS3Client.builder
+    val inputStream =
+      builder.build().getObject(bucketName, key).getObjectContent
+    val rawJson = Source.fromInputStream(inputStream).mkString
+    decode[T](rawJson).left map { e =>
+      ConfigFailure(s"Could not read secret config file from S3://$bucketName/$key: ${e.toString}")
+    }
+  }
+
+  def build(): Either[ConfigFailure, Config] = {
+    val stage = Option(System.getenv("Stage")).getOrElse("DEV")
+    for {
+      zuoraConfig <- zuoraCredentials(stage)
+      sfConfig <- salesforceCredentials(stage)
+    } yield Config(zuoraConfig, sfConfig)
+  }
+}

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/HolidayStopDetail.scala

@@ -0,0 +1,133 @@
+package com.gu.holidaystopbackfill
+
+import java.time.temporal.ChronoUnit
+import java.time.{DayOfWeek, LocalDate}
+
+import cats.implicits._
+import com.gu.salesforce.SalesforceAuthenticate.SFAuthConfig
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest.{HolidayStopRequestEndDate, HolidayStopRequestStartDate, NewHolidayStopRequest, ProductName, SubscriptionName, SubscriptionNameLookup}
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestActionedZuoraRef.{HolidayStopRequestActionedZuoraChargeCode, HolidayStopRequestActionedZuoraChargePrice, HolidayStopRequestActionedZuoraRef, HolidayStopRequestDetails, StoppedPublicationDate}
+import com.gu.util.Time
+import com.softwaremill.sttp.Response
+import io.circe.generic.auto._
+import io.circe.parser.decode
+
+case class ZuoraHolidayStop(subscriptionName: String, chargeNumber: String, startDate: LocalDate, endDate: LocalDate, creditPrice: Double)
+
+object ZuoraHolidayStop {
+
+  def holidayStopsAlreadyInZuora(queryResponse: String => Response[String])(startThreshold: LocalDate, endThreshold: Option[LocalDate]): Either[ZuoraFetchFailure, Seq[ZuoraHolidayStop]] = {
+    val response = queryResponse(Queries.preexistingHolidayStopQuery(startThreshold, endThreshold.getOrElse(LocalDate.MAX)))
+    def decodeMultiline(s: String): Either[ZuoraFetchFailure, Seq[ZuoraHolidayStop]] = {
+      val failureOrList = s.split('\n').map { line =>
+        decode[ZuoraHolidayStop](line).left.map(e => ZuoraFetchFailure(e.getMessage))
+      }.toList.sequence
+      failureOrList
+    }
+    for {
+      body <- response.body.left.map(ZuoraFetchFailure)
+      stop <- decodeMultiline(body)
+    } yield stop
+  }
+}
+
+object SalesforceHolidayStop {
+
+  def holidayStopsAlreadyInSalesforce(sfCredentials: SFAuthConfig)(startThreshold: LocalDate, endThreshold: Option[LocalDate]): Either[SalesforceFetchFailure, Seq[HolidayStopRequestDetails]] = {
+    Salesforce.holidayStopRequestDetails(sfCredentials)(ProductName("Guardian Weekly"), startThreshold, endThreshold.getOrElse(LocalDate.MAX))
+  }
+
+  def holidayStopRequestsToBeBackfilled(inZuora: Seq[ZuoraHolidayStop], inSalesforce: Seq[HolidayStopRequestDetails]): Seq[NewHolidayStopRequest] = {
+
+    val salesforceSubscriptionNames = inSalesforce.map(_.subscriptionName.value)
+
+    inZuora
+      .filterNot { zuoraStop =>
+        salesforceSubscriptionNames.contains(zuoraStop.subscriptionName)
+      }
+      .map { zuoraStop =>
+        NewHolidayStopRequest(
+          HolidayStopRequestStartDate(Time.toJodaDate(zuoraStop.startDate)),
+          HolidayStopRequestEndDate(Time.toJodaDate(zuoraStop.endDate)),
+          SubscriptionNameLookup(SubscriptionName(zuoraStop.subscriptionName))
+        )
+      }
+      .distinct
+  }
+
+  def zuoraRefsToBeBackfilled(inZuora: Seq[ZuoraHolidayStop], inSalesforce: Seq[HolidayStopRequestDetails]): Seq[HolidayStopRequestActionedZuoraRef] = {
+
+    def applicableDates(
+      fromInclusive: LocalDate,
+      toInclusive: LocalDate,
+      p: LocalDate => Boolean
+    ): List[LocalDate] = {
+      val dateRange = 0 to ChronoUnit.DAYS.between(fromInclusive, toInclusive).toInt
+      dateRange.foldLeft(List.empty[LocalDate]) { (acc, i) =>
+        val d = fromInclusive.plusDays(i)
+        if (p(d)) acc :+ d
+        else acc
+      }
+    }
+
+    /*
+     * We take legacy holiday stops that have a range of dates
+     * and we generate a new holiday stop for each stopped publication date
+     * that falls into that date range.
+     * Then we divide the credit price equally into each of the new holiday stops.
+     */
+    val stoppedPublications = inZuora.foldLeft(Seq.empty[ZuoraHolidayStop]) { (acc, stop) =>
+      val stopDates = applicableDates(stop.startDate, stop.endDate, { _.getDayOfWeek == DayOfWeek.FRIDAY })
+      val stops = stopDates map { date =>
+        ZuoraHolidayStop(stop.subscriptionName, stop.chargeNumber, date, date, stop.creditPrice / stopDates.length)
+      }
+      acc ++ stops
+    }
+
+    /*
+     * These are our criteria for determining if a legacy holiday stop in Zuora
+     * is actually the same as a Zuora ref recorded in Salesforce.
+     */
+    def isSame(z: ZuoraHolidayStop, sf: HolidayStopRequestDetails): Boolean =
+      z.subscriptionName == sf.subscriptionName.value &&
+        z.chargeNumber == sf.chargeCode.value &&
+        z.startDate == sf.stoppedPublicationDate.value
+
+    /*
+     * This map is used to find the corresponding request ID for the subscription in Salesforce.
+     * There should be a request ID available for each subscription and stopped publication date
+     * as in the first pass the parent holiday requests will have been populated.
+     */
+    val sfRequestIds = inSalesforce
+      .map { sfStop => (sfStop.subscriptionName, sfStop.stoppedPublicationDate) -> sfStop.requestId }
+      .toMap
+
+    stoppedPublications
+      .filterNot { zuoraStop => inSalesforce.exists { sfStop => isSame(zuoraStop, sfStop) } }
+      .map { stop =>
+        HolidayStopRequestActionedZuoraRef(
+          sfRequestIds((SubscriptionName(stop.subscriptionName), StoppedPublicationDate(stop.startDate))),
+          HolidayStopRequestActionedZuoraChargeCode(stop.chargeNumber),
+          HolidayStopRequestActionedZuoraChargePrice(stop.creditPrice),
+          StoppedPublicationDate(stop.startDate)
+        )
+      }
+      .distinct
+  }
+
+  def holidayStopRequestsAddedToSalesforce(sfCredentials: SFAuthConfig, dryRun: Boolean)(requests: Seq[NewHolidayStopRequest]): Either[SalesforceUpdateFailure, Unit] =
+    if (dryRun) {
+      println("++++++++++++++++++++++++++++++")
+      requests.foreach(println)
+      println("++++++++++++++++++++++++++++++")
+      Right(())
+    } else Salesforce.holidayStopCreateResponse(sfCredentials)(requests)
+
+  def zuoraRefsAddedToSalesforce(sfCredentials: SFAuthConfig, dryRun: Boolean)(zuoraRefs: Seq[HolidayStopRequestActionedZuoraRef]): Either[SalesforceUpdateFailure, Unit] =
+    if (dryRun) {
+      println("-----------------------------")
+      zuoraRefs.foreach(println)
+      println("-----------------------------")
+      Right(())
+    } else Salesforce.holidayStopUpdateResponse(sfCredentials)(zuoraRefs)
+}

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Queries.scala

@@ -0,0 +1,20 @@
+package com.gu.holidaystopbackfill
+
+import java.time.LocalDate
+
+object Queries {
+
+  def preexistingHolidayStopQuery(startThreshold: LocalDate, endThreshold: LocalDate): String = s"""
+      | select s.name subscriptionName, c.chargeNumber, c.holidaystart__c startDate,
+      |   c.holidayend__c endDate, t.price creditPrice
+      | from rateplanchargetier t
+      | join rateplancharge c on t.rateplanchargeid = c.id
+      | join rateplan p on c.rateplanid = p.id
+      | join subscription s on p.subscriptionid = s.id
+      | where c.name = 'Holiday Credit'
+      | and c.holidaystart__c >= '${startThreshold.toString}'
+      | and c.holidaystart__c <= '${endThreshold.toString}'
+      | and p.name = 'Guardian Weekly Holiday Credit'
+      | order by s.name, c.chargenumber
+      """.stripMargin
+}

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Salesforce.scala

@@ -0,0 +1,56 @@
+package com.gu.holidaystopbackfill
+
+import java.time.LocalDate
+
+import com.gu.effects.RawEffects
+import com.gu.salesforce.SalesforceAuthenticate.SFAuthConfig
+import com.gu.salesforce.SalesforceClient
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequest.{CreateHolidayStopRequest, HolidayStopRequest, NewHolidayStopRequest, ProductName}
+import com.gu.salesforce.holiday_stops.SalesforceHolidayStopRequestActionedZuoraRef.{CreateHolidayStopRequestActionedZuoraRef, HolidayStopRequestActionedZuoraRef, HolidayStopRequestDetails}
+import com.gu.salesforce.holiday_stops.{SalesforceHolidayStopRequest, SalesforceHolidayStopRequestActionedZuoraRef}
+import com.gu.util.Time
+import com.gu.util.resthttp.JsonHttp
+import scalaz.{-\/, \/-}
+
+object Salesforce {
+
+  def holidayStopRequests(sfCredentials: SFAuthConfig)(productNamePrefix: String, thresholdDate: LocalDate): Either[SalesforceFetchFailure, Seq[HolidayStopRequest]] =
+    SalesforceClient(RawEffects.response, sfCredentials).value.flatMap { sfAuth =>
+      val sfGet = sfAuth.wrapWith(JsonHttp.getWithParams)
+      val fetchOp = SalesforceHolidayStopRequest.LookupByDateAndProductNamePrefix(sfGet)
+      fetchOp(Time.toJodaDate(thresholdDate), SalesforceHolidayStopRequest.ProductName(productNamePrefix))
+    }.toDisjunction match {
+      case -\/(failure) => Left(SalesforceFetchFailure(failure.toString))
+      case \/-(requests) => Right(requests)
+    }
+
+  def holidayStopRequestDetails(sfCredentials: SFAuthConfig)(productNamePrefix: ProductName, startThreshold: LocalDate, endThreshold: LocalDate): Either[SalesforceFetchFailure, Seq[HolidayStopRequestDetails]] =
+    SalesforceClient(RawEffects.response, sfCredentials).value.flatMap { sfAuth =>
+      val sfGet = sfAuth.wrapWith(JsonHttp.getWithParams)
+      val fetchOp = SalesforceHolidayStopRequestActionedZuoraRef.LookupByProductNamePrefixAndDateRange(sfGet)
+      fetchOp(productNamePrefix, startThreshold, endThreshold)
+    }.toDisjunction match {
+      case -\/(failure) => Left(SalesforceFetchFailure(failure.toString))
+      case \/-(details) => Right(details)
+    }
+
+  def holidayStopUpdateResponse(sfCredentials: SFAuthConfig)(zuoraRefs: Seq[HolidayStopRequestActionedZuoraRef]): Either[SalesforceUpdateFailure, Unit] =
+    SalesforceClient(RawEffects.response, sfCredentials).value.map { sfAuth =>
+      val sfGet = sfAuth.wrapWith(JsonHttp.post)
+      val sendOp = CreateHolidayStopRequestActionedZuoraRef(sfGet)
+      zuoraRefs.map(sendOp).find(_.isFailure)
+    }.toDisjunction match {
+      case -\/(failure) => Left(SalesforceUpdateFailure(failure.toString))
+      case _ => Right(())
+    }
+
+  def holidayStopCreateResponse(sfCredentials: SFAuthConfig)(requests: Seq[NewHolidayStopRequest]): Either[SalesforceUpdateFailure, Unit] =
+    SalesforceClient(RawEffects.response, sfCredentials).value.map { sfAuth =>
+      val sfGet = sfAuth.wrapWith(JsonHttp.post)
+      val createOp = CreateHolidayStopRequest(sfGet)
+      requests.map(createOp).find(_.isFailure)
+    }.toDisjunction match {
+      case -\/(failure) => Left(SalesforceUpdateFailure(failure.toString))
+      case _ => Right(())
+    }
+}

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/StandaloneApp.scala

@@ -0,0 +1,11 @@
+package com.gu.holidaystopbackfill
+
+import java.time.LocalDate
+
+// This is backfill app to be run from a dev machine.
+object StandaloneApp extends App {
+  Backfiller.backfill(LocalDate.of(2017, 4, 1), Some(LocalDate.of(2021, 6, 1)), dryRun = false) match {
+    case Left(failure) => println(s"Failed: $failure")
+    case Right(_) => println("Success!")
+  }
+}

handlers/holiday-stop-processor/src/main/scala/com/gu/holidaystopbackfill/Zuora.scala

@@ -0,0 +1,38 @@
+package com.gu.holidaystopbackfill
+
+import com.softwaremill.sttp._
+import com.softwaremill.sttp.circe._
+import io.circe.generic.auto._
+
+object Zuora {
+
+  implicit val backend: SttpBackend[Id, Nothing] = HttpURLConnectionBackend()
+
+  private def baseUrl(config: ZuoraConfig): String = config.baseUrl.stripSuffix("/v1")
+
+  def accessTokenGetResponse(config: ZuoraConfig): Response[String] = {
+    val request = sttp.post(uri"${baseUrl(config)}/oauth/token")
+      .body(
+        "grant_type" -> "client_credentials",
+        "client_id" -> s"${config.holidayStopProcessor.oauth.clientId}",
+        "client_secret" -> s"${config.holidayStopProcessor.oauth.clientSecret}"
+      )
+    val response = request.send()
+    response
+  }
+
+  def queryGetResponse(config: ZuoraConfig, accessToken: AccessToken)(sql: String): Response[String] = {
+    case class Output(target: String)
+    case class Query(query: String, outputFormat: String, compression: String, output: Output)
+    val request = sttp.post(uri"${baseUrl(config)}/query/jobs")
+      .header("Authorization", s"Bearer ${accessToken.access_token}")
+      .body(Query(
+        query = sql,
+        outputFormat = "JSON",
+        compression = "NONE",
+        output = Output(target = "API_RESPONSE")
+      ))
+    val response = request.send()
+    response
+  }
+}