Make ingest model compliant with RFC (#2942)

* [skip ci] use sourceLocation instead of uploadUrl in DisplayIngest

* [skip ci] Callback has a status object. Type of DisplayStatus is Status

* [skip ci] Modify resources to be bag

* [skip ci] Update RFC

* Fixing compile/test errors due to change in the progress model

* Change one more reference to sourceCallback

* Provider has no label

* Apply auto-formatting rules

* Apply auto-formatting rules

* Remove some more provider labels

* Apply auto-formatting rules

archive/archivist/src/main/scala/uk/ac/wellcome/platform/archive/archivist/flow/ArchiveZipFileFlow.scala

@@ -77,13 +77,13 @@ object ArchiveZipFileFlow extends Logging {
         ProgressStatusUpdate(
           ingestBagRequest.archiveRequestId,
           Progress.Failed,
-          Nil,
+          None,
           errors.map(error => ProgressEvent(error.toString)))
       case Left(archiveError) =>
         ProgressStatusUpdate(
           ingestBagRequest.archiveRequestId,
           Progress.Failed,
-          Nil,
+          None,
           List(ProgressEvent(archiveError.toString)))
     }
 }

archive/archivist/src/main/scala/uk/ac/wellcome/platform/archive/archivist/flow/ZipFileDownloadFlow.scala

@@ -101,7 +101,7 @@ object ZipFileDownloadFlow extends Logging {
         ProgressStatusUpdate(
           archiveError.t.archiveRequestId,
           Progress.Failed,
-          Nil,
+          None,
           List(ProgressEvent(archiveError.toString))
         )
     }

archive/archivist/src/test/scala/uk/ac/wellcome/platform/archive/archivist/ArchivistFeatureTest.scala

@@ -103,7 +103,7 @@ class ArchivistFeatureTest
                 request.archiveRequestId,
                 progressTopic,
                 Progress.Failed,
-                Nil)({ events =>
+                None)({ events =>
                 all(events.map(_.description)) should include regex "Calculated checksum .+ was different from bad_digest"
               })
             }
@@ -171,15 +171,15 @@ class ArchivistFeatureTest
                             invalidRequest1.archiveRequestId,
                             progressTopic,
                             Progress.Failed,
-                            Nil) { events =>
+                            None) { events =>
                             all(events.map(_.description)) should include regex "Calculated checksum .+ was different from bad_digest"
                           }
 
                           assertTopicReceivesProgressStatusUpdate(
                             invalidRequest2.archiveRequestId,
                             progressTopic,
                             Progress.Failed,
-                            Nil) { events =>
+                            None) { events =>
                             all(events.map(_.description)) should include regex "Calculated checksum .+ was different from bad_digest"
                           }
 
@@ -260,7 +260,7 @@ class ArchivistFeatureTest
                     invalidRequestId1,
                     progressTopic,
                     Progress.Failed,
-                    Nil) { events =>
+                    None) { events =>
                     events should have size 1
                     events.head.description should startWith(
                       s"Failed downloading zipFile ${ingestBucket.name}/non-existing1.zip")
@@ -270,7 +270,7 @@ class ArchivistFeatureTest
                     invalidRequestId2,
                     progressTopic,
                     Progress.Failed,
-                    Nil) { events =>
+                    None) { events =>
                     events should have size 1
                     events.head.description should startWith(
                       s"Failed downloading zipFile ${ingestBucket.name}/non-existing2.zip")
@@ -342,7 +342,7 @@ class ArchivistFeatureTest
                             invalidRequest1.archiveRequestId,
                             progressTopic,
                             Progress.Failed,
-                            Nil) { events =>
+                            None) { events =>
                             events should have size 1
                             events.head.description shouldBe "Failed reading file this/does/not/exists.jpg from zip file"
                           }
@@ -351,7 +351,7 @@ class ArchivistFeatureTest
                             invalidRequest2.archiveRequestId,
                             progressTopic,
                             Progress.Failed,
-                            Nil) { events =>
+                            None) { events =>
                             events should have size 1
                             events.head.description shouldBe "Failed reading file this/does/not/exists.jpg from zip file"
                           }
@@ -423,7 +423,7 @@ class ArchivistFeatureTest
                             invalidRequest1.archiveRequestId,
                             progressTopic,
                             Progress.Failed,
-                            Nil) { events =>
+                            None) { events =>
                             events should have size 1
                             events.head.description shouldBe "Failed reading file bag-info.txt from zip file"
                           }
@@ -432,7 +432,7 @@ class ArchivistFeatureTest
                             invalidRequest2.archiveRequestId,
                             progressTopic,
                             Progress.Failed,
-                            Nil) { events =>
+                            None) { events =>
                             events should have size 1
                             events.head.description shouldBe "Failed reading file bag-info.txt from zip file"
                           }

archive/archivist/src/test/scala/uk/ac/wellcome/platform/archive/archivist/flow/ArchiveZipFileFlowTest.scala

@@ -122,7 +122,7 @@ class ArchiveZipFileFlowTest
                     ingestContext.archiveRequestId,
                     reportingTopic,
                     Progress.Failed,
-                    Nil) { events =>
+                    None) { events =>
                     events should have size (zipFile.entries().asScala.size - 1)
                     all(events.map(_.description)) should include regex "Calculated checksum .+ was different from bad_digest"
                   }
@@ -163,7 +163,7 @@ class ArchiveZipFileFlowTest
                     ingestContext.archiveRequestId,
                     reportingTopic,
                     Progress.Failed,
-                    Nil) { events =>
+                    None) { events =>
                     inside(events) {
                       case List(event) =>
                         event.description shouldBe result.head.left.get.toString
@@ -218,7 +218,7 @@ class ArchiveZipFileFlowTest
                     ingestContext.archiveRequestId,
                     reportingTopic,
                     Progress.Failed,
-                    Nil) { events =>
+                    None) { events =>
                     inside(events) {
                       case List(event) =>
                         event.description shouldBe result.head.left.get.toString

archive/archivist/src/test/scala/uk/ac/wellcome/platform/archive/archivist/flow/ZipFileDownloadFlowTest.scala

@@ -105,7 +105,7 @@ class ZipFileDownloadFlowTest
             ingestBagRequest.archiveRequestId,
             progressTopic,
             Progress.Failed,
-            Nil) { events =>
+            None) { events =>
             events should have size 1
             events.head.description should startWith(
               s"Failed downloading zipFile ${objectLocation.namespace}/${objectLocation.key}")

archive/common/src/main/scala/uk/ac/wellcome/platform/archive/common/models/DisplayIngest.scala

@@ -6,13 +6,12 @@ import io.circe.generic.extras.JsonKey
 import uk.ac.wellcome.platform.archive.common.progress.models.{
   Callback,
   Progress,
-  ProgressEvent,
-  Resource
+  ProgressEvent
 }
 
 sealed trait DisplayIngest
 
-case class RequestDisplayIngest(uploadUrl: String,
+case class RequestDisplayIngest(sourceLocation: DisplayLocation,
                                 callback: Option[DisplayCallback],
                                 ingestType: DisplayIngestType,
                                 space: DisplayStorageSpace,
@@ -21,88 +20,82 @@ case class RequestDisplayIngest(uploadUrl: String,
     extends DisplayIngest
 
 case class ResponseDisplayIngest(id: UUID,
-                                 uploadUrl: String,
+                                 sourceLocation: DisplayLocation,
                                  callback: Option[DisplayCallback],
                                  ingestType: DisplayIngestType,
                                  space: DisplayStorageSpace,
-                                 status: DisplayIngestStatus,
-                                 resources: Seq[DisplayIngestResource] =
-                                   Seq.empty,
+                                 status: DisplayStatus,
+                                 bag: Option[IngestDisplayBag] = None,
                                  events: Seq[DisplayProgressEvent] = Seq.empty,
                                  createdDate: String,
                                  lastModifiedDate: String,
                                  @JsonKey("type")
                                  ontologyType: String = "Ingest")
     extends DisplayIngest
 
-case class DisplayCallback(uri: String,
-                           status: Option[String],
+case class IngestDisplayBag(id: String,
+                            @JsonKey("type")
+                            ontologyType: String = "Bag")
+
+case class DisplayCallback(url: String,
+                           status: Option[DisplayStatus],
                            @JsonKey("type")
                            ontologyType: String = "Callback")
 
 case class DisplayIngestType(id: String = "create",
                              @JsonKey("type")
                              ontologyType: String = "IngestType")
 
-case class DisplayIngestResource(id: String,
-                                 @JsonKey("type")
-                                 ontologyType: String = "IngestResource")
-
 case class DisplayStorageSpace(id: String,
                                @JsonKey("type")
                                ontologyType: String = "Space")
 
-case class DisplayIngestStatus(id: String,
-                               @JsonKey("type")
-                               ontologyType: String = "IngestStatus")
+case class DisplayStatus(id: String,
+                         @JsonKey("type")
+                         ontologyType: String = "Status")
 
 case class DisplayProgressEvent(description: String,
                                 createdDate: String,
                                 @JsonKey("type")
                                 ontologyType: String = "ProgressEvent")
 
 case object ResponseDisplayIngest {
-  def apply(progress: Progress): ResponseDisplayIngest = {
-    ResponseDisplayIngest(
-      id = progress.id,
-      uploadUrl = progress.uploadUri.toString,
-      callback = progress.callback.map(DisplayCallback(_)),
-      space = DisplayStorageSpace(progress.space.toString),
-      ingestType = DisplayIngestType(),
-      resources = progress.resources.map(DisplayIngestResource(_)),
-      status = DisplayIngestStatus(progress.status),
-      events = progress.events.map(DisplayProgressEvent(_)),
-      createdDate = progress.createdDate.toString,
-      lastModifiedDate = progress.lastModifiedDate.toString
-    )
-  }
+  def apply(progress: Progress): ResponseDisplayIngest = ResponseDisplayIngest(
+    id = progress.id,
+    sourceLocation = DisplayLocation(progress.sourceLocation),
+    callback = progress.callback.map(DisplayCallback(_)),
+    space = DisplayStorageSpace(progress.space.toString),
+    ingestType = DisplayIngestType(),
+    bag = progress.bag.map(IngestDisplayBag(_)),
+    status = DisplayStatus(progress.status),
+    events = progress.events.map(DisplayProgressEvent(_)),
+    createdDate = progress.createdDate.toString,
+    lastModifiedDate = progress.lastModifiedDate.toString
+  )
 }
 
 case object DisplayProgressEvent {
-  def apply(progressEvent: ProgressEvent): DisplayProgressEvent = {
+  def apply(progressEvent: ProgressEvent): DisplayProgressEvent =
     DisplayProgressEvent(
       progressEvent.description,
       progressEvent.createdDate.toString)
-  }
 }
 
-case object DisplayIngestStatus {
-  def apply(progressStatus: Progress.Status): DisplayIngestStatus = {
-    DisplayIngestStatus(progressStatus.toString)
-  }
-}
+case object DisplayStatus {
+  def apply(progressStatus: Progress.Status): DisplayStatus =
+    DisplayStatus(progressStatus.toString)
 
-case object DisplayIngestResource {
-  def apply(resource: Resource): DisplayIngestResource = {
-    DisplayIngestResource(resource.id.underlying)
-  }
+  def apply(callbackStatus: Callback.CallbackStatus): DisplayStatus =
+    DisplayStatus(callbackStatus.toString)
 }
 
 case object DisplayCallback {
-  def apply(callback: Callback): DisplayCallback = {
-    DisplayCallback(
-      callback.uri.toString,
-      Some(callback.status.toString)
-    )
-  }
+  def apply(callback: Callback): DisplayCallback = DisplayCallback(
+    callback.uri.toString,
+    Some(DisplayStatus(callback.status))
+  )
+}
+
+object IngestDisplayBag {
+  def apply(bagId: BagId): IngestDisplayBag = IngestDisplayBag(bagId.toString)
 }

archive/registrar_http/src/main/scala/uk/ac/wellcome/platform/archive/registrar/http/models/DisplayLocation.scala → archive/common/src/main/scala/uk/ac/wellcome/platform/archive/common/models/DisplayLocation.scala

@@ -1,13 +1,14 @@
-package uk.ac.wellcome.platform.archive.registrar.http.models
+package uk.ac.wellcome.platform.archive.common.models
+
 import io.circe.generic.extras.JsonKey
-import uk.ac.wellcome.platform.archive.registrar.common.models.Location
+import uk.ac.wellcome.platform.archive.common.progress.models.StorageLocation
 
 case class DisplayLocation(provider: DisplayProvider,
                            bucket: String,
                            path: String,
                            @JsonKey("type") ontologyType: String = "Location")
 object DisplayLocation {
-  def apply(location: Location): DisplayLocation =
+  def apply(location: StorageLocation): DisplayLocation =
     DisplayLocation(
       DisplayProvider(location.provider),
       location.location.namespace,

archive/common/src/main/scala/uk/ac/wellcome/platform/archive/common/models/DisplayProvider.scala

@@ -0,0 +1,11 @@
+package uk.ac.wellcome.platform.archive.common.models
+
+import io.circe.generic.extras.JsonKey
+import uk.ac.wellcome.platform.archive.common.progress.models.StorageProvider
+
+case class DisplayProvider(id: String,
+                           @JsonKey("type") ontologyType: String = "Provider")
+object DisplayProvider {
+  def apply(provider: StorageProvider): DisplayProvider =
+    DisplayProvider(id = provider.id)
+}

archive/common/src/main/scala/uk/ac/wellcome/platform/archive/common/progress/models/Progress.scala

@@ -5,14 +5,17 @@ import java.time.Instant
 import java.util.UUID
 
 import uk.ac.wellcome.platform.archive.common.json.URIConverters
-import uk.ac.wellcome.platform.archive.common.models.RequestDisplayIngest
+import uk.ac.wellcome.platform.archive.common.models.{
+  BagId,
+  RequestDisplayIngest
+}
 
 case class Progress(id: UUID,
-                    uploadUri: URI,
+                    sourceLocation: StorageLocation,
                     space: Namespace,
                     callback: Option[Callback] = None,
                     status: Progress.Status = Progress.Initialised,
-                    resources: Seq[Resource] = Seq.empty,
+                    bag: Option[BagId] = None,
                     createdDate: Instant = Instant.now,
                     lastModifiedDate: Instant = Instant.now,
                     events: Seq[ProgressEvent] = Seq.empty)
@@ -44,9 +47,9 @@ case object Progress extends URIConverters {
   def apply(createRequest: RequestDisplayIngest): Progress = {
     Progress(
       id = generateId,
-      uploadUri = URI.create(createRequest.uploadUrl),
+      sourceLocation = StorageLocation(createRequest.sourceLocation),
       callback = Callback(createRequest.callback.map(displayCallback =>
-        URI.create(displayCallback.uri))),
+        URI.create(displayCallback.url))),
       space = Namespace(createRequest.space.id),
       status = Progress.Initialised
     )

archive/common/src/main/scala/uk/ac/wellcome/platform/archive/common/progress/models/ProgressUpdate.scala

@@ -2,6 +2,8 @@ package uk.ac.wellcome.platform.archive.common.progress.models
 
 import java.util.UUID
 
+import uk.ac.wellcome.platform.archive.common.models.BagId
+
 sealed trait ProgressUpdate {
   val id: UUID
   val events: Seq[ProgressEvent]
@@ -12,7 +14,7 @@ case class ProgressEventUpdate(id: UUID, events: Seq[ProgressEvent])
 
 case class ProgressStatusUpdate(id: UUID,
                                 status: Progress.Status,
-                                affectedResources: Seq[Resource],
+                                affectedBag: Option[BagId],
                                 events: Seq[ProgressEvent] = List.empty)
     extends ProgressUpdate
 
@@ -22,9 +24,3 @@ case class ProgressCallbackStatusUpdate(id: UUID,
     extends ProgressUpdate
 
 case class FailedProgressUpdate(e: Throwable, update: ProgressUpdate)
-
-case class Resource(id: ResourceIdentifier)
-
-case class ResourceIdentifier(underlying: String) extends AnyVal {
-  override def toString: String = underlying
-}

archive/common/src/main/scala/uk/ac/wellcome/platform/archive/common/progress/models/StorageLocation.scala

@@ -0,0 +1,20 @@
+package uk.ac.wellcome.platform.archive.common.progress.models
+import uk.ac.wellcome.platform.archive.common.models.{
+  DisplayLocation,
+  DisplayProvider
+}
+import uk.ac.wellcome.storage.ObjectLocation
+
+case class StorageLocation(provider: StorageProvider, location: ObjectLocation)
+object StorageLocation {
+  def apply(displayLocation: DisplayLocation): StorageLocation =
+    StorageLocation(
+      StorageProvider(displayLocation.provider),
+      ObjectLocation(displayLocation.bucket, displayLocation.path))
+}
+
+case class StorageProvider(id: String)
+object StorageProvider {
+  def apply(displayProvider: DisplayProvider): StorageProvider =
+    StorageProvider(displayProvider.id)
+}