-
Notifications
You must be signed in to change notification settings - Fork 40
/
Pipeline.scala
66 lines (54 loc) · 2.09 KB
/
Pipeline.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon. All Rights Reserved.
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//: http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//:
//: ----------------------------------------------------------------------------
package nelson
import cats.effect.IO
import nelson.CatsHelpers._
import cats.syntax.applicativeError._
import fs2.{Sink, Stream}
import journal.Logger
import scala.util.control.NonFatal
object Pipeline {
import Manifest.Action
val log = Logger[Pipeline.type]
object sinks {
/**
* Actually execute the workflow specified by the actionable on the stream.
*/
def runAction(cfg: NelsonConfig): Sink[IO, Action] = {
Sink { action =>
action.run((cfg, action.config)).recoverWith {
case NonFatal(e) =>
e.printStackTrace
IO(log.warn(s"unexpected error occoured whilst running the workflow: ${e.getMessage}, cause: ${e.getCause}"))
}
}
}
} // e/o sinks
/**
* Wire the stream of actionable instances to the
* effect generating sinks and observations.
*/
def task(config: NelsonConfig)(effects: Sink[IO, Action]): IO[Unit] = {
def par[A](ps: Stream[IO, Stream[IO, A]]): Stream[IO, A] = {
implicit val ec = config.pools.defaultExecutor
val withErrors = ps.join(config.pipeline.concurrencyLimit).attempt
withErrors.observeW(config.auditor.errorSink).stripW
}
val p: Stream[IO, Stream[IO, Unit]] = config.queue.dequeue.map(a => Stream.emit(a).covary[IO].to(effects))
par(p).compile.drain
}
}