This repository has been archived by the owner on May 3, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 404
/
ScalaSpeedModelManager.scala
56 lines (48 loc) · 1.72 KB
/
ScalaSpeedModelManager.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
/*
* Copyright (c) 2014, Cloudera, Inc. All Rights Reserved.
*
* Cloudera, Inc. licenses this file to you under the Apache License,
* Version 2.0 (the "License"). You may not use this file except in
* compliance with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for
* the specific language governing permissions and limitations under the
* License.
*/
package com.cloudera.oryx.api.speed
import org.apache.hadoop.conf.Configuration
import org.apache.spark.rdd.RDD
import com.cloudera.oryx.api.KeyMessage
/**
* Scala counterpart to Java SpeedModelManager.
*
* @tparam K type of key read from input topic
* @tparam M type of message read from input topic
* @tparam U type of update message read/written
* @since 2.0.0
*/
trait ScalaSpeedModelManager[K,M,U] {
/**
* Called by the framework to initiate a continuous process of reading models, and reading
* from the input topic and updating model state in memory, and issuing updates to the
* update topic. This will be executed asynchronously and may block.
*
* @param updateIterator iterator to read models from
* @param hadoopConf Hadoop context, which may be required for reading from HDFS
* @since 2.0.0
*/
def consume(updateIterator: Iterator[KeyMessage[String,U]], hadoopConf: Configuration): Unit
/**
* @param newData RDD of raw new data from the topic
* @return updates to publish on the update topic
* @since 2.0.0
*/
def buildUpdates(newData: RDD[(K,M)]): Iterable[U]
/**
* @since 2.0.0
*/
def close(): Unit = {}
}