/
MeetupReceiver.scala
98 lines (78 loc) · 2.5 KB
/
MeetupReceiver.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
/**
*
*/
package receiver
import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.storage.StorageLevel
import org.apache.spark.Logging
import com.ning.http.client.AsyncHttpClientConfig
import com.ning.http.client._
import scala.collection.mutable.ArrayBuffer
import java.io.OutputStream
import java.io.ByteArrayInputStream
import java.io.InputStreamReader
import java.io.BufferedReader
import java.io.InputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream
/**
* @author szelvenskiy
*
*/
class MeetupReceiver(url: String) extends Receiver[String](StorageLevel.MEMORY_AND_DISK_2) with Logging {
@transient var client: AsyncHttpClient = _
@transient var inputPipe: PipedInputStream = _
@transient var outputPipe: PipedOutputStream = _
def onStart() {
val cf = new AsyncHttpClientConfig.Builder()
cf.setRequestTimeout(Integer.MAX_VALUE)
cf.setReadTimeout(Integer.MAX_VALUE)
cf.setPooledConnectionIdleTimeout(Integer.MAX_VALUE)
client= new AsyncHttpClient(cf.build())
inputPipe = new PipedInputStream(1024 * 1024)
outputPipe = new PipedOutputStream(inputPipe)
val producerThread = new Thread(new DataConsumer(inputPipe))
producerThread.start()
client.prepareGet(url).execute(new AsyncHandler[Unit]{
def onBodyPartReceived(bodyPart: HttpResponseBodyPart) = {
bodyPart.writeTo(outputPipe)
AsyncHandler.STATE.CONTINUE
}
def onStatusReceived(status: HttpResponseStatus) = {
AsyncHandler.STATE.CONTINUE
}
def onHeadersReceived(headers: HttpResponseHeaders) = {
AsyncHandler.STATE.CONTINUE
}
def onCompleted = {
println("completed")
}
def onThrowable(t: Throwable)={
t.printStackTrace()
}
})
}
def onStop() {
if (Option(client).isDefined) client.close()
if (Option(outputPipe).isDefined) {
outputPipe.flush()
outputPipe.close()
}
if (Option(inputPipe).isDefined) {
inputPipe.close()
}
}
class DataConsumer(inputStream: InputStream) extends Runnable
{
override
def run()
{
val bufferedReader = new BufferedReader( new InputStreamReader( inputStream ))
var input=bufferedReader.readLine()
while(input!=null){
store(input)
input=bufferedReader.readLine()
}
}
}
}