Skip to content
Newer
Older
100644 48 lines (41 sloc) 2.17 KB
2b93087 Custom Flume source and Flume configuration
Jon Natkins authored Aug 29, 2012
1 # Licensed to the Apache Software Foundation (ASF) under one
2 # or more contributor license agreements. See the NOTICE file
3 # distributed with this work for additional information
4 # regarding copyright ownership. The ASF licenses this file
5 # to you under the Apache License, Version 2.0 (the
6 # "License"); you may not use this file except in compliance
7 # with the License. You may obtain a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing,
12 # software distributed under the License is distributed on an
13 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14 # KIND, either express or implied. See the License for the
15 # specific language governing permissions and limitations
16 # under the License.
17
18
19 # The configuration file needs to define the sources,
20 # the channels and the sinks.
21 # Sources, channels and sinks are defined per agent,
22 # in this case called 'TwitterAgent'
23
24 TwitterAgent.sources = Twitter
25 TwitterAgent.channels = MemChannel
26 TwitterAgent.sinks = HDFS
27
28 TwitterAgent.sources.Twitter.type = com.cloudera.flume.source.TwitterSource
b7f2a3d Channel name was wrong in flume.conf
Jonathan Natkins authored Aug 30, 2012
29 TwitterAgent.sources.Twitter.channels = MemChannel
30 TwitterAgent.sources.Twitter.consumerKey = <required>
2b93087 Custom Flume source and Flume configuration
Jon Natkins authored Aug 29, 2012
31 TwitterAgent.sources.Twitter.consumerSecret = <required>
32 TwitterAgent.sources.Twitter.accessToken = <required>
33 TwitterAgent.sources.Twitter.accessTokenSecret = <required>
aeb1b06 Add keywords parameter to flume.conf
Jon Natkins authored Sep 21, 2012
34 TwitterAgent.sources.Twitter.keywords = hadoop, big data, analytics, bigdata, cloudera, data science, data scientiest, business intelligence, mapreduce, data warehouse, data warehousing, mahout, hbase, nosql, newsql, businessintelligence, cloudcomputing
2b93087 Custom Flume source and Flume configuration
Jon Natkins authored Aug 29, 2012
35
b7f2a3d Channel name was wrong in flume.conf
Jonathan Natkins authored Aug 30, 2012
36 TwitterAgent.sinks.HDFS.channel = MemChannel
2b93087 Custom Flume source and Flume configuration
Jon Natkins authored Aug 29, 2012
37 TwitterAgent.sinks.HDFS.type = hdfs
38 TwitterAgent.sinks.HDFS.hdfs.path = hdfs://hadoop1:8020/user/flume/tweets/%Y/%m/%d/%H/
39 TwitterAgent.sinks.HDFS.hdfs.fileType = DataStream
40 TwitterAgent.sinks.HDFS.hdfs.writeFormat = Text
4a26522 Following Flume best practices a bit more closely
Jon Natkins authored Oct 4, 2012
41 TwitterAgent.sinks.HDFS.hdfs.batchSize = 1000
2b93087 Custom Flume source and Flume configuration
Jon Natkins authored Aug 29, 2012
42 TwitterAgent.sinks.HDFS.hdfs.rollSize = 0
4a26522 Following Flume best practices a bit more closely
Jon Natkins authored Oct 4, 2012
43 TwitterAgent.sinks.HDFS.hdfs.rollCount = 10000
2b93087 Custom Flume source and Flume configuration
Jon Natkins authored Aug 29, 2012
44
45 TwitterAgent.channels.MemChannel.type = memory
46 TwitterAgent.channels.MemChannel.capacity = 10000
4a26522 Following Flume best practices a bit more closely
Jon Natkins authored Oct 4, 2012
47 TwitterAgent.channels.MemChannel.transactionCapacity = 100
Something went wrong with that request. Please try again.