Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

cleanup and fixed an indexing issue

  • Loading branch information...
commit dea03e2451d0be9b9d43a83276ced315f29c9258 1 parent f7e5141
@javasoze authored
View
2  config/TwitterStreamer.conf
@@ -30,7 +30,7 @@ sensei.search.spring.url = "http://localhost:18080/sensei-rpc/SenseiSpringRPCSer
search.perPage = 10
-search.highlight.dohighlight = true
+search.highlight.dohighlight = false
search.highlight.pretag = "<span class=\"hl\">"
search.highlight.posttag = "</span>"
View
58 config/log4j/log4j.properties
@@ -1,9 +1,19 @@
-log4j.rootLogger=INFO, sensei
+log4j.rootLogger=INFO, chirper
log4j.appender.console1=org.apache.log4j.ConsoleAppender
log4j.appender.console1.layout=org.apache.log4j.PatternLayout
log4j.appender.console1.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c] [%x] %m%n
+log4j.appender.chirper=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.chirper.encoding=UTF-8
+
+log4j.appender.chirper.File=logs/chirper.log
+
+log4j.appender.chirper.DatePattern='.'yyyy-MM-dd
+
+log4j.appender.chirper.layout=org.apache.log4j.PatternLayout
+log4j.appender.chirper.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%x] %m%n
+
log4j.appender.sensei=org.apache.log4j.DailyRollingFileAppender
log4j.appender.sensei.encoding=UTF-8
@@ -15,3 +25,49 @@ log4j.appender.sensei.DatePattern='.'yyyy-MM-dd
log4j.appender.sensei.layout=org.apache.log4j.PatternLayout
log4j.appender.sensei.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%x] %m%n
+log4j.appender.voldemort=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.voldemort.encoding=UTF-8
+
+log4j.appender.voldemort.File=logs/voldemort-server.log
+
+log4j.appender.voldemort.DatePattern='.'yyyy-MM-dd
+
+log4j.appender.voldemort.layout=org.apache.log4j.PatternLayout
+log4j.appender.voldemort.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%x] %m%n
+
+log4j.appender.kafka=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.kafka.encoding=UTF-8
+
+log4j.appender.kafka.File=logs/kafka-server.log
+
+log4j.appender.kafka.DatePattern='.'yyyy-MM-dd
+
+log4j.appender.kafka.layout=org.apache.log4j.PatternLayout
+log4j.appender.kafka.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%x] %m%n
+
+
+log4j.appender.zookeeper=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.zookeeper.encoding=UTF-8
+
+log4j.appender.zookeeper.File=logs/zookeeper.log
+
+log4j.appender.zookeeper.DatePattern='.'yyyy-MM-dd
+
+log4j.appender.zookeeper.layout=org.apache.log4j.PatternLayout
+log4j.appender.zookeeper.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%x] %m%n
+
+log4j.appender.streamer=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.streamer.encoding=UTF-8
+
+log4j.appender.streamer.File=logs/streamer.log
+
+log4j.appender.streamer.DatePattern='.'yyyy-MM-dd
+
+log4j.appender.streamer.layout=org.apache.log4j.PatternLayout
+log4j.appender.streamer.layout.ConversionPattern=%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%x] %m%n
+
+log4j.logger.voldemort=DEBUG,voldemort
+log4j.logger.com.sensei=INFO,sensei
+log4j.logger.kafka=INFO,kafka
+log4j.logger.com.linkedin.chirper.streamer=DEBUG,streamer
+log4j.logger.org.apache.zookeeper=INFO,zookeeper
View
2  config/sensei/chirper/schema.xml
@@ -15,7 +15,7 @@
text - searchable
-->
- <table uid="id_str" delete-field="" skip-field="">
+ <table uid="id" delete-field="" skip-field="">
<column name="time" type="long" />
<column name="text" type="text" index="ANALYZED" store="NO" termvector="NO" />
</table>
View
2  config/sensei/chirper/sensei.properties
@@ -30,8 +30,8 @@ sensei.index.manager.default.kafka.host = localhost
sensei.index.manager.default.kafka.port = 9092
sensei.index.manager.default.kafka.topic = tweets
sensei.index.manager.default.kafka.batchsize = 100
+sensei.index.manager.default.filter = tweets-filter
-sensei.index.interpreter.default.filter = tweets-filter
sensei.query.builder.factory = chirper-query-builder-factory
View
BIN  lib/bobo-browse-2.5.0.jar
Binary file not shown
View
BIN  lib/sensei-0.0.1.jar
Binary file not shown
View
BIN  lib/zoie-2.5.0.jar
Binary file not shown
View
6 project/build/ChirperStreamerProject.scala
@@ -20,9 +20,9 @@ class ChirperStreamerProject(info: ProjectInfo) extends DefaultWebProject(info)
val googleCollections = "com.google.collections" % "google-collections" % "1.0" % "runtime"
val fastutil = "fastutil" % "fastutil" % "5.0.5"
- val lucene = "org.apache.lucene" % "lucene-core" % "2.9.1"
- val luceneHighlighter = "org.apache.lucene" % "lucene-highlighter" % "2.9.1"
- val kamikaze = "com.sna-projects.kamikaze" % "kamikaze" % "3.0.3" % "runtime"
+ val lucene = "org.apache.lucene" % "lucene-core" % "3.0.2"
+ val luceneHighlighter = "org.apache.lucene" % "lucene-highlighter" % "3.0.2"
+ val kamikaze = "com.linkedin.kamikaze" % "kamikaze" % "3.0.4" % "runtime"
val protobuf = "com.google.protobuf" % "protobuf-java" % "2.3.0"
val netty = "org.jboss.netty" % "netty" % "3.2.3.Final" % "runtime"
View
4 src/main/scala/com/linkedin/chirper/DefaultConfigs.scala
@@ -3,8 +3,6 @@ package com.linkedin.chirper;
import com.linkedin.led.twitter.config._
import java.nio.charset.Charset
-import proj.zoie.api.DefaultZoieVersion
-import proj.zoie.api.DefaultZoieVersion.DefaultZoieVersionFactory
import proj.zoie.impl.indexing.ZoieConfig
@@ -23,7 +21,7 @@ object DefaultConfigs{
val kafkaport = Config.readInt("kafka.port")
// zoie configuration, use default
- val zoieConfig = new ZoieConfig[DefaultZoieVersion](new DefaultZoieVersionFactory());
+ val zoieConfig = new ZoieConfig();
// voldemort configuration
val voldemortUrl = Config.readString("voldemort.url")
View
14 src/main/scala/com/linkedin/chirper/streamer/ChirperStreamProcessor.scala
@@ -13,7 +13,7 @@ import com.linkedin.led.twitter.streaming.StreamProcessor
import kafka.message._
import kafka.producer._
-
+import org.apache.log4j.Logger
import voldemort.scalmert.client.StoreClient
import voldemort.client.{SocketStoreClientFactory,ClientConfig}
import voldemort.scalmert.Implicits._
@@ -23,6 +23,8 @@ import com.linkedin.chirper.DefaultConfigs
// processes each tweet from the streamer
class ChirperStreamProcessor extends StreamProcessor{
+ val log = Logger.getLogger(classOf[ChirperStreamProcessor])
+
val kafkaTopic = Config.readString("tweet.kafka.topic")
val voldemortStore = Config.readString("tweet.voldemort.store")
@@ -42,18 +44,20 @@ class ChirperStreamProcessor extends StreamProcessor{
// for each tweet
try{
// output to console
- println(line)
+ /// println(line)
val jsonObj = new JSONObject(line)
- val id = jsonObj.getString("id_str")
+ val id = jsonObj.getString("id_str")
// send to voldemort store
tweetStore(id) = line
-
+
+ val tweetString = tweetStore.get(id)
+ log.debug("tweetid: "+id)
// send to kafka
kafkaProducer.send(kafkaTopic,new ByteBufferMessageSet(new Message(line.getBytes(DefaultConfigs.UTF8Charset))))
}
catch{
- case je: JSONException =>
+ case je: JSONException => je.printStackTrace()
case e: Exception => e.printStackTrace()
}
line = reader.readLine()
Please sign in to comment.
Something went wrong with that request. Please try again.