Permalink
Browse files

Added the perf client

  • Loading branch information...
vzhabiuk
vzhabiuk committed Apr 21, 2012
1 parent 0b07e81 commit e38d212f3d55d59e01f0dd6807620d5617e647f2
@@ -6,6 +6,9 @@
import java.util.List;
import java.util.Map;
+import org.json.JSONException;
+import org.json.JSONObject;
+
import com.senseidb.search.client.json.CustomJsonHandler;
import com.senseidb.search.client.json.JsonField;
import com.senseidb.search.client.json.req.filter.Filter;
@@ -160,7 +163,11 @@ public Builder addSort(Sort sort) {
if ("_score".equalsIgnoreCase(sort.getField())) {
request.sort.add("_score");
} else {
- request.sort.add(sort);
+ try {
+ request.sort.add(new JSONObject().put(sort.getField(), sort.getOrder().name()));
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
}
return this;
}
@@ -0,0 +1,7 @@
+-d64 -Xms4g -Xmx40g
+-XX:ReservedCodeCacheSize=256m -XX:MaxPermSize=1536m -XX:+UseParallelGC
+-XX:+DisableExplicitGC -Dname=PerfRunner
+
+rm -rf /tmp/sensei-example-cars-perf
+
+kill -9 $(ps aux | grep 'PerfRunner' | awk '{print $2}')
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- custom facets -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+</beans>
@@ -0,0 +1,11 @@
+log4j.rootLogger=INFO, QueryStatsAppender
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%p %m%n
+log4j.appender.QueryStatsAppender=org.apache.log4j.FileAppender
+log4j.appender.QueryStatsAppender.File=report.log
+log4j.appender.QueryStatsAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.QueryStatsAppender.layout.ConversionPattern= %m%n
+log4j.logger.QueryStats=INFO,QueryStatsAppender
+
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- sensei plugins -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+ <bean id="demo" class="com.senseidb.search.node.impl.DemoZoieSystemFactory$DemoZoieFactoryFactory" />
+</beans>
View
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://javasoze.github.com/sensei/schema/sensei-schema.xsd">
+
+ <!-- valid types:
+ uid - only 1 column can be of type uid
+ int - integer
+ short - short
+ char - character
+ long - long
+ float - float
+ double - double
+ string - string (default)
+ date - date, format string must be provided
+ text - searchable
+ -->
+
+ <table uid="id">
+ <column name="groupid" type="long" />
+ <column name="color" type="string" />
+ <column name="category" type="string" />
+ <column name="city" type="string" />
+ <column name="makemodel" type="string" />
+ <column name="year" type="int" />
+ <column name="price" type="float" />
+ <column name="mileage" type="int" />
+ <!-- multi is default to be false, if set to true, delimiter is used to break the values,
+ default delimiter is ","
+ -->
+ <column name="tags" type="string" multi="true" delimiter=","/>
+ <!-- attributes: indexed,store,termvector are only used when type is text -->
+ <column name="contents" type="text" index="ANALYZED" store="YES" termvector="YES" />
+ </table>
+
+ <!--
+ valid types are:
+ simple - corresponds to com.browseengine.bobo.facets.impl.SimpleFacetHandler
+ path - corresponds to com.browseengine.bobo.facets.impl.PathFacetHandler
+ range - corresponds to com.browseengine.bobo.facets.impl.RangeFacetHandler
+ multi - corresponds to com.browseengine.bobo.facets.impl.MultiFacetHandler
+ compact-multi - corresponds to com.browseengine.bobo.facets.impl.CompactMultiFacetHandler
+ custom - custom facet handler, name must be defined in a spring bean in custom-facets.xml
+ -->
+ <facets>
+ <facet name="groupid" type="simple" />
+ <facet name="color" type="simple" depends=""/>
+ <facet name="category" type="simple" />
+ <facet name="city" type="path">
+ <params>
+ <param name="separator" value="/" />
+ </params>
+ </facet>
+ <facet name="makemodel" type="path" />
+
+ <facet name="year" type="range">
+ <params>
+ <param name="range" value="1993-1994" />
+ <param name="range" value="1995-1996" />
+ <param name="range" value="1997-1998" />
+ <param name="range" value="1999-2000" />
+ <param name="range" value="2001-2002" />
+ </params>
+ </facet>
+ <facet name="mileage" type="range">
+ <params>
+ <param name="range" value="*-12500" />
+ <param name="range" value="12501-15000" />
+ <param name="range" value="15001-17500" />
+ <param name="range" value="17501-*" />
+ </params>
+ </facet>
+ <facet name="price" type="range">
+ <params>
+ <param name="range" value="*,6700" />
+ <param name="range" value="6800,9900" />
+ <param name="range" value="10000,13100" />
+ <param name="range" value="13200,17300" />
+ <param name="range" value="17400,*" />
+ </params>
+ </facet>
+ <facet name="tags" type="multi" />
+
+ <!-- example of a custom facet, defined in custom-facets.xml -->
+ <!--
+ a bean with name "foobar" must be defined
+ <facet name="foobar" type="custom" dynamic="false"/>
+ -->
+ </facets>
+</schema>
@@ -0,0 +1,73 @@
+# sensei node parametersOQ
+sensei.node.id=1
+sensei.node.partitions=0
+
+# sensei network server parameters
+sensei.server.port=1234
+
+# internal broker to node socket settings
+# these are advanced settings, normally no need to change
+#sensei.server.requestThreadCorePoolSize=20
+#sensei.server.requestThreadMaxPoolSize=70
+#sensei.server.requestThreadKeepAliveTimeSecs=300
+
+# sensei cluster parameters
+sensei.cluster.name=sensei-example-cars
+
+# zookeeper settings
+sensei.cluster.url=localhost:2181
+#sensei.cluster.client-name = senseiClient
+#sensei.cluster.timeout=30000
+
+# sensei indexing parameters
+sensei.index.directory = /tmp/sensei-example-cars-perf
+
+sensei.index.batchSize = 2000000
+sensei.index.batchDelay = 5
+sensei.index.maxBatchSize = 2000000
+sensei.index.realtime = true
+
+# indicator of freshness of data, in seconds, a number <=0 implies as fast as possible
+sensei.index.freshness = 0
+sensei.index.skipBadRecords = true
+
+# gateway parameters
+
+sensei.gateway.class= com.senseidb.perf.LinedFileDataProviderMockBuilder
+
+# index manager parameters
+
+sensei.index.manager.default.maxpartition.id = 0
+
+# broker properties
+sensei.broker.port = 8080
+
+# broker server settings
+# default is normally good
+#sensei.broker.minThread = 50
+#sensei.broker.maxThread = 100
+#sensei.broker.maxWaittime = 2000
+
+#sensei.search.cluster.zookeeper.url = localhost:2181
+#sensei.search.cluster.zookeeper.conn.timeout = 30000
+
+#sensei.search.cluster.network.conn.timeout = 1000
+#sensei.search.cluster.network.write.timeout = 150
+#sensei.search.cluster.network.max.conn.per.node = 5
+#sensei.search.cluster.network.stale.timeout.mins = 10
+#sensei.search.cluster.network.stale.cleanup.freq.mins = 10
+custom.plugins.spring.class=com.senseidb.conf.SpringSenseiPluginFactory
+custom.plugins.spring.returnedClass=com.senseidb.svc.impl.AbstractSenseiCoreService
+custom.plugins.spring.springFile=plugins.xml
+sensei.plugin.services.list=custom.plugins.spring
+
+#Sensei plugin factory returns the list of custom facets
+
+# the list of beans might be referenced both by simple name and the full prefix
+#Sensei plugin factory returns the list of custom facets
+custom.facets.spring.class=com.senseidb.conf.SpringSenseiPluginFactory
+custom.facets.spring.returnedClass=com.browseengine.bobo.facets.FacetHandler, com.browseengine.bobo.facets.RuntimeFacetHandlerFactory
+custom.facets.spring.springFile=custom-facets.xml
+# the list of beans might be referenced both by simple name and the full prefix
+sensei.custom.facets.list= custom.facets.spring
+sensei.index.manager.default.eventsPerMin = 100000000
@@ -0,0 +1,47 @@
+package com.senseidb.perf;
+
+import java.io.File;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.json.JSONObject;
+
+import proj.zoie.impl.indexing.StreamDataProvider;
+import proj.zoie.impl.indexing.ZoieConfig;
+
+import com.senseidb.gateway.SenseiGateway;
+import com.senseidb.indexing.DataSourceFilter;
+import com.senseidb.indexing.ShardingStrategy;
+
+public class LinedFileDataProviderMockBuilder extends SenseiGateway<String>{
+
+ private Comparator<String> _versionComparator = ZoieConfig.DEFAULT_VERSION_COMPARATOR;
+
+
+ @Override
+ public StreamDataProvider<JSONObject> buildDataProvider(DataSourceFilter<String> dataFilter,
+ String oldSinceKey,
+ ShardingStrategy shardingStrategy,
+ Set<Integer> partitions) throws Exception
+ {
+
+ String path = config.get("file.path");
+ if (path == null) {
+ path = "data/cars.json";
+ }
+ long offset = oldSinceKey == null ? 0L : Long.parseLong(oldSinceKey);
+
+
+ PerfFileDataProvider provider = new PerfFileDataProvider(_versionComparator, new File(path), 0L, new LinkedBlockingQueue<JSONObject>(30000));
+ if (dataFilter!=null){
+ provider.setFilter(dataFilter);
+ }
+ return provider;
+ }
+
+ @Override
+ public Comparator<String> getVersionComparator() {
+ return _versionComparator;
+ }
+}
@@ -0,0 +1,47 @@
+package com.senseidb.perf;
+
+import java.io.File;
+import java.util.Comparator;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.json.JSONObject;
+
+import proj.zoie.api.DataConsumer.DataEvent;
+
+import com.senseidb.gateway.file.LinedJsonFileDataProvider;
+
+
+public class PerfFileDataProvider extends LinedJsonFileDataProvider {
+
+ public static LinkedBlockingQueue<JSONObject> queue;
+
+
+ public PerfFileDataProvider(Comparator<String> versionComparator, File file, long startingOffset, LinkedBlockingQueue<JSONObject> queue) {
+ super(versionComparator, file, startingOffset);
+ this.queue = queue;
+ }
+
+
+
+
+ @Override
+ public DataEvent<JSONObject> next() {
+ JSONObject object = null;
+ try {
+ object = queue.take();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ if (_offset % 10000 == 0) {
+ System.out.println("Indexed " + _offset + " documents. Queue size = " + queue.size());
+ }
+ if (object != null) {
+ return new DataEvent<JSONObject>(object, String.valueOf(_offset++));
+ }
+
+ return super.next();
+ }
+
+
+}
Oops, something went wrong.

0 comments on commit e38d212

Please sign in to comment.