Merge remote branch 'vzhabiuk/master'

clients/java/src/main/java/com/senseidb/search/client/json/req/SenseiClientRequest.java

@@ -6,6 +6,9 @@
 import java.util.List;
 import java.util.Map;
 
+import org.json.JSONException;
+import org.json.JSONObject;
+
 import com.senseidb.search.client.json.CustomJsonHandler;
 import com.senseidb.search.client.json.JsonField;
 import com.senseidb.search.client.json.req.filter.Filter;
@@ -160,7 +163,11 @@ public Builder addSort(Sort sort) {
       if ("_score".equalsIgnoreCase(sort.getField())) {
         request.sort.add("_score");
       } else {
-        request.sort.add(sort);
+        try {
+          request.sort.add(new JSONObject().put(sort.getField(), sort.getOrder().name()));
+        } catch (JSONException e) {
+         throw new RuntimeException(e);
+        }
       }      
       return this;
     }

perf/conf-perf/commands.txt

@@ -0,0 +1,7 @@
+-d64 -Xms4g -Xmx40g 
+-XX:ReservedCodeCacheSize=256m -XX:MaxPermSize=1536m -XX:+UseParallelGC 
+-XX:+DisableExplicitGC -Dname=PerfRunner
+
+rm -rf /tmp/sensei-example-cars-perf
+
+kill -9 $(ps aux | grep 'PerfRunner' | awk '{print $2}')

perf/conf-perf/custom-facets.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- custom facets -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans 
+                           http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+</beans>

perf/conf-perf/log4j-perf.properties

@@ -0,0 +1,11 @@
+log4j.rootLogger=INFO, QueryStatsAppender
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%p  %m%n
+log4j.appender.QueryStatsAppender=org.apache.log4j.FileAppender
+log4j.appender.QueryStatsAppender.File=report.log
+log4j.appender.QueryStatsAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.QueryStatsAppender.layout.ConversionPattern= %m%n
+log4j.logger.QueryStats=INFO,QueryStatsAppender
+

perf/conf-perf/plugins.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- sensei plugins -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans 
+                           http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+  <bean id="demo" class="com.senseidb.search.node.impl.DemoZoieSystemFactory$DemoZoieFactoryFactory" />
+</beans>

perf/conf-perf/schema.xml

@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema 
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://javasoze.github.com/sensei/schema/sensei-schema.xsd">
+
+  <!-- valid types:
+    uid - only 1 column can be of type uid
+    int - integer
+    short - short 
+    char - character
+    long - long 
+    float - float
+    double - double
+    string - string (default)
+    date - date, format string must be provided
+    text - searchable
+  -->
+
+  <table uid="id">
+	<column name="groupid" type="long" />
+	<column name="color" type="string" />
+	<column name="category" type="string" />
+	<column name="city" type="string" />
+	<column name="makemodel" type="string" />
+	<column name="year" type="int" />
+	<column name="price" type="float" />
+	<column name="mileage" type="int" />
+	<!-- multi is default to be false, if set to true, delimiter is used to break the values,
+	     default delimiter is ","
+	 -->
+	<column name="tags" type="string" multi="true" delimiter=","/>
+	<!-- attributes: indexed,store,termvector are only used when type is text -->
+	<column name="contents" type="text" index="ANALYZED" store="YES" termvector="YES" />
+  </table>
+
+   <!-- 
+     valid types are:
+      simple - corresponds to com.browseengine.bobo.facets.impl.SimpleFacetHandler
+      path - corresponds to com.browseengine.bobo.facets.impl.PathFacetHandler
+      range - corresponds to com.browseengine.bobo.facets.impl.RangeFacetHandler
+      multi - corresponds to com.browseengine.bobo.facets.impl.MultiFacetHandler
+      compact-multi - corresponds to com.browseengine.bobo.facets.impl.CompactMultiFacetHandler
+      custom - custom facet handler, name must be defined in a spring bean in custom-facets.xml
+   -->
+   <facets>
+	 <facet name="groupid" type="simple" />
+	 <facet name="color" type="simple" depends=""/>
+	 <facet name="category" type="simple" />
+	 <facet name="city" type="path">
+	  <params>
+		<param name="separator" value="/" />
+	  </params>
+	 </facet>
+	 <facet name="makemodel" type="path" />
+
+	 <facet name="year" type="range">
+		<params>
+		  <param name="range" value="1993-1994" />
+		  <param name="range" value="1995-1996" />
+		  <param name="range" value="1997-1998" />
+		  <param name="range" value="1999-2000" />
+		  <param name="range" value="2001-2002" />
+		</params>
+	 </facet>
+	 <facet name="mileage" type="range">
+		<params>
+		  <param name="range" value="*-12500" />
+		  <param name="range" value="12501-15000" />
+		  <param name="range" value="15001-17500" />
+		  <param name="range" value="17501-*" />
+		</params>
+	 </facet>
+	 <facet name="price" type="range">
+		<params>
+		  <param name="range" value="*,6700" />
+		  <param name="range" value="6800,9900" />
+		  <param name="range" value="10000,13100" />
+		  <param name="range" value="13200,17300" />
+		  <param name="range" value="17400,*" />
+		</params>
+	 </facet>
+	 <facet name="tags" type="multi" />
+
+	 <!-- example of a custom facet, defined in custom-facets.xml -->
+	 <!-- 
+	     a bean with name "foobar" must be defined
+	   <facet name="foobar" type="custom" dynamic="false"/>  
+	  -->
+   </facets>
+</schema>

perf/conf-perf/sensei.properties

@@ -0,0 +1,73 @@
+# sensei node parametersOQ
+sensei.node.id=1
+sensei.node.partitions=0
+
+# sensei network server parameters
+sensei.server.port=1234
+
+# internal broker to node socket settings
+# these are advanced settings, normally no need to change
+#sensei.server.requestThreadCorePoolSize=20
+#sensei.server.requestThreadMaxPoolSize=70
+#sensei.server.requestThreadKeepAliveTimeSecs=300
+
+# sensei cluster parameters
+sensei.cluster.name=sensei-example-cars
+
+# zookeeper settings
+sensei.cluster.url=localhost:2181
+#sensei.cluster.client-name = senseiClient
+#sensei.cluster.timeout=30000
+
+# sensei indexing parameters
+sensei.index.directory = /tmp/sensei-example-cars-perf
+
+sensei.index.batchSize = 2000000
+sensei.index.batchDelay = 5
+sensei.index.maxBatchSize = 2000000
+sensei.index.realtime = true
+
+# indicator of freshness of data, in seconds, a number <=0 implies as fast as possible
+sensei.index.freshness = 0
+sensei.index.skipBadRecords = true
+
+# gateway parameters
+
+sensei.gateway.class= com.senseidb.perf.LinedFileDataProviderMockBuilder
+
+# index manager parameters
+
+sensei.index.manager.default.maxpartition.id = 0
+
+# broker properties
+sensei.broker.port = 8080
+
+# broker server settings
+# default is normally good
+#sensei.broker.minThread = 50
+#sensei.broker.maxThread = 100
+#sensei.broker.maxWaittime = 2000
+
+#sensei.search.cluster.zookeeper.url = localhost:2181
+#sensei.search.cluster.zookeeper.conn.timeout = 30000
+
+#sensei.search.cluster.network.conn.timeout = 1000
+#sensei.search.cluster.network.write.timeout = 150
+#sensei.search.cluster.network.max.conn.per.node = 5
+#sensei.search.cluster.network.stale.timeout.mins = 10
+#sensei.search.cluster.network.stale.cleanup.freq.mins = 10
+custom.plugins.spring.class=com.senseidb.conf.SpringSenseiPluginFactory
+custom.plugins.spring.returnedClass=com.senseidb.svc.impl.AbstractSenseiCoreService
+custom.plugins.spring.springFile=plugins.xml
+sensei.plugin.services.list=custom.plugins.spring
+
+#Sensei plugin factory returns the list of custom facets
+
+# the list of beans might be referenced both by simple name and the full prefix
+#Sensei plugin factory returns the list of custom facets
+custom.facets.spring.class=com.senseidb.conf.SpringSenseiPluginFactory
+custom.facets.spring.returnedClass=com.browseengine.bobo.facets.FacetHandler, com.browseengine.bobo.facets.RuntimeFacetHandlerFactory
+custom.facets.spring.springFile=custom-facets.xml
+# the list of beans might be referenced both by simple name and the full prefix
+sensei.custom.facets.list= custom.facets.spring
+sensei.index.manager.default.eventsPerMin = 100000000

perf/pom.xml

@@ -25,7 +25,6 @@
 			</plugin>
 		</plugins>
 
-        <directory>${basedir}/perf-sensei-conf/ext</directory>
 	</build>
 
 	<properties>
@@ -38,6 +37,12 @@
 			<artifactId>sensei-core</artifactId>
 			<version>1.1.3-SNAPSHOT</version>
 		</dependency>
+		<dependency>
+			<groupId>${project.groupId}</groupId>
+			<artifactId>sensei-java-client</artifactId>
+			<version>1.1.3-SNAPSHOT</version>
+		</dependency>
+
 		<dependency>
 			<groupId>com.yammer.metrics</groupId>
 			<artifactId>metrics-core_2.8.1</artifactId>

perf/src/main/java/com/senseidb/perf/LinedFileDataProviderMockBuilder.java

@@ -0,0 +1,47 @@
+package com.senseidb.perf;
+
+import java.io.File;
+import java.util.Comparator;
+import java.util.Set;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.json.JSONObject;
+
+import proj.zoie.impl.indexing.StreamDataProvider;
+import proj.zoie.impl.indexing.ZoieConfig;
+
+import com.senseidb.gateway.SenseiGateway;
+import com.senseidb.indexing.DataSourceFilter;
+import com.senseidb.indexing.ShardingStrategy;
+
+public class LinedFileDataProviderMockBuilder  extends SenseiGateway<String>{
+
+  private Comparator<String> _versionComparator = ZoieConfig.DEFAULT_VERSION_COMPARATOR;
+
+
+  @Override
+  public StreamDataProvider<JSONObject> buildDataProvider(DataSourceFilter<String> dataFilter,
+      String oldSinceKey,
+      ShardingStrategy shardingStrategy,
+      Set<Integer> partitions) throws Exception
+  {
+
+    String path = config.get("file.path");
+    if (path == null) {
+      path = "data/cars.json";
+    }
+    long offset = oldSinceKey == null ? 0L : Long.parseLong(oldSinceKey);
+
+
+    PerfFileDataProvider provider = new PerfFileDataProvider(_versionComparator, new File(path), 0L, new LinkedBlockingQueue<JSONObject>(30000));
+    if (dataFilter!=null){
+      provider.setFilter(dataFilter);
+    }
+    return provider;
+  }
+
+  @Override
+  public Comparator<String> getVersionComparator() {
+    return _versionComparator;
+  }
+}

perf/src/main/java/com/senseidb/perf/PerfFileDataProvider.java

@@ -0,0 +1,47 @@
+package com.senseidb.perf;
+
+import java.io.File;
+import java.util.Comparator;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.json.JSONObject;
+
+import proj.zoie.api.DataConsumer.DataEvent;
+
+import com.senseidb.gateway.file.LinedJsonFileDataProvider;
+
+
+public class PerfFileDataProvider extends LinedJsonFileDataProvider {
+
+  public static LinkedBlockingQueue<JSONObject> queue;
+
+
+  public PerfFileDataProvider(Comparator<String> versionComparator, File file, long startingOffset, LinkedBlockingQueue<JSONObject> queue) {
+    super(versionComparator, file, startingOffset);
+    this.queue = queue; 
+  }
+
+
+
+
+  @Override
+  public DataEvent<JSONObject> next() {
+    JSONObject object = null;
+    try {
+      object = queue.take();
+    } catch (InterruptedException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }    
+    if (_offset % 10000 == 0) {
+      System.out.println("Indexed " + _offset + " documents. Queue size = " + queue.size());
+    }
+    if (object != null) {
+      return new DataEvent<JSONObject>(object, String.valueOf(_offset++));
+    }
+
+    return super.next();
+  }
+
+
+}