Merge bb5ba4b into db08fe1

docs/csdk-guide.md

@@ -193,6 +193,18 @@ release the memory and destroy JVM.
     void outputPath(char *path);
 ```
 
+```
+    /**
+      * sets the list of columns that needs to be in sorted order
+      *
+      * @param argc argc argument counter, the number of projection column
+      * @param argv argv is a string array of columns that needs to be sorted.
+      *                  If it is null or by default all dimensions are selected for sorting
+      *                  If it is empty array, no columns are sorted
+      */
+    void sortBy(int argc, char *argv[]);
+```
+
 ```
     /**
      * configure the schema with json style schema
@@ -214,6 +226,119 @@ release the memory and destroy JVM.
     void withHadoopConf(char *key, char *value);
 ```
 
+```
+ /**
+     *  To support the table properties for writer
+     *
+     * @param key properties key
+     * @param value properties value
+     */
+    void withTableProperty(char *key, char *value);
+```
+
+```
+    /**
+     * To support the load options for C++ sdk writer
+     *
+     * @param options key,value pair of load options.
+     * supported keys values are
+     * a. bad_records_logger_enable -- true (write into separate logs), false
+     * b. bad_records_action -- FAIL, FORCE, IGNORE, REDIRECT
+     * c. bad_record_path -- path
+     * d. dateformat -- same as JAVA SimpleDateFormat
+     * e. timestampformat -- same as JAVA SimpleDateFormat
+     * f. complex_delimiter_level_1 -- value to Split the complexTypeData
+     * g. complex_delimiter_level_2 -- value to Split the nested complexTypeData
+     * h. quotechar
+     * i. escapechar
+     *
+     * Default values are as follows.
+     *
+     * a. bad_records_logger_enable -- "false"
+     * b. bad_records_action -- "FAIL"
+     * c. bad_record_path -- ""
+     * d. dateformat -- "" , uses from carbon.properties file
+     * e. timestampformat -- "", uses from carbon.properties file
+     * f. complex_delimiter_level_1 -- "$"
+     * g. complex_delimiter_level_2 -- ":"
+     * h. quotechar -- "\""
+     * i. escapechar -- "\\"
+     *
+     * @return updated CarbonWriterBuilder
+     */
+    void withLoadOption(char *key, char *value);
+```
+
+```
+    /**
+     * sets the taskNo for the writer. CSDKs concurrently running
+     * will set taskNo in order to avoid conflicts in file's name during write.
+     *
+     * @param taskNo is the TaskNo user wants to specify.
+     *               by default it is system time in nano seconds.
+     */
+    void taskNo(long taskNo);
+```
+
+```
+    /**
+     * to set the timestamp in the carbondata and carbonindex index files
+     *
+     * @param timestamp is a timestamp to be used in the carbondata and carbonindex index files.
+     * By default set to zero.
+     * @return updated CarbonWriterBuilder
+     */
+    void uniqueIdentifier(long timestamp);
+```
+
+```
+    /**
+     * To make c++ sdk writer thread safe.
+     *
+     * @param numOfThreads should number of threads in which writer is called in multi-thread scenario
+     *                      default C++ sdk writer is not thread safe.
+     *                      can use one writer instance in one thread only.
+     */
+    void withThreadSafe(short numOfThreads) ;
+```
+
+```
+    /**
+     * To set the carbondata file size in MB between 1MB-2048MB
+     *
+     * @param blockSize is size in MB between 1MB to 2048 MB
+     * default value is 1024 MB
+     */
+    void withBlockSize(int blockSize);
+```
+
+```
+    /**
+     * To set the blocklet size of CarbonData file
+     *
+     * @param blockletSize is blocklet size in MB
+     *        default value is 64 MB
+     * @return updated CarbonWriterBuilder
+     */
+    void withBlockletSize(int blockletSize);
+```
+
+```
+    /**
+     * @param localDictionaryThreshold is localDictionaryThreshold, default is 10000
+     * @return updated CarbonWriterBuilder
+     */
+    void localDictionaryThreshold(int localDictionaryThreshold);
+```
+
+```
+    /**
+     * @param enableLocalDictionary enable local dictionary, default is false
+     * @return updated CarbonWriterBuilder
+     */
+    void enableLocalDictionary(bool enableLocalDictionary);
+```
+
 ```
     /**
      * @param appName appName which is writing the carbondata files

examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java

@@ -56,11 +56,9 @@ public static void main(String[] args) {
             fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2));
             fields[9] = new Field("varcharField", DataTypes.VARCHAR);
             fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING));
-            Map<String, String> map = new HashMap<>();
-            map.put("complex_delimiter_level_1", "#");
             CarbonWriter writer = CarbonWriter.builder()
                 .outputPath(path)
-                .withLoadOptions(map)
+                .withLoadOption("complex_delimiter_level_1", "#")
                 .withCsvInput(new Schema(fields))
                 .writtenBy("CarbonReaderExample")
                 .build();

examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3ReadExample.java

@@ -43,26 +43,23 @@ public static void main(String[] args) throws Exception {
             System.exit(0);
         }
 
-        String path = "s3a://sdk/WriterOutput";
+        String path = "s3a://sdk/WriterOutput/carbondata5";
         if (args.length > 3) {
             path=args[3];
         }
 
         // Read data
-
         EqualToExpression equalToExpression = new EqualToExpression(
             new ColumnExpression("name", DataTypes.STRING),
             new LiteralExpression("robot1", DataTypes.STRING));
 
-        Configuration configuration = new Configuration();
-        configuration.set(ACCESS_KEY, args[0]);
-        configuration.set(SECRET_KEY, args[1]);
-        configuration.set(ENDPOINT, args[2]);
         CarbonReader reader = CarbonReader
             .builder(path, "_temp")
             .projection(new String[]{"name", "age"})
             .filter(equalToExpression)
-            .withHadoopConf(configuration)
+            .withHadoopConf(ACCESS_KEY, args[0])
+            .withHadoopConf(SECRET_KEY, args[1])
+            .withHadoopConf(ENDPOINT, args[2])
             .build();
 
         System.out.println("\nData:");
@@ -79,7 +76,9 @@ public static void main(String[] args) throws Exception {
         CarbonReader reader2 = CarbonReader
             .builder(path, "_temp")
             .projection(new String[]{"name", "age"})
-            .withHadoopConf(configuration)
+            .withHadoopConf(ACCESS_KEY, args[0])
+            .withHadoopConf(SECRET_KEY, args[1])
+            .withHadoopConf(ENDPOINT, args[2])
             .build();
 
         System.out.println("\nData:");

store/CSDK/CMakeLists.txt

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 cmake_minimum_required(VERSION 2.8)
-project(CJDK)
+project(CSDK)
 set(CMAKE_BUILD_TYPE Debug)
 SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
 
@@ -27,8 +27,8 @@ set(SOURCE_FILES src/CarbonReader.cpp src/CarbonReader.h test/main.cpp src/Carbo
         src/CarbonRow.cpp src/CarbonWriter.h  src/CarbonWriter.cpp src/CarbonSchemaReader.h
         src/CarbonSchemaReader.cpp src/Schema.h src/Schema.cpp src/CarbonProperties.cpp src/CarbonProperties.h)
 
-add_executable(CJDK ${SOURCE_FILES})
+add_executable(CSDK ${SOURCE_FILES})
 get_filename_component(JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY)
 message(${JAVA_JVM_LIBRARY_DIR})
-target_link_libraries(CJDK ${JAVA_JVM_LIBRARY})
+target_link_libraries(CSDK ${JAVA_JVM_LIBRARY})
 

store/CSDK/src/CarbonReader.h

@@ -94,8 +94,8 @@ class CarbonReader {
     /**
      * Configure the projection column names of carbon reader
      *
-     * @param argc argument counter
-     * @param argv argument vector
+     * @param argc argument counter, the number of projection column
+     * @param argv argument vector, projection column names
      */
     void projection(int argc, char *argv[]);
 

store/CSDK/src/CarbonSchemaReader.cpp

@@ -34,7 +34,7 @@ jobject CarbonSchemaReader::readSchema(char *path) {
         throw std::runtime_error("path parameter can't be NULL.");
     }
     jmethodID methodID = jniEnv->GetStaticMethodID(carbonSchemaReaderClass, "readSchema",
-                                                   "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/Schema;");
+        "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/Schema;");
     if (methodID == NULL) {
         throw std::runtime_error("Can't find the method in java: readSchema");
     }
@@ -53,7 +53,7 @@ jobject CarbonSchemaReader::readSchema(char *path, bool validateSchema) {
         throw std::runtime_error("path parameter can't be NULL.");
     }
     jmethodID methodID = jniEnv->GetStaticMethodID(carbonSchemaReaderClass, "readSchema",
-                                                   "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/Schema;");
+        "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/Schema;");
     if (methodID == NULL) {
         throw std::runtime_error("Can't find the method in java: readSchema");
     }