Merge abd09f7 into 54bcf49

.gitignore

@@ -12,6 +12,7 @@
 .settings
 .cache
 target/
+store/CSDK/cmake-build-debug/*
 .project
 .classpath
 metastore_db/

core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java

@@ -506,6 +506,10 @@ public final class CarbonCommonConstants {
    * File separator
    */
   public static final String FILE_SEPARATOR = "/";
+  /**
+   * ARRAY separator
+   */
+  public static final String ARRAY_SEPARATOR = "\001";
   /**
    * MAX_QUERY_EXECUTION_TIME
    */

docs/CSDK-guide.md

@@ -0,0 +1,197 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more 
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership. 
+    The ASF licenses this file to you under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with 
+    the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+    
+    Unless required by applicable law or agreed to in writing, software 
+    distributed under the License is distributed on an "AS IS" BASIS, 
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and 
+    limitations under the License.
+-->
+
+# CSDK Guide
+
+CarbonData CSDK provides C++ interface to write and read carbon file. 
+CSDK use JNI to invoke java SDK in C++ code.
+
+
+# CSDK Reader
+This CSDK reader reads CarbonData file and carbonindex file at a given path.
+External client can make use of this reader to read CarbonData files in C++ 
+code and without CarbonSession.
+
+
+In the carbon jars package, there exist a carbondata-sdk.jar, 
+including SDK reader for CSDK.
+## Quick example
+```
+// 1. init JVM
+JavaVM *jvm;
+JNIEnv *initJVM() {
+    JNIEnv *env;
+    JavaVMInitArgs vm_args;
+    int parNum = 3;
+    int res;
+    JavaVMOption options[parNum];
+
+    options[0].optionString = "-Djava.compiler=NONE";
+    options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
+    options[2].optionString = "-verbose:jni";
+    vm_args.version = JNI_VERSION_1_8;
+    vm_args.nOptions = parNum;
+    vm_args.options = options;
+    vm_args.ignoreUnrecognized = JNI_FALSE;
+
+    res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
+    if (res < 0) {
+        fprintf(stderr, "\nCan't create Java VM\n");
+        exit(1);
+    }
+
+    return env;
+}
+
+// 2. create carbon reader and read data 
+// 2.1 read data from local disk
+/**
+ * test read data from local disk, without projection
+ *
+ * @param env  jni env
+ * @return
+ */
+bool readFromLocalWithoutProjection(JNIEnv *env) {
+
+    CarbonReader carbonReaderClass;
+    carbonReaderClass.builder(env, "../resources/carbondata", "test");
+    carbonReaderClass.build();
+
+    while (carbonReaderClass.hasNext()) {
+        jobjectArray row = carbonReaderClass.readNextRow();
+        jsize length = env->GetArrayLength(row);
+        int j = 0;
+        for (j = 0; j < length; j++) {
+            jobject element = env->GetObjectArrayElement(row, j);
+            char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+            printf("%s\t", str);
+        }
+        printf("\n");
+    }
+    carbonReaderClass.close();
+}
+
+// 2.2 read data from S3
+
+/**
+ * read data from S3
+ * parameter is ak sk endpoint
+ *
+ * @param env jni env
+ * @param argv argument vector
+ * @return
+ */
+bool readFromS3(JNIEnv *env, char *argv[]) {
+    CarbonReader reader;
+
+    char *args[3];
+    // "your access key"
+    args[0] = argv[1];
+    // "your secret key"
+    args[1] = argv[2];
+    // "your endPoint"
+    args[2] = argv[3];
+
+    reader.builder(env, "s3a://sdk/WriterOutput", "test");
+    reader.withHadoopConf(3, args);
+    reader.build();
+    printf("\nRead data from S3:\n");
+    while (reader.hasNext()) {
+        jobjectArray row = reader.readNextRow();
+        jsize length = env->GetArrayLength(row);
+
+        int j = 0;
+        for (j = 0; j < length; j++) {
+            jobject element = env->GetObjectArrayElement(row, j);
+            char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+            printf("%s\t", str);
+        }
+        printf("\n");
+    }
+
+    reader.close();
+}
+
+// 3. destory JVM
+    (jvm)->DestroyJavaVM();
+```
+Find example code at main.cpp of CSDK module
+
+## API List
+```
+    /**
+     * create a CarbonReaderBuilder object for building carbonReader,
+     * CarbonReaderBuilder object  can configure different parameter
+     *
+     * @param env JNIEnv
+     * @param path data store path
+     * @param tableName table name
+     * @return CarbonReaderBuilder object
+     */
+    jobject builder(JNIEnv *env, char *path, char *tableName);
+
+    /**
+     * Configure the projection column names of carbon reader
+     *
+     * @param argc argument counter
+     * @param argv argument vector
+     * @return CarbonReaderBuilder object
+     */
+    jobject projection(int argc, char *argv[]);
+
+    /**
+     *  build carbon reader with argument vector
+     *  it support multiple parameter
+     *  like: key=value
+     *  for example: fs.s3a.access.key=XXXX, XXXX is user's access key value
+     *
+     * @param argc argument counter
+     * @param argv argument vector
+     * @return CarbonReaderBuilder object
+     **/
+    jobject withHadoopConf(int argc, char *argv[]);
+
+    /**
+     * build carbonReader object for reading data
+     * it support read data from load disk
+     *
+     * @return carbonReader object
+     */
+    jobject build();
+
+    /**
+     * Whether it has next row data
+     *
+     * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
+     */
+    jboolean hasNext();
+
+    /**
+     * read next row from data
+     *
+     * @return object array of one row
+     */
+    jobjectArray readNextRow();
+
+    /**
+     * close the carbon reader
+     *
+     * @return  boolean value
+     */
+    jboolean close();
+
+```

examples/spark2/pom.xml

@@ -80,7 +80,7 @@
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
-      <version>4.2</version>
+      <version>${httpclient.version}</version>
     </dependency>
     <dependency>
       <groupId>org.scalatest</groupId>

pom.xml

@@ -113,6 +113,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <snappy.version>1.1.2.6</snappy.version>
     <hadoop.version>2.7.2</hadoop.version>
+    <httpclient.version>4.2.5</httpclient.version>
     <scala.binary.version>2.11</scala.binary.version>
     <scala.version>2.11.8</scala.version>
     <hadoop.deps.scope>compile</hadoop.deps.scope>
@@ -483,6 +484,7 @@
       <id>hadoop-2.8</id>
       <properties>
         <hadoop.version>2.8.3</hadoop.version>
+        <httpclient.version>4.5.2</httpclient.version>
       </properties>
     </profile>
     <profile>

store/CSDK/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required (VERSION 2.8)
+project (CJDK)
+set(CMAKE_BUILD_TYPE Debug)
+SET (CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
+
+find_package (JNI REQUIRED)
+
+include_directories (${JNI_INCLUDE_DIRS})
+
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+set (SOURCE_FILES CarbonReader.cpp CarbonReader.h main.cpp)
+
+add_executable (CJDK ${SOURCE_FILES})
+get_filename_component (JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY)
+message (${JAVA_JVM_LIBRARY_DIR})
+target_link_libraries (CJDK ${JAVA_JVM_LIBRARY} )
+

store/CSDK/CarbonReader.cpp

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "CarbonReader.h"
+#include <jni.h>
+
+jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) {
+
+    jniEnv = env;
+    jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+    jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+        "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+    jstring jpath = env->NewStringUTF(path);
+    jstring jtableName = env->NewStringUTF(tableName);
+    jvalue args[2];
+    args[0].l = jpath;
+    args[1].l = jtableName;
+    carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::builder(JNIEnv *env, char *path) {
+    jniEnv = env;
+    jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+    jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+        "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+    jstring jpath = env->NewStringUTF(path);
+    jvalue args[1];
+    args[0].l = jpath;
+    carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::projection(int argc, char *argv[]) {
+    jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+    jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "projection",
+        "([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+    jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;");
+    jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass, NULL);
+    for (int i = 0; i < argc; ++i) {
+        jstring value = jniEnv->NewStringUTF(argv[i]);
+        jniEnv->SetObjectArrayElement(array, i, value);
+    }
+
+    jvalue args[1];
+    args[0].l = array;
+    carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::withHadoopConf(char *key, char *value) {
+    jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+    jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "withHadoopConf",
+        "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+
+    jvalue args[2];
+    args[0].l = jniEnv->NewStringUTF(key);
+    args[1].l = jniEnv->NewStringUTF(value);
+    carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args);
+    return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::build() {
+    jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+    jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build",
+        "()Lorg/apache/carbondata/sdk/file/CarbonReader;");
+    carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID);
+    return carbonReaderObject;
+}
+
+jboolean CarbonReader::hasNext() {
+    jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+    jmethodID hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z");
+    unsigned char hasNext = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID);
+    return hasNext;
+}
+
+jobjectArray CarbonReader::readNextRow() {
+    jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+    jmethodID readNextRow2ID = jniEnv->GetMethodID(carbonReader, "readNextStringRow", "()[Ljava/lang/Object;");
+    jobjectArray row = (jobjectArray) jniEnv->CallObjectMethod(carbonReaderObject, readNextRow2ID);
+    return row;
+}
+
+jboolean CarbonReader::close() {
+    jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+    jmethodID closeID = jniEnv->GetMethodID(carbonReader, "close", "()V");
+    jniEnv->CallBooleanMethod(carbonReaderObject, closeID);
+}