Skip to content

Commit

Permalink
[CARBONDATA-3000] Provide C++ interface for writing carbon data in CSDK
Browse files Browse the repository at this point in the history
1.suport string, short, int, long, double, float, array<string>, boolean data type
2.provide builder, build, write, close interface
3.TODO: support S3

fix error

optimize
  • Loading branch information
xubo245 committed Oct 26, 2018
1 parent 019f5cd commit 8cd7d00
Show file tree
Hide file tree
Showing 6 changed files with 535 additions and 1 deletion.
2 changes: 1 addition & 1 deletion store/CSDK/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ find_package(JNI REQUIRED)
include_directories(${JNI_INCLUDE_DIRS})

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
set(SOURCE_FILES src/CarbonReader.cpp src/CarbonReader.h test/main.cpp src/CarbonRow.h src/CarbonRow.cpp)
set(SOURCE_FILES src/CarbonReader.cpp src/CarbonReader.h test/main.cpp src/CarbonRow.h src/CarbonRow.cpp src/CarbonWriter.h src/CarbonWriter.cpp)

add_executable(CJDK ${SOURCE_FILES})
get_filename_component(JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY)
Expand Down
90 changes: 90 additions & 0 deletions store/CSDK/src/CarbonWriter.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "CarbonWriter.h"

void CarbonWriter::builder(JNIEnv *env) {
jniEnv = env;
carbonWriter = env->FindClass("org/apache/carbondata/sdk/file/CarbonWriter");
jmethodID carbonWriterBuilderID = env->GetStaticMethodID(carbonWriter, "builder",
"()Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
carbonWriterBuilderObject = env->CallStaticObjectMethod(carbonWriter, carbonWriterBuilderID);
}

void CarbonWriter::outputPath(char *path) {
jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject);
jmethodID carbonWriterBuilderID = jniEnv->GetMethodID(carbonWriterBuilderClass, "outputPath",
"(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
jstring jPath = jniEnv->NewStringUTF(path);
jvalue args[1];
args[0].l = jPath;
carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, carbonWriterBuilderID, args);
}

void CarbonWriter::withCsvInput(char *jsonSchema) {

jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject);
jmethodID carbonWriterBuilderID = jniEnv->GetMethodID(carbonWriterBuilderClass, "withCsvInput",
"(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
jstring jPath = jniEnv->NewStringUTF(jsonSchema);
jvalue args[1];
args[0].l = jPath;
carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, carbonWriterBuilderID, args);
};

void CarbonWriter::withHadoopConf(char *key, char *value) {
jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject);
jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "withHadoopConf",
"(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
jvalue args[2];
args[0].l = jniEnv->NewStringUTF(key);
args[1].l = jniEnv->NewStringUTF(value);
carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}

void CarbonWriter::writtenBy(char *appName) {
jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject);
jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "writtenBy",
"(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;");
jvalue args[1];
args[0].l = jniEnv->NewStringUTF(appName);
carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args);
}

void CarbonWriter::build() {
jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject);
jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "build",
"()Lorg/apache/carbondata/sdk/file/CarbonWriter;");
carbonWriterObject = jniEnv->CallObjectMethod(carbonWriterBuilderObject, methodID);
carbonWriter = jniEnv->GetObjectClass(carbonWriterObject);
writeID = jniEnv->GetMethodID(carbonWriter, "write", "(Ljava/lang/Object;)V");
}

void CarbonWriter::write(jobject obj) {
jvalue args[1];
args[0].l = obj;
jniEnv->CallBooleanMethodA(carbonWriterObject, writeID, args);
};

jboolean CarbonWriter::close() {
jclass carbonWriter = jniEnv->GetObjectClass(carbonWriterObject);
jmethodID closeID = jniEnv->GetMethodID(carbonWriter, "close", "()V");
jniEnv->CallBooleanMethod(carbonWriterObject, closeID);
if (jniEnv->ExceptionCheck()) {
throw jniEnv->ExceptionOccurred();
}
}
111 changes: 111 additions & 0 deletions store/CSDK/src/CarbonWriter.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include <jni.h>

class CarbonWriter {
private:
/**
* jni env
*/
JNIEnv *jniEnv;

/**
* carbonWriterBuilder object for building carbonWriter
* it can configure some operation
*/
jobject carbonWriterBuilderObject;

/**
* carbonWriter object for writing data
*/
jobject carbonWriterObject;

/**
* carbon writer class
*/
jclass carbonWriter;

/**
* write method id
*/
jmethodID writeID;
public:
/**
* create a CarbonWriterBuilder object for building carbonWriter,
* CarbonWriterBuilder object can configure different parameter
*
* @param env JNIEnv
* @return CarbonWriterBuilder object
*/
void builder(JNIEnv *env);

/**
* Sets the output path of the writer builder
*
* @param path is the absolute path where output files are written
* This method must be called when building CarbonWriterBuilder
* @return updated CarbonWriterBuilder
*/
void outputPath(char *path);

/**
* configure the schema with json style schema
*
* @param jsonSchema json style schema
* @return updated CarbonWriterBuilder
*/
void withCsvInput(char *jsonSchema);

/**
* configure parameter, including ak,sk and endpoint
*
* @param key key word
* @param value value
* @return CarbonWriterBuilder object
*/
void withHadoopConf(char *key, char *value);

/**
* @param appName appName which is writing the carbondata files
*/
void writtenBy(char *appName);

/**
* build carbonWriter object for writing data
* it support write data from load disk
*
* @return carbonWriter object
*/
void build();

/**
* Write an object to the file, the format of the object depends on the
* implementation.
* Note: This API is not thread safe
*/
void write(jobject obj);

/**
* close the carbon Writer
*
* @return boolean value
*/
jboolean close();
};


Loading

0 comments on commit 8cd7d00

Please sign in to comment.