From 5f4115bf64ed4d9fc2b0ae8dc667937f18e5fbb0 Mon Sep 17 00:00:00 2001 From: xubo245 Date: Sat, 27 Oct 2018 12:10:31 +0800 Subject: [PATCH] optimize --- .../carbondata/examples/sdk/SDKS3Example.java | 13 +- store/CSDK/src/CarbonWriter.cpp | 91 ++++++- store/CSDK/src/CarbonWriter.h | 11 +- store/CSDK/test/main.cpp | 240 +++++++++--------- .../sdk/file/CSVCarbonWriterTest.java | 3 +- 5 files changed, 227 insertions(+), 131 deletions(-) diff --git a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java index 245d3e89a43..1aad7c80306 100644 --- a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java +++ b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java @@ -52,7 +52,7 @@ public static void main(String[] args) throws Exception { CarbonProperties.getInstance() .addProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_DIRECT_WRITE_TO_STORE_PATH, "true"); - String path = "s3a://sdk/WriterOutput"; + String path = "s3a://sdk/WriterOutput4"; if (args.length > 3) { path=args[3]; } @@ -62,7 +62,7 @@ public static void main(String[] args) throws Exception { num = Integer.parseInt(args[4]); } - Configuration conf = new Configuration(false); + Configuration conf = new Configuration(true); conf.set(Constants.ACCESS_KEY, args[0]); conf.set(Constants.SECRET_KEY, args[1]); conf.set(Constants.ENDPOINT, args[2]); @@ -70,8 +70,13 @@ public static void main(String[] args) throws Exception { Field[] fields = new Field[2]; fields[0] = new Field("name", DataTypes.STRING); fields[1] = new Field("age", DataTypes.INT); - CarbonWriterBuilder builder = CarbonWriter.builder().outputPath(path).withHadoopConf(conf); - CarbonWriter writer = builder.withCsvInput(new Schema(fields)).build(); + CarbonWriter writer = CarbonWriter + .builder() + .outputPath(path) + .withHadoopConf(conf) + .withCsvInput(new Schema(fields)) + .writtenBy("SDKS3Example") + .build(); for (int i = 0; i < num; i++) { writer.write(new String[]{"robot" + (i % 10), String.valueOf(i)}); diff --git a/store/CSDK/src/CarbonWriter.cpp b/store/CSDK/src/CarbonWriter.cpp index 2f2419c9da7..05497c66a72 100644 --- a/store/CSDK/src/CarbonWriter.cpp +++ b/store/CSDK/src/CarbonWriter.cpp @@ -15,41 +15,83 @@ * limitations under the License. */ +#include #include "CarbonWriter.h" void CarbonWriter::builder(JNIEnv *env) { + if (env == NULL) { + throw std::runtime_error("JNIEnv parameter can't be NULL."); + } jniEnv = env; carbonWriter = env->FindClass("org/apache/carbondata/sdk/file/CarbonWriter"); + if (carbonWriter == NULL) { + throw std::runtime_error("Can't find the class in java: org/apache/carbondata/sdk/file/CarbonWriter"); + } jmethodID carbonWriterBuilderID = env->GetStaticMethodID(carbonWriter, "builder", "()Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;"); + if (carbonWriterBuilderID == NULL) { + throw std::runtime_error("Can't find the method in java: carbonWriterBuilder"); + } carbonWriterBuilderObject = env->CallStaticObjectMethod(carbonWriter, carbonWriterBuilderID); } +bool CarbonWriter::checkBuilder() { + if (carbonWriterBuilderObject == NULL) { + throw std::runtime_error("carbonWriterBuilder Object can't be NULL. Please call builder method first."); + } +} + void CarbonWriter::outputPath(char *path) { + if (path == NULL) { + throw std::runtime_error("path parameter can't be NULL."); + } + checkBuilder(); jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject); - jmethodID carbonWriterBuilderID = jniEnv->GetMethodID(carbonWriterBuilderClass, "outputPath", + jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "outputPath", "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: outputPath"); + } jstring jPath = jniEnv->NewStringUTF(path); jvalue args[1]; args[0].l = jPath; - carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, carbonWriterBuilderID, args); + carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args); } void CarbonWriter::withCsvInput(char *jsonSchema) { - + if (jsonSchema == NULL) { + throw std::runtime_error("jsonSchema parameter can't be NULL."); + } + checkBuilder(); jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject); - jmethodID carbonWriterBuilderID = jniEnv->GetMethodID(carbonWriterBuilderClass, "withCsvInput", + jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "withCsvInput", "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: withCsvInput"); + } jstring jPath = jniEnv->NewStringUTF(jsonSchema); jvalue args[1]; args[0].l = jPath; - carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, carbonWriterBuilderID, args); + carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } }; void CarbonWriter::withHadoopConf(char *key, char *value) { + if (key == NULL) { + throw std::runtime_error("key parameter can't be NULL."); + } + if (value == NULL) { + throw std::runtime_error("value parameter can't be NULL."); + } + checkBuilder(); jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject); jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "withHadoopConf", "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: withHadoopConf"); + } jvalue args[2]; args[0].l = jniEnv->NewStringUTF(key); args[1].l = jniEnv->NewStringUTF(value); @@ -57,33 +99,62 @@ void CarbonWriter::withHadoopConf(char *key, char *value) { } void CarbonWriter::writtenBy(char *appName) { + checkBuilder(); jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject); jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "writtenBy", - "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;"); + "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonWriterBuilder;"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: writtenBy"); + } jvalue args[1]; args[0].l = jniEnv->NewStringUTF(appName); carbonWriterBuilderObject = jniEnv->CallObjectMethodA(carbonWriterBuilderObject, methodID, args); } void CarbonWriter::build() { + checkBuilder(); jclass carbonWriterBuilderClass = jniEnv->GetObjectClass(carbonWriterBuilderObject); jmethodID methodID = jniEnv->GetMethodID(carbonWriterBuilderClass, "build", "()Lorg/apache/carbondata/sdk/file/CarbonWriter;"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: build"); + } carbonWriterObject = jniEnv->CallObjectMethod(carbonWriterBuilderObject, methodID); - carbonWriter = jniEnv->GetObjectClass(carbonWriterObject); - writeID = jniEnv->GetMethodID(carbonWriter, "write", "(Ljava/lang/Object;)V"); + + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } } void CarbonWriter::write(jobject obj) { + if (carbonWriterObject == NULL) { + throw std::runtime_error("Please call build first."); + } + if (writeID == NULL) { + carbonWriter = jniEnv->GetObjectClass(carbonWriterObject); + writeID = jniEnv->GetMethodID(carbonWriter, "write", "(Ljava/lang/Object;)V"); + if (writeID == NULL) { + throw std::runtime_error("Can't find the method in java: write"); + } + } jvalue args[1]; args[0].l = obj; jniEnv->CallBooleanMethodA(carbonWriterObject, writeID, args); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } }; jboolean CarbonWriter::close() { + if (carbonWriterObject == NULL) { + throw std::runtime_error("Please call build first."); + } jclass carbonWriter = jniEnv->GetObjectClass(carbonWriterObject); - jmethodID closeID = jniEnv->GetMethodID(carbonWriter, "close", "()V"); - jniEnv->CallBooleanMethod(carbonWriterObject, closeID); + jmethodID methodID = jniEnv->GetMethodID(carbonWriter, "close", "()V"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: close"); + } + jniEnv->CallBooleanMethod(carbonWriterObject, methodID); if (jniEnv->ExceptionCheck()) { throw jniEnv->ExceptionOccurred(); } diff --git a/store/CSDK/src/CarbonWriter.h b/store/CSDK/src/CarbonWriter.h index b954849dba5..1ea5baa5367 100644 --- a/store/CSDK/src/CarbonWriter.h +++ b/store/CSDK/src/CarbonWriter.h @@ -28,7 +28,7 @@ class CarbonWriter { * carbonWriterBuilder object for building carbonWriter * it can configure some operation */ - jobject carbonWriterBuilderObject; + jobject carbonWriterBuilderObject = NULL; /** * carbonWriter object for writing data @@ -43,7 +43,14 @@ class CarbonWriter { /** * write method id */ - jmethodID writeID; + jmethodID writeID = NULL; + + /** + * check whether has called builder + * + * @return true or throw exception + */ + bool checkBuilder(); public: /** * create a CarbonWriterBuilder object for building carbonWriter, diff --git a/store/CSDK/test/main.cpp b/store/CSDK/test/main.cpp index 15fd49aa069..2766e5ccf70 100644 --- a/store/CSDK/test/main.cpp +++ b/store/CSDK/test/main.cpp @@ -217,123 +217,135 @@ bool tryCatchException(JNIEnv *env) { * test write data to local disk * * @param env jni env - * @return + * @param path file path + * @param argc argument counter + * @param argv argument vector + * @return true or throw exception */ -bool writeToLocal(JNIEnv *env, char *path) { +bool testWriteData(JNIEnv *env, char *path, int argc, char *argv[]) { char *jsonSchema = "[{stringField:string},{shortField:short},{intField:int},{longField:long},{doubleField:double},{boolField:boolean},{dateField:date},{timeField:timestamp},{floatField:float},{arrayField:array}]"; - - CarbonWriter carbonWriterClass; - carbonWriterClass.builder(env); - carbonWriterClass.outputPath(path); - carbonWriterClass.withCsvInput(jsonSchema); - carbonWriterClass.writtenBy("CSDK"); - carbonWriterClass.build(); - - int rowNum = 10; - int size = 10; - long longValue = 0; - double doubleValue = 0; - float floatValue = 0; - jclass objClass = env->FindClass("java/lang/String"); - for (int i = 0; i < rowNum; ++i) { - jobjectArray arr = env->NewObjectArray(size, objClass, 0); - char ctrInt[10]; - gcvt(i, 10, ctrInt); - - char a[15] = "robot"; - strcat(a, ctrInt); - jobject stringField = env->NewStringUTF(a); - env->SetObjectArrayElement(arr, 0, stringField); - - char ctrShort[10]; - gcvt(i % 10000, 10, ctrShort); - jobject shortField = env->NewStringUTF(ctrShort); - env->SetObjectArrayElement(arr, 1, shortField); - - jobject intField = env->NewStringUTF(ctrInt); - env->SetObjectArrayElement(arr, 2, intField); - - - char ctrLong[10]; - gcvt(longValue, 10, ctrLong); - longValue = longValue + 2; - jobject longField = env->NewStringUTF(ctrLong); - env->SetObjectArrayElement(arr, 3, longField); - - char ctrDouble[10]; - gcvt(doubleValue, 10, ctrDouble); - doubleValue = doubleValue + 2; - jobject doubleField = env->NewStringUTF(ctrDouble); - env->SetObjectArrayElement(arr, 4, doubleField); - - jobject boolField = env->NewStringUTF("true"); - env->SetObjectArrayElement(arr, 5, boolField); - - jobject dateField = env->NewStringUTF(" 2019-03-02"); - env->SetObjectArrayElement(arr, 6, dateField); - - jobject timeField = env->NewStringUTF("2019-02-12 03:03:34"); - env->SetObjectArrayElement(arr, 7, timeField); - - char ctrFloat[10]; - gcvt(floatValue, 10, ctrFloat); - floatValue = floatValue + 2; - jobject floatField = env->NewStringUTF(ctrFloat); - env->SetObjectArrayElement(arr, 8, floatField); - - jobject arrayField = env->NewStringUTF("Hello#World#From#Carbon"); - env->SetObjectArrayElement(arr, 9, arrayField); - - - carbonWriterClass.write(arr); - - env->DeleteLocalRef(stringField); - env->DeleteLocalRef(shortField); - env->DeleteLocalRef(intField); - env->DeleteLocalRef(longField); - env->DeleteLocalRef(doubleField); - env->DeleteLocalRef(floatField); - env->DeleteLocalRef(dateField); - env->DeleteLocalRef(timeField); - env->DeleteLocalRef(boolField); - env->DeleteLocalRef(arrayField); - env->DeleteLocalRef(arr); - } - carbonWriterClass.close(); - - CarbonReader carbonReader; - carbonReader.builder(env, path); - carbonReader.build(); - int i = 0; - CarbonRow carbonRow(env); - while (carbonReader.hasNext()) { - jobject row = carbonReader.readNextRow(); - i++; - carbonRow.setCarbonRow(row); - printf("%s\t%d\t%ld\t", carbonRow.getString(0), carbonRow.getInt(1), carbonRow.getLong(2)); - jobjectArray array1 = carbonRow.getArray(3); - jsize length = env->GetArrayLength(array1); - int j = 0; - for (j = 0; j < length; j++) { - jobject element = env->GetObjectArrayElement(array1, j); - char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); - printf("%s\t", str); + try { + CarbonWriter writer; + writer.builder(env); + if (argc > 3) { + writer.withHadoopConf("fs.s3a.access.key", argv[1]); + writer.withHadoopConf("fs.s3a.secret.key", argv[2]); + writer.withHadoopConf("fs.s3a.endpoint", argv[3]); } - printf("%d\t", carbonRow.getShort(4)); - printf("%d\t", carbonRow.getInt(5)); - printf("%ld\t", carbonRow.getLong(6)); - printf("%lf\t", carbonRow.getDouble(7)); - bool bool1 = carbonRow.getBoolean(8); - if (bool1) { - printf("true\t"); - } else { - printf("false\t"); + writer.outputPath(path); + writer.withCsvInput(jsonSchema); + writer.writtenBy("CSDK"); + writer.build(); + + int rowNum = 10; + int size = 10; + long longValue = 0; + double doubleValue = 0; + float floatValue = 0; + jclass objClass = env->FindClass("java/lang/String"); + for (int i = 0; i < rowNum; ++i) { + jobjectArray arr = env->NewObjectArray(size, objClass, 0); + char ctrInt[10]; + gcvt(i, 10, ctrInt); + + char a[15] = "robot"; + strcat(a, ctrInt); + jobject stringField = env->NewStringUTF(a); + env->SetObjectArrayElement(arr, 0, stringField); + + char ctrShort[10]; + gcvt(i % 10000, 10, ctrShort); + jobject shortField = env->NewStringUTF(ctrShort); + env->SetObjectArrayElement(arr, 1, shortField); + + jobject intField = env->NewStringUTF(ctrInt); + env->SetObjectArrayElement(arr, 2, intField); + + + char ctrLong[10]; + gcvt(longValue, 10, ctrLong); + longValue = longValue + 2; + jobject longField = env->NewStringUTF(ctrLong); + env->SetObjectArrayElement(arr, 3, longField); + + char ctrDouble[10]; + gcvt(doubleValue, 10, ctrDouble); + doubleValue = doubleValue + 2; + jobject doubleField = env->NewStringUTF(ctrDouble); + env->SetObjectArrayElement(arr, 4, doubleField); + + jobject boolField = env->NewStringUTF("true"); + env->SetObjectArrayElement(arr, 5, boolField); + + jobject dateField = env->NewStringUTF(" 2019-03-02"); + env->SetObjectArrayElement(arr, 6, dateField); + + jobject timeField = env->NewStringUTF("2019-02-12 03:03:34"); + env->SetObjectArrayElement(arr, 7, timeField); + + char ctrFloat[10]; + gcvt(floatValue, 10, ctrFloat); + floatValue = floatValue + 2; + jobject floatField = env->NewStringUTF(ctrFloat); + env->SetObjectArrayElement(arr, 8, floatField); + + jobject arrayField = env->NewStringUTF("Hello#World#From#Carbon"); + env->SetObjectArrayElement(arr, 9, arrayField); + + + writer.write(arr); + + env->DeleteLocalRef(stringField); + env->DeleteLocalRef(shortField); + env->DeleteLocalRef(intField); + env->DeleteLocalRef(longField); + env->DeleteLocalRef(doubleField); + env->DeleteLocalRef(floatField); + env->DeleteLocalRef(dateField); + env->DeleteLocalRef(timeField); + env->DeleteLocalRef(boolField); + env->DeleteLocalRef(arrayField); + env->DeleteLocalRef(arr); } - printf("%f\t\n", carbonRow.getFloat(9)); - env->DeleteLocalRef(row); + writer.close(); + + CarbonReader carbonReader; + carbonReader.builder(env, path); + carbonReader.build(); + int i = 0; + CarbonRow carbonRow(env); + while (carbonReader.hasNext()) { + jobject row = carbonReader.readNextRow(); + i++; + carbonRow.setCarbonRow(row); + printf("%s\t%d\t%ld\t", carbonRow.getString(0), carbonRow.getInt(1), carbonRow.getLong(2)); + jobjectArray array1 = carbonRow.getArray(3); + jsize length = env->GetArrayLength(array1); + int j = 0; + for (j = 0; j < length; j++) { + jobject element = env->GetObjectArrayElement(array1, j); + char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE); + printf("%s\t", str); + } + printf("%d\t", carbonRow.getShort(4)); + printf("%d\t", carbonRow.getInt(5)); + printf("%ld\t", carbonRow.getLong(6)); + printf("%lf\t", carbonRow.getDouble(7)); + bool bool1 = carbonRow.getBoolean(8); + if (bool1) { + printf("true\t"); + } else { + printf("false\t"); + } + printf("%f\t\n", carbonRow.getFloat(9)); + env->DeleteLocalRef(row); + } + carbonReader.close(); + } catch (jthrowable ex) { + env->ExceptionDescribe(); + env->ExceptionClear(); } - carbonReader.close(); } /** @@ -522,15 +534,15 @@ int main(int argc, char *argv[]) { // init jvm JNIEnv *env; env = initJVM(); - char *S3WritePath = "s3a://sdk/ges/write2"; + char *S3WritePath = "s3a://sdk/csdk/"; if (argc > 3) { readFromS3(env, argv); - writeToS3(env, S3WritePath, argv); + testWriteData(env, S3WritePath, 4, argv); } else { tryCatchException(env); readFromLocalWithoutProjection(env); - writeToLocal(env, "./data"); + testWriteData(env, "./data", 1, argv); readFromLocal(env); } (jvm)->DestroyJavaVM(); diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java index d066fce3c37..90ea9095fb2 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java @@ -113,7 +113,8 @@ public void testWriteFilesBuildWithJsonSchema() throws IOException, InvalidLoadO CarbonWriterBuilder builder = CarbonWriter .builder() .outputPath(path) - .withCsvInput(schema); + .withCsvInput(schema) + .writtenBy("testWriteFilesBuildWithJsonSchema"); CarbonWriter writer = builder.build(); for (int i = 0; i < 10; i++) {