From 1de02bc141983773e229f3f96fe7b8559a2e265c Mon Sep 17 00:00:00 2001 From: xubo245 Date: Wed, 14 Nov 2018 17:03:17 +0800 Subject: [PATCH] [CARBONDATA-3097] Support folder path in getVersionDetails and support getVersionDetails in CSDK support get Verson from S3 optimize --- docs/csdk-guide.md | 27 ++++++ docs/sdk-guide.md | 15 ++++ store/CSDK/src/CarbonSchemaReader.cpp | 39 +++++++++ store/CSDK/src/CarbonSchemaReader.h | 23 ++++++ store/CSDK/test/main.cpp | 35 +++++++- .../sdk/file/CarbonSchemaReader.java | 34 +++++++- .../sdk/file/CarbonSchemaReaderTest.java | 82 +++++++++++++++++++ 7 files changed, 251 insertions(+), 4 deletions(-) diff --git a/docs/csdk-guide.md b/docs/csdk-guide.md index ef3f252bff2..644d52432e9 100644 --- a/docs/csdk-guide.md +++ b/docs/csdk-guide.md @@ -439,6 +439,33 @@ jobject readSchema(char *path, Configuration conf); jobject readSchema(char *path, bool validateSchema, Configuration conf); ``` +``` +/** + * This method return the version details in formatted string by reading from carbondata file + * + * @param path carbondata file path or folder path + * @return string with information of who has written this file + * in which carbondata project version + * @throws IOException + */ +char *getVersionDetails(char *path); +``` + +``` +/** + * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. + * + * @param path carbondata file path or folder path + * @param conf configuration support, can set s3a AK,SK, + * end point and other conf with this + * @return string with information of who has written this file + * in which carbondata project version + * @throws IOException + */ +char *getVersionDetails(char *path, Configuration conf); +``` + ### Schema ``` /** diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md index 573b5952418..8355940bf4b 100644 --- a/docs/sdk-guide.md +++ b/docs/sdk-guide.md @@ -862,6 +862,21 @@ public static Schema readSchema(String path, boolean validateSchema, Configurati * @throws IOException */ public static String getVersionDetails(String dataFilePath); +``` + + ``` +/** + * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. + * + * @param path carbondata file path or folder path + * @param conf configuration support, can set s3a AK,SK, + * end point and other conf with this + * @return string with information of who has written this file + * in which carbondata project version + * @throws IOException + */ +public static String getVersionDetails(String path, Configuration conf)' ``` ### Class org.apache.carbondata.sdk.file.Schema diff --git a/store/CSDK/src/CarbonSchemaReader.cpp b/store/CSDK/src/CarbonSchemaReader.cpp index 5fcd4271410..799ef047e3b 100644 --- a/store/CSDK/src/CarbonSchemaReader.cpp +++ b/store/CSDK/src/CarbonSchemaReader.cpp @@ -78,4 +78,43 @@ jobject CarbonSchemaReader::readSchema(char *path, bool validateSchema, Configur jobject CarbonSchemaReader::readSchema(char *path, bool validateSchema) { Configuration conf(jniEnv); return readSchema(path, validateSchema, conf); +} + +char *CarbonSchemaReader::getVersionDetails(char *path) { + if (path == NULL) { + throw std::runtime_error("path parameter can't be NULL."); + } + jmethodID methodID = jniEnv->GetStaticMethodID(carbonSchemaReaderClass, "getVersionDetails", + "(Ljava/lang/String;)Ljava/lang/String;"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: getVersionDetails"); + } + jstring jPath = jniEnv->NewStringUTF(path); + jvalue args[1]; + args[0].l = jPath; + jobject result = jniEnv->CallStaticObjectMethodA(carbonSchemaReaderClass, methodID, args); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } + return (char *) jniEnv->GetStringUTFChars((jstring) result, JNI_FALSE); +} + +char *CarbonSchemaReader::getVersionDetails(char *path, Configuration conf) { + if (path == NULL) { + throw std::runtime_error("path parameter can't be NULL."); + } + jmethodID methodID = jniEnv->GetStaticMethodID(carbonSchemaReaderClass, "getVersionDetails", + "(Ljava/lang/String;Lorg/apache/hadoop/conf/Configuration;)Ljava/lang/String;"); + if (methodID == NULL) { + throw std::runtime_error("Can't find the method in java: getVersionDetails"); + } + jstring jPath = jniEnv->NewStringUTF(path); + jvalue args[2]; + args[0].l = jPath; + args[1].l = conf.getConfigurationObject(); + jobject result = jniEnv->CallStaticObjectMethodA(carbonSchemaReaderClass, methodID, args); + if (jniEnv->ExceptionCheck()) { + throw jniEnv->ExceptionOccurred(); + } + return (char *) jniEnv->GetStringUTFChars((jstring) result, JNI_FALSE); } \ No newline at end of file diff --git a/store/CSDK/src/CarbonSchemaReader.h b/store/CSDK/src/CarbonSchemaReader.h index 5ea5f9f4c1f..3067d10456c 100644 --- a/store/CSDK/src/CarbonSchemaReader.h +++ b/store/CSDK/src/CarbonSchemaReader.h @@ -86,4 +86,27 @@ class CarbonSchemaReader { */ jobject readSchema(char *path, bool validateSchema); + /** + * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. + * + * @param path carbondata file path or folder path + * @return string with information of who has written this file + * in which carbondata project version + * @throws IOException + */ + char *getVersionDetails(char *path); + + /** + * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. + * + * @param path carbondata file path or folder path + * @param conf configuration support, can set s3a AK,SK, + * end point and other conf with this + * @return string with information of who has written this file + * in which carbondata project version + * @throws IOException + */ + char *getVersionDetails(char *path, Configuration conf); }; \ No newline at end of file diff --git a/store/CSDK/test/main.cpp b/store/CSDK/test/main.cpp index 10a659968a7..bc3122c9d23 100644 --- a/store/CSDK/test/main.cpp +++ b/store/CSDK/test/main.cpp @@ -236,6 +236,36 @@ bool tryCarbonRowException(JNIEnv *env, char *path) { printResultWithException(env, carbonReaderClass); } +/* + * test get Version Details from path + * + * @param env jni env + * @return whether it is success + */ +bool getVersionDetails(JNIEnv *env, char *Path, char **argv, int argc) { + printf("\nget version details from path:\n"); + try { + Configuration conf(env); + if (argc > 3) { + conf.set("fs.s3a.access.key", argv[1]); + conf.set("fs.s3a.secret.key", argv[2]); + conf.set("fs.s3a.endpoint", argv[3]); + } + CarbonSchemaReader carbonSchemaReader(env); + char *version; + if (argc > 2) { + version = carbonSchemaReader.getVersionDetails(Path, conf); + printf("\n%s\t\n", version); + } else { + version = carbonSchemaReader.getVersionDetails(Path); + printf("\n%s\t\n", version); + } + } catch (jthrowable e) { + env->ExceptionDescribe(); + } + return true; +} + /** * test read data from local disk, without projection * @@ -844,9 +874,10 @@ int main(int argc, char *argv[]) { testWriteData(env, S3WritePath, 4, argv); readSchema(env, S3WritePath, true, argv,4); readSchema(env, S3WritePath, false, argv, 4); - readFromS3(env, S3ReadPath, argv); +// readFromS3(env, S3ReadPath, argv); testWithTableProperty(env, "s3a://csdk/dataProperty", 4, argv); testSortBy(env, "s3a://csdk/dataSort", 4, argv); + getVersionDetails(env, S3WritePath, argv, 4); testReadNextRow(env, S3Path, 100000, argv, 4, false); testReadNextRow(env, S3Path, 100000, argv, 4, true); @@ -856,6 +887,7 @@ int main(int argc, char *argv[]) { int batch = 32000; int printNum = 32000; + char *writePath = "./data"; tryCatchException(env); tryCarbonRowException(env, smallFilePath); testCarbonProperties(env); @@ -867,6 +899,7 @@ int main(int argc, char *argv[]) { testSortBy(env, "./dataSort", 1, argv); readSchema(env, path, false, argv, 1); readSchema(env, path, true, argv, 1); + getVersionDetails(env, writePath, argv, 1); testReadNextRow(env, path, printNum, argv, 0, true); testReadNextRow(env, path, printNum, argv, 0, false); diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java index cde609b4c53..00029fc0cfd 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonSchemaReader.java @@ -285,12 +285,40 @@ private static Schema readSchemaFromIndexFile(String indexFilePath, Configuratio /** * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. * - * @param dataFilePath - * @return + * @param path carbondata file path or folder path + * @param conf configuration support, can set s3a AK,SK, + * end point and other conf with this + * @return string with information of who has written this file + * in which carbondata project version * @throws IOException */ - public static String getVersionDetails(String dataFilePath) throws IOException { + public static String getVersionDetails(String path, Configuration conf) throws IOException { + if (path.endsWith(INDEX_FILE_EXT)) { + throw new RuntimeException("Can't get version details from carbonindex file."); + } else if (path.endsWith(CARBON_DATA_EXT)) { + return getVersionDetailsFromDataFile(path); + } else { + String dataFilePath = getCarbonFile(path, CARBON_DATA_EXT, conf)[0].getAbsolutePath(); + return getVersionDetailsFromDataFile(dataFilePath); + } + } + + /** + * This method return the version details in formatted string by reading from carbondata file + * default won't validate the version details between different carbondata files. + * + * @param path carbondata file path or folder path + * @return string with information of who has written this file + * in which carbondata project version + * @throws IOException + */ + public static String getVersionDetails(String path) throws IOException { + return getVersionDetails(path, new Configuration()); + } + + private static String getVersionDetailsFromDataFile(String dataFilePath) throws IOException { long fileSize = FileFactory.getCarbonFile(dataFilePath, FileFactory.getFileType(dataFilePath)).getSize(); FileReader fileReader = FileFactory.getFileHolder(FileFactory.getFileType(dataFilePath)); diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonSchemaReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonSchemaReaderTest.java index 1e70ada17e6..8d21ad10f81 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonSchemaReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonSchemaReaderTest.java @@ -18,6 +18,7 @@ package org.apache.carbondata.sdk.file; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -35,13 +36,19 @@ import org.junit.Before; import org.junit.Test; +import static org.apache.carbondata.core.util.path.CarbonTablePath.CARBON_DATA_EXT; +import static org.apache.carbondata.core.util.path.CarbonTablePath.INDEX_FILE_EXT; + public class CarbonSchemaReaderTest extends TestCase { String path = "./testWriteFiles"; @Before public void setUp() throws IOException, InvalidLoadOptionException { FileUtils.deleteDirectory(new File(path)); + writeData(); + } + private void writeData() throws IOException, InvalidLoadOptionException { Field[] fields = new Field[12]; fields[0] = new Field("stringField", DataTypes.STRING); fields[1] = new Field("shortField", DataTypes.SHORT); @@ -236,6 +243,81 @@ public void testReadSchemaWithDifferentSchema() { } } + @Test + public void testGetVersionDetailsAndValidate() { + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithoutValidate() { + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithCarbonDataFile() { + try { + File[] dataFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(CARBON_DATA_EXT); + } + }); + String versionDetails = CarbonSchemaReader.getVersionDetails(dataFiles[0].getAbsolutePath()); + Assert.assertTrue(versionDetails.contains("CarbonSchemaReaderTest in version: ")); + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + + @Test + public void testGetVersionDetailsWithCarbonIndexFile() { + try { + File[] indexFiles = new File(path).listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(INDEX_FILE_EXT); + } + }); + CarbonSchemaReader.getVersionDetails(indexFiles[0].getAbsolutePath()); + Assert.fail(); + } catch (Throwable e) { + Assert.assertTrue(e.getMessage() + .equalsIgnoreCase("Can't get version details from carbonindex file.")); + } + } + + public void testGetVersionDetailsWithTheSameSchema() { + try { + writeData(); + try { + String versionDetails = CarbonSchemaReader + .getVersionDetails(path); + Assert.assertTrue(versionDetails + .contains("CarbonSchemaReaderTest in version: ")); + } catch (Exception e) { + Assert.fail(); + } + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(); + } + } + @After public void tearDown() throws IOException { FileUtils.deleteDirectory(new File(path));