diff --git a/pom.xml b/pom.xml index e1ff256c..a57f3ff5 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,7 @@ nlp xgboost mixserv + systemtest diff --git a/systemtest/README.md b/systemtest/README.md new file mode 100644 index 00000000..2b1167e2 --- /dev/null +++ b/systemtest/README.md @@ -0,0 +1,211 @@ + +## Usage + +### Initialization + +Define `CommonInfo`, `Runner` and `Team` in each your test class. + +#### `CommonInfo` + +* `SystemTestCommonInfo` + +`CommonInfo` holds common information of test class, for example, +you can refer to auto-defined path to resources. This should be defined as `private static`. + + +#### `Runner` + +* `HiveSystemTestRunner` +* `TDSystemTestRunner` + +`Runner` represents a test environment and its configuration. This must be defined with `@ClassRule` +as `public static` because of JUnit spec. You can add test class initializations by `#initBy(...)` +with class methods of `HQ`, which are abstract domain-specific hive queries, in instance initializer +of each `Runner`. + + +#### `Team` + +* `SystemTestTeam` + +`Team` manages `Runner`s each test method. This must be defined with `@Rule` as `public` because of +JUnit spec. You can set `Runner`s via constructor argument as common in class and via `#add(...)` +as method-local and add test method initializations by `#initBy(...)` and test case by `#set(...)` +with class methods of `HQ`. Finally, don't forget call `#run()` to enable set `HQ`s. +As an alternative method, by `#set(HQ.autoMatchingByFileName(${filename}))` with queries predefined in +`auto-defined/path/init/${filename}`, `auto-defined/path/case/${filename}` and +`auto-defined/path/answer/${filename}`, you can do auto matching test. + + +### External properties + +You can use external properties at `systemtest/src/test/resources/hivemall/*`, default is `hiverunner.properties` +for `HiveSystemTestRunner` and `td.properties` for `TDSystemTestRunner`. Also user-defined properties file can +be loaded via constructor of `Runner` by file name. + + +## Notice + +* DDL and insert statement should be called via class methods of `HQ` because of wrapping hive queries +and several runner-specific APIs, don't call them via string statement +* Also you can use low-level API via an instance of `Runner`, independent of `Team` +* You can use `IO.getFromResourcePath(...)` to get answer whose format is TSV +* Table created in initialization of runner should be used as immutable, don't neither insert nor update +* TD client configs in properties file prior to $HOME/.td/td.conf +* Don't use insert w/ big data, use file upload instead + +## Quick example + +```java +package hivemall; +// here is several imports +public class QuickExample { + private static SystemTestCommonInfo ci = new SystemTestCommonInfo(QuickExample.class); + + @ClassRule + public static HiveSystemTestRunner hRunner = new HiveSystemTestRunner(ci) { + { + initBy(HQ.uploadByResourcePathAsNewTable("color", ci.initDir + "color.tsv", + new LinkedHashMap() { + { + put("name", "string"); + put("red", "int"); + put("green", "int"); + put("blue", "int"); + } + })); // create table `color`, which is marked as immutable, for this test class + + // add function from hivemall class + initBy(HQ.fromStatement("CREATE TEMPORARY FUNCTION hivemall_version as 'hivemall.HivemallVersionUDF'")); + } + }; + + @ClassRule + public static TDSystemTestRunner tRunner = new TDSystemTestRunner(ci) { + { + initBy(HQ.uploadByResourcePathAsNewTable("color", ci.initDir + "color.tsv", + new LinkedHashMap() { + { + put("name", "string"); + put("red", "int"); + put("green", "int"); + put("blue", "int"); + } + })); // create table `color`, which is marked as immutable, for this test class + } + }; + + @Rule + public SystemTestTeam team = new SystemTestTeam(hRunner); // set hRunner as default runner + + @Rule + public ExpectedException predictor = ExpectedException.none(); + + + @Test + public void test0() throws Exception { + team.add(tRunner, hRunner); // test on HiveRunner -> TD -> HiveRunner (NOTE: state of DB is retained in each runner) + team.set(HQ.fromStatement("SELECT name FROM color WHERE blue = 255 ORDER BY name"), "azure\tblue\tmagenta", true); // ordered test + team.run(); // this call is required + } + + @Test + public void test1() throws Exception { + // test on HiveRunner once only + String tableName = "users"; + team.initBy(HQ.createTable(tableName, new LinkedHashMap() { + { + put("name", "string"); + put("age", "int"); + put("favorite_color", "string"); + } + })); // create local table in this test method `users` for each set runner(only hRunner here) + team.initBy(HQ.insert(tableName, Arrays.asList("name", "age", "favorite_color"), Arrays.asList( + new Object[]{"Karen", 16, "orange"}, new Object[]{"Alice", 17, "pink"}))); // insert into `users` + team.set(HQ.fromStatement("SELECT CONCAT('rgb(', red, ',', green, ',', blue, ')') FROM " + + tableName + " u LEFT JOIN color c on u.favorite_color = c.name"), "rgb(255,165,0)\trgb(255,192,203)"); // unordered test + team.run(); // this call is required + } + + @Test + public void test2() throws Exception { + // You can also use runner's raw API directly + for(RawHQ q: HQ.fromStatements("SELECT hivemall_version();SELECT hivemall_version();")) { + System.out.println(hRunner.exec(q).get(0)); + } + // raw API doesn't require `SystemTestTeam#run()` + } + + @Test + public void test3() throws Exception { + // test on HiveRunner once only + // auto matching by files which name is `test3` in `case/` and `answer/` + team.set(HQ.autoMatchingByFileName("test3"), ci); // unordered test + team.run(); // this call is required + } + + @Test + public void test4() throws Exception { + // test on HiveRunner once only + predictor.expect(Throwable.class); // you can use systemtest w/ other rules + team.set(HQ.fromStatement("invalid queryyy"), "never used"); // this query throws an exception + team.run(); // this call is required + // thrown exception will be caught by `ExpectedException` rule + } +} +``` + +The above requires following files + +* `systemtest/src/test/resources/hivemall/QuickExample/init/color.tsv` (`systemtest/src/test/resources/${path/to/package}/${className}/init/${fileName}`) + +```tsv +blue 0 0 255 +lavender 230 230 250 +magenta 255 0 255 +violet 238 130 238 +purple 128 0 128 +azure 240 255 255 +lightseagreen 32 178 170 +orange 255 165 0 +orangered 255 69 0 +red 255 0 0 +pink 255 192 203 +``` + +* `systemtest/src/test/resources/hivemall/QuickExample/case/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/case/${fileName}`) + +```sql +-- write your hive queries +-- comments like this and multiple queries in one row are allowed +SELECT blue FROM color WHERE name = 'lavender'; +SELECT green FROM color WHERE name LIKE 'orange%'; +SELECT name FROM color WHERE blue = 255; +``` + +* `systemtest/src/test/resources/hivemall/QuickExample/answer/test3` (`systemtest/src/test/resources/${path/to/package}/${className}/answer/${fileName}`) + +tsv format is required + +```tsv +250 +165 69 +azure blue magenta +``` diff --git a/systemtest/pom.xml b/systemtest/pom.xml new file mode 100644 index 00000000..0debee03 --- /dev/null +++ b/systemtest/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + + io.github.myui + hivemall + 0.4.2-rc.2 + ../pom.xml + + + hivemall-systemtest + System test for Hivemall + jar + + + + io.github.myui + hivemall-core + 0.4.2-rc.2 + + + junit + junit + 4.12 + + + com.klarna + hiverunner + 3.0.0 + + + com.treasuredata.client + td-client + 0.7.25 + jar-with-dependencies + + + org.apache.commons + commons-csv + 1.1 + + + org.msgpack + msgpack-core + 0.8.9 + + + org.hamcrest + hamcrest-library + 1.3 + + + + + target + target/classes + ${project.artifactId}-${project.version} + target/test-classes + + + + com.mycila + license-maven-plugin + 2.8 + +
${project.parent.basedir}/resources/license-header.txt
+ + ${build.year} + ${project.organization.name} + + + src/main/**/*.java + src/test/**/*.java + + UTF-8 + + ${project.parent.basedir}/resources/header-definition.xml + + + + + + + + diff --git a/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java new file mode 100644 index 00000000..f7f372ff --- /dev/null +++ b/systemtest/src/main/java/com/klarna/hiverunner/Extractor.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.klarna.hiverunner; + +import com.klarna.hiverunner.config.HiveRunnerConfig; +import org.junit.rules.TemporaryFolder; + +public class Extractor { + public static StandaloneHiveServerContext getStandaloneHiveServerContext( + TemporaryFolder basedir, HiveRunnerConfig hiveRunnerConfig) { + return new StandaloneHiveServerContext(basedir, hiveRunnerConfig); + } + + public static HiveServerContainer getHiveServerContainer(HiveServerContext context) { + return new HiveServerContainer(context); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java b/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java new file mode 100644 index 00000000..b86c1cf8 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/MsgpackConverter.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest; + +import hivemall.utils.lang.Preconditions; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessagePacker; +import org.msgpack.value.ValueFactory; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.util.List; +import java.util.zip.GZIPOutputStream; + +public class MsgpackConverter { + @Nonnull + private final File file; + @Nonnull + private final List header; + @Nonnull + private final CSVFormat format; + + public MsgpackConverter(@CheckForNull File file, @CheckForNull List header, + @CheckForNull CSVFormat format) { + Preconditions.checkNotNull(file); + Preconditions.checkNotNull(header); + Preconditions.checkNotNull(format); + Preconditions.checkArgument(file.exists(), "%s not found", file.getPath()); + + this.file = file; + this.header = header; + this.format = format; + } + + public byte[] asByteArray(final boolean needTimeColumn) throws Exception { + final ByteArrayOutputStream os = new ByteArrayOutputStream(); + final MessagePacker packer = MessagePack.newDefaultPacker(new GZIPOutputStream(os)); + final BufferedReader br = new BufferedReader(new FileReader(file)); + try { + // always skip header, use user-defined or existing table's + final CSVParser parser = format.withSkipHeaderRecord().parse(br); + final long time = System.currentTimeMillis() / 1000; + for (CSVRecord record : parser.getRecords()) { + final ValueFactory.MapBuilder map = ValueFactory.newMapBuilder(); + + // add `time` column if needed && not exists + if (needTimeColumn && !header.contains("time")) { + map.put(ValueFactory.newString("time"), ValueFactory.newInteger(time)); + } + + // pack each value in row + int i = 0; + for (String val : record) { + map.put(ValueFactory.newString(header.get(i)), ValueFactory.newString(val)); + i++; + } + packer.packValue(map.build()); + } + } finally { + br.close(); + packer.close(); + } + + return os.toByteArray(); + } + + public byte[] asByteArray() throws Exception { + return asByteArray(true); + } + + public File asFile(@CheckForNull File to, final boolean needTimeColumn) throws Exception { + Preconditions.checkNotNull(to); + Preconditions.checkArgument(to.exists(), "%s not found", to.getPath()); + + FileOutputStream os = null; + try { + os = new FileOutputStream(to); + os.write(asByteArray(needTimeColumn)); + return to; + } finally { + if (os != null) { + os.close(); + } + } + } + + public File asFile(File to) throws Exception { + return asFile(to, true); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java b/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java new file mode 100644 index 00000000..c2b00344 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/exception/QueryExecutionException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.exception; + +import javax.annotation.Nonnull; + +public class QueryExecutionException extends RuntimeException { + public QueryExecutionException(@Nonnull final String message) { + super(message); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java new file mode 100644 index 00000000..e0047a65 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/CreateTableHQ.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import javax.annotation.Nonnull; +import java.util.LinkedHashMap; +import java.util.Map; + +public class CreateTableHQ extends TableHQ { + @Nonnull + public final LinkedHashMap header; + + CreateTableHQ(@Nonnull final String tableName, + @Nonnull final LinkedHashMap header) { + super(tableName); + + this.header = header; + } + + public String getTableDeclaration() { + final StringBuilder sb = new StringBuilder(); + sb.append("("); + for (Map.Entry e : header.entrySet()) { + sb.append(e.getKey()); + sb.append(" "); + sb.append(e.getValue()); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java new file mode 100644 index 00000000..c09ae247 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/DropTableHQ.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import javax.annotation.Nonnull; + +public class DropTableHQ extends TableHQ { + DropTableHQ(@Nonnull final String tableName) { + super(tableName); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQ.java b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java new file mode 100644 index 00000000..05933a4c --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQ.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import com.google.common.io.Resources; +import com.klarna.hiverunner.CommandShellEmulation; +import com.klarna.hiverunner.sql.StatementsSplitter; +import hivemall.systemtest.model.lazy.LazyMatchingResource; +import hivemall.utils.lang.Preconditions; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.io.File; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; + +/** + * Domain-specific Hive Query Factory + */ +public class HQ { + + private HQ() {} + + @Nonnull + public static RawHQ fromStatement(String query) { + Preconditions.checkNotNull(query); + + final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); + final List split = StatementsSplitter.splitStatements(formatted); + + Preconditions.checkArgument( + 1 == split.size(), + "Detected %s queries, should be exactly one. Use `HQ.fromStatements` for multi queries.", + split.size()); + + return new RawHQ(split.get(0)); + } + + @Nonnull + public static List fromStatements(@CheckForNull final String queries) { + Preconditions.checkNotNull(queries); + + final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(queries); + final List split = StatementsSplitter.splitStatements(formatted); + final List results = new ArrayList(); + for (String q : split) { + results.add(new RawHQ(q)); + } + return results; + } + + @Nonnull + public static LazyMatchingResource autoMatchingByFileName(@CheckForNull final String fileName, + @CheckForNull final Charset charset) { + Preconditions.checkNotNull(fileName); + Preconditions.checkNotNull(charset); + + return new LazyMatchingResource(fileName, charset); + } + + @Nonnull + public static LazyMatchingResource autoMatchingByFileName(final String fileName) { + return autoMatchingByFileName(fileName, Charset.defaultCharset()); + } + + @Nonnull + public static List fromResourcePath(final String resourcePath, final Charset charset) { + return autoMatchingByFileName(resourcePath, charset).toStrict(""); + } + + @Nonnull + public static List fromResourcePath(final String resourcePath) { + return fromResourcePath(resourcePath, Charset.defaultCharset()); + } + + @Nonnull + public static TableListHQ tableList() { + return new TableListHQ(); + } + + @Nonnull + public static CreateTableHQ createTable(@CheckForNull final String tableName, + @CheckForNull final LinkedHashMap header) { + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(header); + + return new CreateTableHQ(tableName, header); + } + + @Nonnull + public static DropTableHQ dropTable(@CheckForNull final String tableName) { + Preconditions.checkNotNull(tableName); + + return new DropTableHQ(tableName); + } + + @Nonnull + public static InsertHQ insert(@CheckForNull final String tableName, + @CheckForNull final List header, @CheckForNull final List data) { + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(header); + Preconditions.checkNotNull(data); + + return new InsertHQ(tableName, header, data); + } + + @Nonnull + public static UploadFileToExistingHQ uploadByFullPathToExisting( + @CheckForNull final String tableName, @CheckForNull final String fullPath) { + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(fullPath); + + return new UploadFileToExistingHQ(tableName, new File(fullPath)); + } + + @Nonnull + public static UploadFileToExistingHQ uploadByResourcePathToExisting(final String tableName, + final String resourcePath) { + return uploadByFullPathToExisting(tableName, Resources.getResource(resourcePath).getPath()); + } + + @Nonnull + public static UploadFileAsNewTableHQ uploadByFullPathAsNewTable( + @CheckForNull final String tableName, @CheckForNull final String fullPath, + @CheckForNull final LinkedHashMap header) { + Preconditions.checkNotNull(tableName); + Preconditions.checkNotNull(fullPath); + Preconditions.checkNotNull(header); + + final File file = new File(fullPath); + + Preconditions.checkArgument(file.exists(), "%s not found", file.getPath()); + + return new UploadFileAsNewTableHQ(tableName, file, header); + } + + @Nonnull + public static UploadFileAsNewTableHQ uploadByResourcePathAsNewTable(final String tableName, + final String resourcePath, final LinkedHashMap header) { + return uploadByFullPathAsNewTable(tableName, Resources.getResource(resourcePath).getPath(), + header); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java b/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java new file mode 100644 index 00000000..00081002 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/HQBase.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +public interface HQBase { +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java new file mode 100644 index 00000000..3d69f139 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/InsertHQ.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import hivemall.utils.lang.Preconditions; + +import javax.annotation.Nonnull; +import java.util.List; + +public class InsertHQ extends TableHQ { + @Nonnull + public final List data; + @Nonnull + public final List header; + + InsertHQ(@Nonnull final String tableName, @Nonnull final List header, + @Nonnull final List data) { + super(tableName); + + int l = 0; + for (Object[] objs : data) { + Preconditions.checkArgument(objs.length == header.size(), + "l.%s : Mismatch between number of elements in row(%s) and length of header(%s)", + l, objs.length, header.size()); + l++; + } + + this.data = data; + this.header = header; + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java new file mode 100644 index 00000000..1d6f0202 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/RawHQ.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import javax.annotation.Nonnull; + +public class RawHQ implements HQBase { + @Nonnull + public final String query; + + RawHQ(@Nonnull final String query) { + this.query = query; + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java new file mode 100644 index 00000000..2f9f3c94 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableHQ.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import javax.annotation.Nonnull; + +public abstract class TableHQ implements HQBase { + @Nonnull + public final String tableName; + + TableHQ(@Nonnull final String tableName) { + this.tableName = tableName; + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java new file mode 100644 index 00000000..146adbd5 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/TableListHQ.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +public class TableListHQ implements HQBase { + TableListHQ() {} +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java new file mode 100644 index 00000000..70915577 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileAsNewTableHQ.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import javax.annotation.Nonnull; +import java.io.File; +import java.util.LinkedHashMap; + +public class UploadFileAsNewTableHQ extends UploadFileHQ { + @Nonnull + public final LinkedHashMap header; + + UploadFileAsNewTableHQ(@Nonnull final String tableName, @Nonnull final File file, + @Nonnull final LinkedHashMap header) { + super(tableName, file); + + this.header = header; + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java new file mode 100644 index 00000000..35b35c93 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileHQ.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import hivemall.utils.lang.Preconditions; + +import javax.annotation.Nonnull; +import java.io.File; + +public abstract class UploadFileHQ extends TableHQ { + public enum Format { + MSGPACK, CSV, TSV, UNKNOWN + } + + @Nonnull + public final File file; + @Nonnull + public final Format format; + + UploadFileHQ(@Nonnull final String tableName, @Nonnull final File file) { + super(tableName); + + Preconditions.checkArgument(file.exists(), "%s not found", file.getPath()); + + this.file = file; + this.format = guessFormat(file); + } + + private Format guessFormat(File file) { + final String fileName = file.getName(); + if (fileName.endsWith(".msgpack.gz")) { + return Format.MSGPACK; + } else if (fileName.endsWith(".csv")) { + return Format.CSV; + } else if (fileName.endsWith(".tsv")) { + return Format.TSV; + } else { + return Format.UNKNOWN; + } + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java new file mode 100644 index 00000000..1f5c28d9 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/UploadFileToExistingHQ.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model; + +import javax.annotation.Nonnull; +import java.io.File; + +public class UploadFileToExistingHQ extends UploadFileHQ { + UploadFileToExistingHQ(@Nonnull final String tableName, @Nonnull final File file) { + super(tableName, file); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java new file mode 100644 index 00000000..16f14eac --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/model/lazy/LazyMatchingResource.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.model.lazy; + +import com.klarna.hiverunner.CommandShellEmulation; +import com.klarna.hiverunner.sql.StatementsSplitter; +import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.utils.IO; +import hivemall.utils.lang.Preconditions; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; + +public class LazyMatchingResource { + @Nonnull + private final String fileName; + @Nonnull + private final Charset charset; + + public LazyMatchingResource(@Nonnull final String fileName, @Nonnull final Charset charset) { + this.fileName = fileName; + this.charset = charset; + } + + public List toStrict(@CheckForNull final String caseDir) { + Preconditions.checkNotNull(caseDir); + + final String query = IO.getFromResourcePath(caseDir + fileName, charset); + final String formatted = CommandShellEmulation.HIVE_CLI.transformScript(query); + final List split = StatementsSplitter.splitStatements(formatted); + final List results = new ArrayList(); + for (String q : split) { + results.add(HQ.fromStatement(q)); + } + return results; + } + + public String[] getAnswers(@CheckForNull final String answerDir) { + Preconditions.checkNotNull(answerDir); + + return IO.getFromResourcePath(answerDir + fileName).split(IO.QD); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java new file mode 100644 index 00000000..db1edc7b --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/runner/HiveSystemTestRunner.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.runner; + +import com.klarna.hiverunner.CommandShellEmulation; +import com.klarna.hiverunner.Extractor; +import com.klarna.hiverunner.HiveServerContainer; +import com.klarna.hiverunner.HiveServerContext; +import com.klarna.hiverunner.HiveShell; +import com.klarna.hiverunner.builder.HiveShellBuilder; +import com.klarna.hiverunner.config.HiveRunnerConfig; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.UploadFileToExistingHQ; +import org.junit.rules.TemporaryFolder; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class HiveSystemTestRunner extends SystemTestRunner { + private HiveServerContainer container; + private TemporaryFolder tmpFolder; + private HiveShell hShell; + + public HiveSystemTestRunner(final SystemTestCommonInfo ci, final String propertiesFile) { + super(ci, propertiesFile); + } + + public HiveSystemTestRunner(final SystemTestCommonInfo ci) { + super(ci, "hiverunner.properties"); + } + + @Override + void initRunner() { + try { + tmpFolder = new TemporaryFolder() { + { + create(); + getRoot().setWritable(true, false); + } + }; + final HiveRunnerConfig config = new HiveRunnerConfig() { + { + // required + setHiveExecutionEngine(props.getProperty("hive.execution.engine", "mr")); + + // optional + if (props.containsKey("enableTimeout")) { + setTimeoutEnabled(Boolean.valueOf(props.getProperty("enableTimeout"))); + } + if (props.containsKey("timeoutRetryLimit")) { + setTimeoutRetries(Integer.valueOf(props.getProperty("timeoutRetryLimit"))); + } + if (props.containsKey("timeoutSeconds")) { + setTimeoutSeconds(Integer.valueOf(props.getProperty("timeoutSeconds"))); + } + if (props.containsKey("commandShellEmulation")) { + setCommandShellEmulation(CommandShellEmulation.valueOf(props.getProperty("commandShellEmulation"))); + } + } + }; + final HiveServerContext ctx = Extractor.getStandaloneHiveServerContext(tmpFolder, + config); + container = Extractor.getHiveServerContainer(ctx); + @SuppressWarnings("serial") + final HiveShellBuilder builder = new HiveShellBuilder() { + { + putAllProperties(new HashMap() { + { + put("LOCAL.HDFS.DIR", "${hadoop.tmp.dir}"); + } + }); + setCommandShellEmulation(config.getCommandShellEmulation()); + setHiveServerContainer(container); + } + }; + + hShell = builder.buildShell(); + hShell.start(); + } catch (IOException ex) { + throw new RuntimeException("Failed to init HiveRunner. " + ex.getMessage()); + } + } + + @Override + void finRunner() { + if (container != null) { + container.tearDown(); + } + if (tmpFolder != null) { + tmpFolder.delete(); + } + } + + @Override + public List exec(@Nonnull final RawHQ hq) { + logger.info("executing: `" + hq.query + "`"); + + return hShell.executeQuery(hq.query); + } + + @Override + List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) throws Exception { + logger.info("executing: insert " + hq.file.getPath() + " into " + hq.tableName + " on " + + dbName); + + switch (hq.format) { + case CSV: + hShell.insertInto(dbName, hq.tableName) + .addRowsFromDelimited(hq.file, ",", null) + .commit(); + break; + case TSV: + hShell.insertInto(dbName, hq.tableName).addRowsFromTsv(hq.file).commit(); + break; + case MSGPACK: + throw new Exception("MessagePack is not supported in HiveSystemTestRunner"); + case UNKNOWN: + throw new Exception("Input csv or tsv"); + } + + return Collections.singletonList("uploaded " + hq.file.getName() + " into " + hq.tableName); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java new file mode 100644 index 00000000..82b433f8 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestCommonInfo.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.runner; + +import javax.annotation.Nonnull; + +public class SystemTestCommonInfo { + private static final String CASE = "case"; + private static final String ANSWER = "answer"; + private static final String INIT = "init"; + + @Nonnull + public final String baseDir; + @Nonnull + public final String caseDir; + @Nonnull + public final String answerDir; + @Nonnull + public final String initDir; + @Nonnull + public final String dbName; + + public SystemTestCommonInfo(@Nonnull final Class clazz) { + baseDir = clazz.getName().replace(".", "/"); + caseDir = baseDir + "/" + CASE + "/"; + answerDir = baseDir + "/" + ANSWER + "/"; + initDir = baseDir + "/" + INIT + "/"; + dbName = clazz.getName().replace(".", "_").toLowerCase(); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java new file mode 100644 index 00000000..e1421745 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestRunner.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.runner; + +import com.google.common.io.Resources; +import hivemall.systemtest.exception.QueryExecutionException; +import hivemall.systemtest.model.CreateTableHQ; +import hivemall.systemtest.model.DropTableHQ; +import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.HQBase; +import hivemall.systemtest.model.InsertHQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.TableHQ; +import hivemall.systemtest.model.TableListHQ; +import hivemall.systemtest.model.UploadFileAsNewTableHQ; +import hivemall.systemtest.model.UploadFileHQ; +import hivemall.systemtest.model.UploadFileToExistingHQ; +import hivemall.systemtest.utils.IO; +import hivemall.utils.lang.Preconditions; +import org.hamcrest.Matchers; +import org.junit.Assert; +import org.junit.rules.ExternalResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +public abstract class SystemTestRunner extends ExternalResource { + static final Logger logger = LoggerFactory.getLogger(SystemTestRunner.class); + @Nonnull + private final List classInitHqs; + @Nonnull + private final Set immutableTables; + @Nonnull + final String dbName; + @Nonnull + final Properties props; + + SystemTestRunner(@CheckForNull SystemTestCommonInfo ci, @CheckForNull String propertiesFile) { + Preconditions.checkNotNull(ci); + Preconditions.checkNotNull(propertiesFile); + + classInitHqs = new ArrayList(); + immutableTables = new HashSet(); + dbName = ci.dbName; + + final String path = "hivemall/" + propertiesFile; + try { + InputStream is = null; + try { + props = new Properties(); + is = new FileInputStream(Resources.getResource(path).getPath()); + props.load(is); + } finally { + if (is != null) { + is.close(); + } + } + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to load properties from " + path + ". " + + ex.getMessage()); + } + } + + @Override + protected void before() throws Exception { + initRunner(); + prepareDB(); // initialize database + } + + @Override + protected void after() { + try { + cleanDB(); // clean up database + } catch (Exception ex) { + throw new QueryExecutionException("Failed to clean up temporary database. " + + ex.getMessage()); + } finally { + finRunner(); + } + } + + abstract void initRunner(); + + abstract void finRunner(); + + protected void initBy(@Nonnull final HQBase hq) { + classInitHqs.add(hq); + } + + protected void initBy(@Nonnull final List hqs) { + classInitHqs.addAll(hqs); + } + + // fix to temporary database and user-defined init (should be called per Test class) + private void prepareDB() throws Exception { + createDB(dbName); + use(dbName); + for (HQBase q : classInitHqs) { + exec(q); + + if (q instanceof CreateTableHQ) { + // memo table initialized each class as immutable + immutableTables.add(((CreateTableHQ) q).tableName); + } else if (q instanceof UploadFileAsNewTableHQ) { + immutableTables.add(((UploadFileAsNewTableHQ) q).tableName); + } + } + } + + // drop temporary database (should be called per Test class) + private void cleanDB() throws Exception { + dropDB(dbName); + } + + // drop temporary tables (should be called per Test method) + void resetDB() throws Exception { + final List tables = tableList(); + for (String t : tables) { + if (!immutableTables.contains(t)) { + dropTable(HQ.dropTable(t)); + } + } + } + + // >execute HQBase + public List exec(@Nonnull final HQBase hq) throws Exception { + if (hq instanceof RawHQ) { + return exec((RawHQ) hq); + } else if (hq instanceof TableHQ) { + return exec((TableHQ) hq); + } else if (hq instanceof TableListHQ) { + return tableList(); + } else { + throw new IllegalArgumentException("Unexpected query type: " + hq.getClass()); + } + } + + // >>execute RawHQ + abstract protected List exec(@Nonnull final RawHQ hq) throws Exception; + + // >>execute TableHQ + List exec(@Nonnull final TableHQ hq) throws Exception { + if (hq instanceof CreateTableHQ) { + return createTable((CreateTableHQ) hq); + } else if (hq instanceof DropTableHQ) { + return dropTable((DropTableHQ) hq); + } else if (hq instanceof InsertHQ) { + return insert((InsertHQ) hq); + } else if (hq instanceof UploadFileHQ) { + return exec((UploadFileHQ) hq); + } else { + throw new IllegalArgumentException("Unexpected query type: " + hq.getClass()); + } + } + + // >>>execute UploadFileHQ + List exec(@Nonnull final UploadFileHQ hq) throws Exception { + if (hq instanceof UploadFileAsNewTableHQ) { + return uploadFileAsNewTable((UploadFileAsNewTableHQ) hq); + } else if (hq instanceof UploadFileToExistingHQ) { + return uploadFileToExisting((UploadFileToExistingHQ) hq); + } else { + throw new IllegalArgumentException("Unexpected query type: " + hq.getClass()); + } + } + + // matching HQBase + void matching(@Nonnull final HQBase hq, @CheckForNull final String answer, final boolean ordered) + throws Exception { + Preconditions.checkNotNull(answer); + + List result = exec(hq); + + if (ordered) { + // take order into consideration (like list) + Assert.assertThat(result, Matchers.contains(answer.split(IO.RD))); + } else { + // not take order into consideration (like multiset) + Assert.assertThat(result, Matchers.containsInAnyOrder(answer.split(IO.RD))); + } + } + + // matching HQBase (ordered == false) + void matching(@Nonnull final HQBase hq, @CheckForNull final String answer) throws Exception { + matching(hq, answer, false); + } + + List createDB(@Nonnull final String dbName) throws Exception { + logger.info("executing: create database if not exists" + dbName); + + return exec(HQ.fromStatement("CREATE DATABASE IF NOT EXISTS " + dbName)); + } + + List dropDB(@Nonnull final String dbName) throws Exception { + logger.info("executing: drop database if exists " + dbName); + + return exec(HQ.fromStatement("DROP DATABASE IF EXISTS " + dbName + " CASCADE")); + } + + List use(@Nonnull final String dbName) throws Exception { + logger.info("executing: use " + dbName); + + return exec(HQ.fromStatement("USE " + dbName)); + } + + List tableList() throws Exception { + logger.info("executing: show tables on " + dbName); + + return exec(HQ.fromStatement("SHOW TABLES")); + } + + List createTable(@Nonnull final CreateTableHQ hq) throws Exception { + logger.info("executing: create table " + hq.tableName + " if not exists on " + dbName); + + return exec(HQ.fromStatement("CREATE TABLE IF NOT EXISTS " + hq.tableName + + hq.getTableDeclaration())); + } + + List dropTable(@Nonnull final DropTableHQ hq) throws Exception { + logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); + + return exec(HQ.fromStatement("DROP TABLE IF EXISTS " + hq.tableName)); + } + + List insert(@Nonnull final InsertHQ hq) throws Exception { + logger.info("executing: insert into " + hq.tableName + " on " + dbName); + + // *WORKAROUND* + // `INSERT INTO TABLE ... VALUES ...` + // cannot use array() and map() with `VALUES` on hiverunner(v3.0.0), + // cannot insert anything on TD(v20160901) + // `WITH ... AS (SELECT ...) INSERT INTO TABLE ... SELECT * FROM ...` + // can insert anything on hiverunner(v3.0.0) + // cannot use map on TD(v20160901) + final StringBuilder sb = new StringBuilder(); + sb.append("WITH temporary_table_for_with_clause AS ("); + for (Object[] row : hq.data) { + sb.append("SELECT "); + for (int i = 0; i < hq.header.size(); i++) { + sb.append(serialize(row[i])); + sb.append(" "); + sb.append(hq.header.get(i)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(" UNION ALL "); + } + sb.delete(sb.length() - 11, sb.length()); + sb.append(") INSERT INTO TABLE "); + sb.append(hq.tableName); + sb.append(" SELECT * FROM temporary_table_for_with_clause"); + + return exec(HQ.fromStatement(sb.toString())); + } + + List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableHQ hq) throws Exception { + logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() + + " if not exists on " + dbName); + + createTable(HQ.createTable(hq.tableName, hq.header)); + return uploadFileToExisting(HQ.uploadByFullPathToExisting(hq.tableName, hq.file.getPath())); + } + + abstract List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) + throws Exception; + + private String serialize(@Nullable final Object val) { + // NOTE: this method is low-performance, don't use w/ big data + if (val instanceof String) { + return "'" + String.valueOf(val) + "'"; + } else if (val instanceof Object[]) { + final Object[] objs = (Object[]) val; + final StringBuilder sb = new StringBuilder(); + sb.append("array("); + for (Object o : objs) { + sb.append(serialize(o)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } else if (val instanceof List) { + final List list = (List) val; + final StringBuilder sb = new StringBuilder(); + sb.append("array("); + for (Object o : list) { + sb.append(serialize(o)); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } else if (val instanceof Map) { + final Map map = (Map) val; + final StringBuilder sb = new StringBuilder(); + sb.append("map("); + for (Map.Entry e : map.entrySet()) { + sb.append(serialize(e.getKey())); + sb.append(","); + sb.append(serialize(e.getValue())); + sb.append(","); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } else { + return String.valueOf(val); + } + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java new file mode 100644 index 00000000..fcd2fcb3 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/runner/SystemTestTeam.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.runner; + +import hivemall.systemtest.exception.QueryExecutionException; +import hivemall.systemtest.model.HQBase; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.lazy.LazyMatchingResource; +import hivemall.utils.lang.Preconditions; +import org.junit.rules.ExternalResource; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +public class SystemTestTeam extends ExternalResource { + @Nonnull + private final List runners; + @Nonnull + private final List reachGoal; + + @Nonnull + private final List initHqs; + @Nonnull + private final Map, Boolean> entries; + + private boolean needRun = false; // remind `run()` + + public SystemTestTeam(final SystemTestRunner... runners) { + this.runners = new ArrayList(); + this.reachGoal = new ArrayList(); // distinct + this.initHqs = new ArrayList(); + this.entries = new LinkedHashMap, Boolean>(); + + this.runners.addAll(Arrays.asList(runners)); + } + + @Override + protected void after() { + if (needRun) { + throw new IllegalStateException("Call `SystemTestTeam#run()`"); + } + + for (SystemTestRunner runner : reachGoal) { + try { + runner.resetDB(); + } catch (Exception ex) { + throw new QueryExecutionException("Failed to resetPerMethod database. " + + ex.getMessage()); + } + } + } + + // add additional runner for each @Test method + public void add(final SystemTestRunner... runners) { + this.runners.addAll(Arrays.asList(runners)); + } + + // add initialization for each @Test method + public void initBy(@Nonnull final HQBase hq) { + initHqs.add(hq); + + needRun = true; + } + + public void initBy(@Nonnull final List hqs) { + initHqs.addAll(hqs); + + needRun = true; + } + + public void set(@Nonnull final HQBase hq, @CheckForNull final String expected, boolean ordered) { + Preconditions.checkNotNull(expected); + + entries.put(pair(hq, expected), ordered); + + needRun = true; + } + + public void set(@Nonnull final HQBase hq, @CheckForNull final String expected) { + Preconditions.checkNotNull(expected); + + entries.put(pair(hq, expected), false); + + needRun = true; + } + + public void set(@Nonnull final List hqs, + @CheckForNull final List expecteds, @CheckForNull final List ordereds) { + Preconditions.checkNotNull(expecteds); + Preconditions.checkNotNull(ordereds); + Preconditions.checkArgument(hqs.size() == expecteds.size(), + "Mismatch between number of queries(%s) and length of answers(%s)", hqs.size(), + expecteds.size()); + Preconditions.checkArgument(hqs.size() == ordereds.size(), + "Mismatch between number of queries(%s) and correspond ordered flags(%s)", hqs.size(), + ordereds.size()); + + for (int i = 0; i < expecteds.size(); i++) { + set(hqs.get(i), expecteds.get(i), ordereds.get(i)); + } + + needRun = true; + } + + public void set(@Nonnull final List hqs, + @CheckForNull final List expecteds) { + final List ordereds = new ArrayList(); + for (int i = 0; i < hqs.size(); i++) { + ordereds.add(false); + } + set(hqs, expecteds, ordereds); + } + + public void set(@Nonnull final LazyMatchingResource hq, + @CheckForNull final SystemTestCommonInfo ci, final boolean ordered) { + final List rhqs = hq.toStrict(ci.caseDir); + final String[] answers = hq.getAnswers(ci.answerDir); + + Preconditions.checkArgument(rhqs.size() == answers.length, + "Mismatch between number of queries(%s) and length of answers(%s)", rhqs.size(), + answers.length); + + for (int i = 0; i < answers.length; i++) { + set(rhqs.get(i), answers[i], ordered); + } + + needRun = true; + } + + public void set(@Nonnull final LazyMatchingResource hq, final SystemTestCommonInfo ci) { + set(hq, ci, false); + } + + public void run() throws Exception { + needRun = false; + + if (runners.size() == 0) { + throw new IllegalStateException("Set at least one runner."); + } + + for (SystemTestRunner runner : runners) { + if (!reachGoal.contains(runner)) { + // initialization each @Test methods + for (HQBase q : initHqs) { + runner.exec(q); + } + reachGoal.add(runner); + } + + for (Entry, Boolean> entry : entries.entrySet()) { + runner.matching(entry.getKey().getKey(), entry.getKey().getValue(), + entry.getValue()); + } + } + } + + private Entry pair(HQBase hq, String answer) { + return new SimpleEntry(hq, answer); + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java new file mode 100644 index 00000000..b2f82903 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/runner/TDSystemTestRunner.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.runner; + +import com.google.common.base.Function; +import com.treasuredata.client.ExponentialBackOff; +import com.treasuredata.client.TDClient; +import com.treasuredata.client.model.TDBulkImportSession; +import com.treasuredata.client.model.TDColumn; +import com.treasuredata.client.model.TDColumnType; +import com.treasuredata.client.model.TDJobRequest; +import com.treasuredata.client.model.TDJobSummary; +import com.treasuredata.client.model.TDResultFormat; +import com.treasuredata.client.model.TDTable; +import hivemall.systemtest.MsgpackConverter; +import hivemall.systemtest.exception.QueryExecutionException; +import hivemall.systemtest.model.CreateTableHQ; +import hivemall.systemtest.model.DropTableHQ; +import hivemall.systemtest.model.HQ; +import hivemall.systemtest.model.RawHQ; +import hivemall.systemtest.model.UploadFileAsNewTableHQ; +import hivemall.systemtest.model.UploadFileToExistingHQ; +import hivemall.systemtest.utils.IO; +import org.apache.commons.csv.CSVFormat; + +import javax.annotation.Nonnull; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class TDSystemTestRunner extends SystemTestRunner { + private TDClient client; + + private int execFinishRetryLimit = 7; + private int fileUploadPerformRetryLimit = 7; + private int fileUploadCommitBackOff = 5; + private int fileUploadCommitRetryLimit = 7; + + public TDSystemTestRunner(final SystemTestCommonInfo ci, final String propertiesFile) { + super(ci, propertiesFile); + } + + public TDSystemTestRunner(final SystemTestCommonInfo ci) { + super(ci, "td.properties"); + } + + @Override + void initRunner() { + // optional + if (props.containsKey("execFinishRetryLimit")) { + execFinishRetryLimit = Integer.valueOf(props.getProperty("execFinishRetryLimit")); + } + if (props.containsKey("fileUploadPerformRetryLimit")) { + fileUploadPerformRetryLimit = Integer.valueOf(props.getProperty("fileUploadPerformRetryLimit")); + } + if (props.containsKey("fileUploadCommitBackOff")) { + fileUploadCommitBackOff = Integer.valueOf(props.getProperty("fileUploadCommitBackOff")); + } + if (props.containsKey("fileUploadCommitRetryLimit")) { + fileUploadCommitRetryLimit = Integer.valueOf(props.getProperty("fileUploadCommitRetryLimit")); + } + + boolean fromPropertiesFile = false; + for (Map.Entry e : props.entrySet()) { + final String key = e.getKey().toString(); + if (key.startsWith("td.client.")) { + fromPropertiesFile = true; + System.setProperty(key, e.getValue().toString()); + } + } + + if (fromPropertiesFile) { + client = TDClient.newBuilder(false).build(); // use *.properties + } else { + client = TDClient.newClient(); // use $HOME/.td/td.conf + } + } + + @Override + void finRunner() { + if (client != null) { + client.close(); + } + } + + @Override + public List exec(@Nonnull final RawHQ hq) throws Exception { + logger.info("executing: `" + hq.query + "`"); + + final TDJobRequest req = TDJobRequest.newHiveQuery(dbName, hq.query); + final String id = client.submit(req); + + final ExponentialBackOff backOff = new ExponentialBackOff(); + TDJobSummary job = client.jobStatus(id); + int nRetries = 0; + while (!job.getStatus().isFinished()) { + if (nRetries > execFinishRetryLimit) { + throw new Exception("Exceed standard of finish check retry repetition: " + + execFinishRetryLimit); + } + + Thread.sleep(backOff.nextWaitTimeMillis()); + job = client.jobStatus(id); + + nRetries++; + } + + return client.jobResult(id, TDResultFormat.TSV, new Function>() { + @Override + public List apply(InputStream input) { + final List results = new ArrayList(); + BufferedReader reader = null; + try { + try { + reader = new BufferedReader(new InputStreamReader(input)); + String line; + while ((line = reader.readLine()) != null) { + results.addAll(Arrays.asList(line.split(IO.RD))); + } + } finally { + if (reader != null) { + reader.close(); + } + } + } catch (IOException ex) { + throw new QueryExecutionException("Failed to read results from TD. " + + ex.getMessage()); + } + return results; + } + }); + } + + @Override + List createDB(@Nonnull final String dbName) throws Exception { + logger.info("executing: create database if not exists " + dbName); + + client.createDatabaseIfNotExists(dbName); + return Collections.singletonList("created " + dbName); + } + + @Override + List dropDB(@Nonnull final String dbName) throws Exception { + logger.info("executing: drop database if exists " + dbName); + + client.deleteDatabaseIfExists(dbName); + return Collections.singletonList("dropped " + dbName); + } + + @Override + List use(@Nonnull final String dbName) throws Exception { + return Collections.singletonList("No need to execute `USE` statement on TD, so skipped `USE " + + dbName + "`"); + } + + @Override + List tableList() throws Exception { + logger.info("executing: show tables on " + dbName); + + final List tables = client.listTables(dbName); + final List result = new ArrayList(); + for (TDTable t : tables) { + result.add(t.getName()); + } + return result; + } + + @Override + List createTable(@Nonnull final CreateTableHQ hq) throws Exception { + logger.info("executing: create table " + hq.tableName + " if not exists on " + dbName); + + final List columns = new ArrayList(); + for (Map.Entry e : hq.header.entrySet()) { + columns.add(new TDColumn(e.getKey(), TDColumnType.parseColumnType(e.getValue()))); + } + client.createTableIfNotExists(dbName, hq.tableName); + client.updateTableSchema(dbName, hq.tableName, columns); + return Collections.singletonList("created " + hq.tableName + " on " + dbName); + } + + @Override + List dropTable(@Nonnull final DropTableHQ hq) throws Exception { + logger.info("executing: drop table " + hq.tableName + " if exists on " + dbName); + + client.deleteTableIfExists(dbName, hq.tableName); + return Collections.singletonList("dropped " + hq.tableName); + } + + @Override + List uploadFileAsNewTable(@Nonnull final UploadFileAsNewTableHQ hq) throws Exception { + logger.info("executing: create " + hq.tableName + " based on " + hq.file.getPath() + + " if not exists on " + dbName); + + createTable(HQ.createTable(hq.tableName, hq.header)); + + final String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); + final String partName = "part-of-" + String.valueOf(sessionName); + client.createBulkImportSession(sessionName, dbName, hq.tableName); + + try { + // upload file as msgpack + switch (hq.format) { + case MSGPACK: + client.uploadBulkImportPart(sessionName, partName, hq.file); + break; + case CSV: { + final File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + + client.uploadBulkImportPart(sessionName, partName, + new MsgpackConverter(hq.file, new ArrayList(hq.header.keySet()), + CSVFormat.DEFAULT).asFile(to)); + break; + } + case TSV: { + final File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + + client.uploadBulkImportPart(sessionName, partName, + new MsgpackConverter(hq.file, new ArrayList(hq.header.keySet()), + CSVFormat.TDF).asFile(to)); + break; + } + case UNKNOWN: + throw new Exception("Input msgpack.gz, csv or tsv"); + } + + client.freezeBulkImportSession(sessionName); + client.performBulkImportSession(sessionName); + final ExponentialBackOff backOff = new ExponentialBackOff(); + TDBulkImportSession session = client.getBulkImportSession(sessionName); + int performNRetries = 0; + while (session.getStatus() == TDBulkImportSession.ImportStatus.PERFORMING) { + if (performNRetries > fileUploadPerformRetryLimit) { + throw new Exception("Exceed standard of perform check retry repetition: " + + fileUploadPerformRetryLimit); + } + + logger.debug("Waiting bulk import completion"); + Thread.sleep(backOff.nextWaitTimeMillis()); + session = client.getBulkImportSession(sessionName); + + performNRetries++; + } + + client.commitBulkImportSession(sessionName); + session = client.getBulkImportSession(sessionName); + int commitNRetries = 0; + while (session.getStatus() != TDBulkImportSession.ImportStatus.COMMITTED) { + if (commitNRetries > fileUploadCommitRetryLimit) { + throw new Exception("Exceed standard of commit check retry repetition: " + + fileUploadCommitRetryLimit); + } + + logger.info("Waiting bulk import perform step completion"); + Thread.sleep(TimeUnit.SECONDS.toMillis(fileUploadCommitBackOff)); + session = client.getBulkImportSession(sessionName); + + commitNRetries++; + } + } finally { + client.deleteBulkImportSession(sessionName); + } + + return Collections.singletonList("uploaded " + hq.file.getName() + " into " + hq.tableName); + } + + @Override + List uploadFileToExisting(@Nonnull final UploadFileToExistingHQ hq) throws Exception { + logger.info("executing: insert " + hq.file.getPath() + " into " + hq.tableName + " on " + + dbName); + + final String sessionName = "session-" + String.valueOf(System.currentTimeMillis()); + final String partName = "part-of-" + String.valueOf(sessionName); + client.createBulkImportSession(sessionName, dbName, hq.tableName); + + try { + // upload file as msgpack + switch (hq.format) { + case MSGPACK: + client.uploadBulkImportPart(sessionName, partName, hq.file); + break; + case CSV: { + File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + client.uploadBulkImportPart(sessionName, partName, new MsgpackConverter( + hq.file, getHeaderFromTD(hq.tableName), CSVFormat.DEFAULT).asFile(to)); + break; + } + case TSV: { + File to = File.createTempFile(hq.file.getName(), ".msgpack.gz"); + to.deleteOnExit(); + client.uploadBulkImportPart(sessionName, partName, new MsgpackConverter( + hq.file, getHeaderFromTD(hq.tableName), CSVFormat.TDF).asFile(to)); + break; + } + case UNKNOWN: + throw new Exception("Input msgpack.gz, csv or tsv"); + } + + client.freezeBulkImportSession(sessionName); + client.performBulkImportSession(sessionName); + final ExponentialBackOff backOff = new ExponentialBackOff(); + TDBulkImportSession session = client.getBulkImportSession(sessionName); + while (session.getStatus() == TDBulkImportSession.ImportStatus.PERFORMING) { + logger.debug("Waiting bulk import completion"); + Thread.sleep(backOff.nextWaitTimeMillis()); + session = client.getBulkImportSession(sessionName); + } + + client.commitBulkImportSession(sessionName); + session = client.getBulkImportSession(sessionName); + while (session.getStatus() != TDBulkImportSession.ImportStatus.COMMITTED) { + logger.info("Waiting bulk import perform step completion"); + Thread.sleep(TimeUnit.SECONDS.toMillis(fileUploadCommitBackOff)); + session = client.getBulkImportSession(sessionName); + } + } finally { + client.deleteBulkImportSession(sessionName); + } + + return Collections.singletonList("uploaded " + hq.file.getName() + " into " + hq.tableName); + } + + private List getHeaderFromTD(@Nonnull final String tableName) { + final List header = new ArrayList(); + for (TDTable t : client.listTables(dbName)) { + if (t.getName().equals(tableName)) { + List cols = t.getColumns(); + for (TDColumn col : cols) { + header.add(col.getName()); + } + break; + } + } + return header; + } +} diff --git a/systemtest/src/main/java/hivemall/systemtest/utils/IO.java b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java new file mode 100644 index 00000000..04309451 --- /dev/null +++ b/systemtest/src/main/java/hivemall/systemtest/utils/IO.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.systemtest.utils; + +import com.google.common.io.Resources; +import hivemall.utils.lang.Preconditions; + +import javax.annotation.CheckForNull; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; + +public class IO { + public static final String RD = "\t"; // row delimiter + public static final String QD = "\n"; // query delimiter + + private IO() {} + + public static String getFromFullPath(@CheckForNull final String fullPath, final Charset charset) { + Preconditions.checkNotNull(fullPath); + + return new String(readAllBytes(fullPath), charset); + } + + public static String getFromFullPath(@CheckForNull final String fullPath) { + return getFromFullPath(fullPath, Charset.defaultCharset()); + } + + public static String getFromResourcePath(@CheckForNull final String resourcePath, + final Charset charset) { + Preconditions.checkNotNull(resourcePath); + + final String fullPath = Resources.getResource(resourcePath).getPath(); + return getFromFullPath(fullPath, charset); + } + + public static String getFromResourcePath(@CheckForNull final String resourcePath) { + return getFromResourcePath(resourcePath, Charset.defaultCharset()); + } + + private static byte[] readAllBytes(final String filePath) { + final File file = new File(filePath); + + Preconditions.checkArgument(file.exists(), "%s not found", filePath); + + final int len = (int) file.length(); + final byte[] buf = new byte[len]; + + InputStream is = null; + try { + try { + is = new FileInputStream(file); + is.read(buf); + } finally { + if (is != null) { + is.close(); + } + } + } catch (IOException ex) { + throw new RuntimeException("Failed to read " + filePath + ". " + ex.getMessage()); + } + + return buf; + } +} diff --git a/systemtest/src/test/resources/hivemall/hiverunner.properties b/systemtest/src/test/resources/hivemall/hiverunner.properties new file mode 100644 index 00000000..5870bf15 --- /dev/null +++ b/systemtest/src/test/resources/hivemall/hiverunner.properties @@ -0,0 +1,6 @@ +# properties for HiveRunner +hive.execution.engine=mr +#enableTimeout=false +#timeoutRetryLimit=2 +#timeoutSeconds=30 +#commandShellEmulation=HIVE_CLI diff --git a/systemtest/src/test/resources/hivemall/td.properties b/systemtest/src/test/resources/hivemall/td.properties new file mode 100644 index 00000000..4ba88f3c --- /dev/null +++ b/systemtest/src/test/resources/hivemall/td.properties @@ -0,0 +1,13 @@ +# properties for TD +execFinishRetryLimit=7 +fileUploadPerformRetryLimit=7 +fileUploadCommitBackOff=5 +fileUploadCommitRetryLimit=7 + +# TD client configs like following prior to $HOME/.td/td.conf +#td.client.user= +#td.client.apikey= +#td.client.password= +#td.client.endpoint= +#td.client.port= +#td.client.usessl=