From 1a0591e244ea84096d01032ff2f0f0e379ebf367 Mon Sep 17 00:00:00 2001 From: birdstorm Date: Fri, 22 Dec 2017 17:15:41 +0800 Subject: [PATCH] fix Load tests problem (#154) --- .../scala/com/pingcap/spark/JDBCWrapper.scala | 20 +++-- .../scala/com/pingcap/spark/TestCase.scala | 84 ++++++++++--------- 2 files changed, 55 insertions(+), 49 deletions(-) diff --git a/integtest/src/main/scala/com/pingcap/spark/JDBCWrapper.scala b/integtest/src/main/scala/com/pingcap/spark/JDBCWrapper.scala index 7393459c05..c45feb3ceb 100644 --- a/integtest/src/main/scala/com/pingcap/spark/JDBCWrapper.scala +++ b/integtest/src/main/scala/com/pingcap/spark/JDBCWrapper.scala @@ -168,20 +168,24 @@ class JDBCWrapper(prop: Properties) extends LazyLogging { } def loadTable(path: String): Unit = { - logger.info("Loading data from : " + path) - val lines = readFile(path) - val (table, schema, rows) = (lines.head, lines(1).split(Pattern.quote(Sep)).toList, lines.drop(2)) - val rowData: List[List[Any]] = rows.map { - rowFromString(_, schema) + try { + logger.info("Loading data from : " + path) + val lines = readFile(path) + val (table, schema, rows) = (lines.head, lines(1).split(Pattern.quote(Sep)).toList, lines.drop(2)) + val rowData: List[List[Any]] = rows.map { + rowFromString(_, schema) + } + rowData.map(insertRow(_, schema, table)) + } catch { + case e: Exception => logger.error("Error loading table from path: " + e.getMessage) } - rowData.map(insertRow(_, schema, table)) } def init(databaseName: String): String = { if (databaseName != null) { - logger.info("?????" + databaseName) + logger.info("fetching " + databaseName) if (!databaseExists(databaseName)) { - createDatabase(databaseName, false) + createDatabase(databaseName, cleanup = false) } connection.setCatalog(databaseName) currentDatabaseName = databaseName diff --git a/integtest/src/main/scala/com/pingcap/spark/TestCase.scala b/integtest/src/main/scala/com/pingcap/spark/TestCase.scala index 22e085ed5f..9d9169b6cd 100644 --- a/integtest/src/main/scala/com/pingcap/spark/TestCase.scala +++ b/integtest/src/main/scala/com/pingcap/spark/TestCase.scala @@ -133,12 +133,11 @@ class TestCase(val prop: Properties) extends LazyLogging { + " Tests succeeded: " + (testsExecuted - testsFailed - testsSkipped) + " Tests failed: " + testsFailed + " Tests skipped: " + testsSkipped) + jdbc.close() + spark.close() + spark_jdbc.close() case _ => } - - jdbc.close() - spark.close() - spark_jdbc.close() } protected def work(parentPath: String, run: Boolean, load: Boolean, compareNeeded: Boolean): Unit = { @@ -156,49 +155,53 @@ class TestCase(val prop: Properties) extends LazyLogging { "ignored") logger.info(s"run=${run.toString} load=${load.toString} compareNeeded=${compareNeeded.toString}") - if (!ignoreCases.exists(_.equalsIgnoreCase(dbName))) { - if (dir.isDirectory) { - dir.listFiles().map { f => - if (f.isDirectory) { - dirs += f.getAbsolutePath - } else { - if (f.getName.endsWith(DDLSuffix)) { - ddls += f.getAbsolutePath - } else if (f.getName.endsWith(DataSuffix)) { - dataFiles += f.getAbsolutePath - } else if (f.getName.endsWith(SQLSuffix)) { - testCases += ((f.getName, readFile(f.getAbsolutePath).mkString("\n"))) + try { + if (!ignoreCases.exists(_.equalsIgnoreCase(dbName))) { + if (dir.isDirectory) { + dir.listFiles().map { f => + if (f.isDirectory) { + dirs += f.getAbsolutePath + } else { + if (f.getName.endsWith(DDLSuffix)) { + ddls += f.getAbsolutePath + } else if (f.getName.endsWith(DataSuffix)) { + dataFiles += f.getAbsolutePath + } else if (f.getName.endsWith(SQLSuffix)) { + testCases += ((f.getName, readFile(f.getAbsolutePath).mkString("\n"))) + } } } + } else { + throw new IllegalArgumentException("Cannot prepare non-folder") } - } else { - throw new IllegalArgumentException("Cannot prepare non-folder") - } - if (load) { - logger.info(s"Switch to $dbName") - dbName = jdbc.init(dbName) - logger.info("Load data... ") - ddls.foreach { file => { - logger.info(s"Register for DDL script $file") - jdbc.createTable(file) - } - } - dataFiles.foreach { file => { - logger.info(s"Register for data loading script $file") - jdbc.loadTable(file) - } + if (load) { + logger.info(s"Switch to $dbName") + dbName = jdbc.init(dbName) + logger.info("Load data... ") + ddls.foreach { file => { + logger.info(s"Register for DDL script $file") + jdbc.createTable(file) + } + } + dataFiles.foreach { file => { + logger.info(s"Register for data loading script $file") + jdbc.loadTable(file) + } + } } - } - if (run) { - if (!dbAssigned || useDatabase.exists(_.equalsIgnoreCase(dbName))) { - test(dbName, testCases, compareNeeded) + if (run) { + if (!dbAssigned || useDatabase.exists(_.equalsIgnoreCase(dbName))) { + test(dbName, testCases, compareNeeded) + } } - } - dirs.foreach { dir => - work(dir, run, load, compareNeeded) + dirs.foreach { dir => + work(dir, run, load, compareNeeded) + } } + } catch { + case e: Exception => logger.error("Unexpected error occured: " + e.getMessage) } } @@ -608,11 +611,10 @@ class TestCase(val prop: Properties) extends LazyLogging { private def testInline(dbName: String, testCases: ArrayBuffer[(String, String)]): Unit = { if (dbName.equalsIgnoreCase("test_index")) { + testAndCalc(new IssueTestCase(prop), dbName, testCases) testAndCalc(new TestIndex(prop), dbName, testCases) } else if (dbName.equalsIgnoreCase("tispark_test")) { testAndCalc(new DAGTestCase(prop), dbName, testCases) - } else if (dbName.equalsIgnoreCase("issue_test")) { - testAndCalc(new IssueTestCase(prop), dbName, testCases) } else { test(dbName, testCases) }