fix Load tests problem (pingcap#154)

wanfangdata · Dec 22, 2017 · 1a0591e · 1a0591e
1 parent 056aaf7
commit 1a0591e
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 49 deletions.
diff --git a/integtest/src/main/scala/com/pingcap/spark/JDBCWrapper.scala b/integtest/src/main/scala/com/pingcap/spark/JDBCWrapper.scala
@@ -168,20 +168,24 @@ class JDBCWrapper(prop: Properties) extends LazyLogging {
   }
 
   def loadTable(path: String): Unit = {
-    logger.info("Loading data from : " + path)
-    val lines = readFile(path)
-    val (table, schema, rows) = (lines.head, lines(1).split(Pattern.quote(Sep)).toList, lines.drop(2))
-    val rowData: List[List[Any]] = rows.map {
-      rowFromString(_, schema)
+    try {
+      logger.info("Loading data from : " + path)
+      val lines = readFile(path)
+      val (table, schema, rows) = (lines.head, lines(1).split(Pattern.quote(Sep)).toList, lines.drop(2))
+      val rowData: List[List[Any]] = rows.map {
+        rowFromString(_, schema)
+      }
+      rowData.map(insertRow(_, schema, table))
+    } catch {
+      case e: Exception => logger.error("Error loading table from path: " + e.getMessage)
     }
-    rowData.map(insertRow(_, schema, table))
   }
 
   def init(databaseName: String): String = {
     if (databaseName != null) {
-      logger.info("?????" + databaseName)
+      logger.info("fetching " + databaseName)
       if (!databaseExists(databaseName)) {
-        createDatabase(databaseName, false)
+        createDatabase(databaseName, cleanup = false)
       }
       connection.setCatalog(databaseName)
       currentDatabaseName = databaseName

diff --git a/integtest/src/main/scala/com/pingcap/spark/TestCase.scala b/integtest/src/main/scala/com/pingcap/spark/TestCase.scala
@@ -133,12 +133,11 @@ class TestCase(val prop: Properties) extends LazyLogging {
           + "  Tests succeeded: " + (testsExecuted - testsFailed - testsSkipped)
           + "  Tests failed: " + testsFailed
           + "  Tests skipped: " + testsSkipped)
+        jdbc.close()
+        spark.close()
+        spark_jdbc.close()
       case _ =>
     }
-
-    jdbc.close()
-    spark.close()
-    spark_jdbc.close()
   }
 
   protected def work(parentPath: String, run: Boolean, load: Boolean, compareNeeded: Boolean): Unit = {
@@ -156,49 +155,53 @@ class TestCase(val prop: Properties) extends LazyLogging {
       "ignored")
 
     logger.info(s"run=${run.toString} load=${load.toString} compareNeeded=${compareNeeded.toString}")
-    if (!ignoreCases.exists(_.equalsIgnoreCase(dbName))) {
-      if (dir.isDirectory) {
-        dir.listFiles().map { f =>
-          if (f.isDirectory) {
-            dirs += f.getAbsolutePath
-          } else {
-            if (f.getName.endsWith(DDLSuffix)) {
-              ddls += f.getAbsolutePath
-            } else if (f.getName.endsWith(DataSuffix)) {
-              dataFiles += f.getAbsolutePath
-            } else if (f.getName.endsWith(SQLSuffix)) {
-              testCases += ((f.getName, readFile(f.getAbsolutePath).mkString("\n")))
+    try {
+      if (!ignoreCases.exists(_.equalsIgnoreCase(dbName))) {
+        if (dir.isDirectory) {
+          dir.listFiles().map { f =>
+            if (f.isDirectory) {
+              dirs += f.getAbsolutePath
+            } else {
+              if (f.getName.endsWith(DDLSuffix)) {
+                ddls += f.getAbsolutePath
+              } else if (f.getName.endsWith(DataSuffix)) {
+                dataFiles += f.getAbsolutePath
+              } else if (f.getName.endsWith(SQLSuffix)) {
+                testCases += ((f.getName, readFile(f.getAbsolutePath).mkString("\n")))
+              }
             }
           }
+        } else {
+          throw new IllegalArgumentException("Cannot prepare non-folder")
         }
-      } else {
-        throw new IllegalArgumentException("Cannot prepare non-folder")
-      }
 
-      if (load) {
-        logger.info(s"Switch to $dbName")
-        dbName = jdbc.init(dbName)
-        logger.info("Load data... ")
-        ddls.foreach { file => {
-          logger.info(s"Register for DDL script $file")
-          jdbc.createTable(file)
-        }
-        }
-        dataFiles.foreach { file => {
-          logger.info(s"Register for data loading script $file")
-          jdbc.loadTable(file)
-        }
+        if (load) {
+          logger.info(s"Switch to $dbName")
+          dbName = jdbc.init(dbName)
+          logger.info("Load data... ")
+          ddls.foreach { file => {
+            logger.info(s"Register for DDL script $file")
+            jdbc.createTable(file)
+          }
+          }
+          dataFiles.foreach { file => {
+            logger.info(s"Register for data loading script $file")
+            jdbc.loadTable(file)
+          }
+          }
         }
-      }
-      if (run) {
-        if (!dbAssigned || useDatabase.exists(_.equalsIgnoreCase(dbName))) {
-          test(dbName, testCases, compareNeeded)
+        if (run) {
+          if (!dbAssigned || useDatabase.exists(_.equalsIgnoreCase(dbName))) {
+            test(dbName, testCases, compareNeeded)
+          }
         }
-      }
 
-      dirs.foreach { dir =>
-        work(dir, run, load, compareNeeded)
+        dirs.foreach { dir =>
+          work(dir, run, load, compareNeeded)
+        }
       }
+    } catch {
+      case e: Exception => logger.error("Unexpected error occured: " + e.getMessage)
     }
   }
 
@@ -608,11 +611,10 @@ class TestCase(val prop: Properties) extends LazyLogging {
 
   private def testInline(dbName: String, testCases: ArrayBuffer[(String, String)]): Unit = {
     if (dbName.equalsIgnoreCase("test_index")) {
+      testAndCalc(new IssueTestCase(prop), dbName, testCases)
       testAndCalc(new TestIndex(prop), dbName, testCases)
     } else if (dbName.equalsIgnoreCase("tispark_test")) {
       testAndCalc(new DAGTestCase(prop), dbName, testCases)
-    } else if (dbName.equalsIgnoreCase("issue_test")) {
-      testAndCalc(new IssueTestCase(prop), dbName, testCases)
     } else {
       test(dbName, testCases)
     }