Skip to content

Commit

Permalink
add comments and remove useless failAfter logic
Browse files Browse the repository at this point in the history
  • Loading branch information
zhichao-li committed Jun 12, 2015
1 parent d9677e1 commit 8418c97
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ case class ScriptTransformation(
child.execute().mapPartitions { iter =>
val cmd = List("/bin/bash", "-c", script)
val builder = new ProcessBuilder(cmd)
// redirectError(Redirect.INHERIT) would consume the error output from buffer and
// then print it to stderr (inherit the target from the current Scala process).
// If without this there would be 2 issues:
// 1) The error msg generated by the script process would be hidden.
// 2) If the error msg is too big to chock up the buffer, the input logic would be hung
builder.redirectError(Redirect.INHERIT)
val proc = builder.start()
val inputStream = proc.getInputStream
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,6 @@

package org.apache.spark.sql.hive.execution

import org.scalatest.concurrent.Timeouts._
import org.scalatest.time.Span
import org.scalatest.time.Millis

import org.apache.spark.sql.catalyst.DefaultParserDialect
import org.apache.spark.sql.catalyst.analysis.EliminateSubQueries
import org.apache.spark.sql.catalyst.errors.DialectException
Expand Down Expand Up @@ -636,21 +632,17 @@ class SQLQuerySuite extends QueryTest {
test("test script transform for stdout") {
val data = (1 to 100000).map { i => (i, i, i) }
data.toDF("d1", "d2", "d3").registerTempTable("script_trans")
failAfter(Span(20000, Millis)) {
assert(100000 ===
sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat' AS (a,b,c) FROM script_trans")
.queryExecution.toRdd.count())
}
assert(100000 ===
sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat' AS (a,b,c) FROM script_trans")
.queryExecution.toRdd.count())
}

test("test script transform for stderr") {
val data = (1 to 100000).map { i => (i, i, i) }
data.toDF("d1", "d2", "d3").registerTempTable("script_trans")
failAfter(Span(20000, Millis)) {
assert(0 ===
sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat 1>&2' AS (a,b,c) FROM script_trans")
.queryExecution.toRdd.count())
}
assert(0 ===
sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat 1>&2' AS (a,b,c) FROM script_trans")
.queryExecution.toRdd.count())
}

test("window function: udaf with aggregate expressin") {
Expand Down

0 comments on commit 8418c97

Please sign in to comment.