From a59f40ec591c5a9b3dde6e04d2105b1f618b5cdc Mon Sep 17 00:00:00 2001 From: zsxwing Date: Thu, 7 May 2015 10:32:15 -0700 Subject: [PATCH 1/2] Eliminate noisy css warn/error logs for UISeleniumSuite --- .../apache/spark/ui/static/streaming-page.css | 2 +- .../org/apache/spark/ui/UISeleniumSuite.scala | 30 ++++++++++++++++- .../hive/thriftserver/UISeleniumSuite.scala | 30 ++++++++++++++++- .../spark/streaming/UISeleniumSuite.scala | 32 +++++++++++++++++-- 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css index 5da9d631ad124..82ef5ee6ac919 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.css @@ -31,7 +31,7 @@ } .tooltip-inner { - max-width: 500px !important; // Make sure we only have one line tooltip + max-width: 500px !important; /* Make sure we only have one line tooltip */ } .line { diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index d53d7f3ba5ae7..def1ad675de45 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -23,12 +23,14 @@ import javax.servlet.http.HttpServletRequest import scala.collection.JavaConversions._ import scala.xml.Node +import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler import org.openqa.selenium.htmlunit.HtmlUnitDriver import org.openqa.selenium.{By, WebDriver} import org.scalatest._ import org.scalatest.concurrent.Eventually._ import org.scalatest.selenium.WebBrowser import org.scalatest.time.SpanSugar._ +import org.w3c.css.sac.CSSParseException import org.apache.spark.LocalSparkContext._ import org.apache.spark._ @@ -44,7 +46,33 @@ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers with Before implicit var webDriver: WebDriver = _ override def beforeAll(): Unit = { - webDriver = new HtmlUnitDriver + webDriver = new HtmlUnitDriver { + + getWebClient.setCssErrorHandler(new DefaultCssErrorHandler { + + private val cssWhiteList = List("bootstrap.min.css", "vis.min.css") + + private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith) + + override def warning(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.warning(e) + } + } + + override def fatalError(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.fatalError(e) + } + } + + override def error(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.error(e) + } + } + }) + } } override def afterAll(): Unit = { diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala index 47541015a3611..f0e0609604d15 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala @@ -21,12 +21,14 @@ package org.apache.spark.sql.hive.thriftserver import scala.util.Random +import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler import org.openqa.selenium.WebDriver import org.openqa.selenium.htmlunit.HtmlUnitDriver import org.scalatest.{Matchers, BeforeAndAfterAll} import org.scalatest.concurrent.Eventually._ import org.scalatest.selenium.WebBrowser import org.scalatest.time.SpanSugar._ +import org.w3c.css.sac.CSSParseException import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.apache.spark.sql.hive.HiveContext @@ -43,7 +45,33 @@ class UISeleniumSuite override def mode: ServerMode.Value = ServerMode.binary override def beforeAll(): Unit = { - webDriver = new HtmlUnitDriver + webDriver = new HtmlUnitDriver { + + getWebClient.setCssErrorHandler(new DefaultCssErrorHandler { + + private val cssWhiteList = List("bootstrap.min.css", "vis.min.css") + + private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith) + + override def warning(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.warning(e) + } + } + + override def fatalError(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.fatalError(e) + } + } + + override def error(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.error(e) + } + } + }) + } super.beforeAll() } diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index 2211f62383ce8..facae338a322d 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -19,12 +19,14 @@ package org.apache.spark.streaming import scala.collection.mutable.Queue +import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler import org.openqa.selenium.WebDriver import org.openqa.selenium.htmlunit.HtmlUnitDriver import org.scalatest._ import org.scalatest.concurrent.Eventually._ import org.scalatest.selenium.WebBrowser import org.scalatest.time.SpanSugar._ +import org.w3c.css.sac.CSSParseException import org.apache.spark._ @@ -32,7 +34,7 @@ import org.apache.spark._ /** - * Selenium tests for the Spark Web UI. + * Selenium tests for the Spark Streaming Web UI. */ class UISeleniumSuite extends FunSuite with WebBrowser with Matchers with BeforeAndAfterAll with TestSuiteBase { @@ -40,7 +42,33 @@ class UISeleniumSuite implicit var webDriver: WebDriver = _ override def beforeAll(): Unit = { - webDriver = new HtmlUnitDriver + webDriver = new HtmlUnitDriver { + + getWebClient.setCssErrorHandler(new DefaultCssErrorHandler { + + private val cssWhiteList = List("bootstrap.min.css", "vis.min.css") + + private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith) + + override def warning(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.warning(e) + } + } + + override def fatalError(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.fatalError(e) + } + } + + override def error(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.error(e) + } + } + }) + } } override def afterAll(): Unit = { From 42027289d4561d584fa078c334abe9cc60354d93 Mon Sep 17 00:00:00 2001 From: zsxwing Date: Tue, 9 Jun 2015 00:14:17 +0800 Subject: [PATCH 2/2] Add SparkUICssErrorHandler for all tests --- .../org/apache/spark/ui/UISeleniumSuite.scala | 51 ++++++++++--------- .../hive/thriftserver/UISeleniumSuite.scala | 29 +---------- .../spark/streaming/UISeleniumSuite.scala | 29 +---------- 3 files changed, 30 insertions(+), 79 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 3dd5075b4dabf..3aa672f8b713c 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -41,6 +41,31 @@ import org.apache.spark.deploy.history.HistoryServerSuite import org.apache.spark.shuffle.FetchFailedException import org.apache.spark.status.api.v1.{JacksonMessageWriter, StageStatus} +private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler { + + private val cssWhiteList = List("bootstrap.min.css", "vis.min.css") + + private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith) + + override def warning(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.warning(e) + } + } + + override def fatalError(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.fatalError(e) + } + } + + override def error(e: CSSParseException): Unit = { + if (!isInWhileList(e.getURI)) { + super.error(e) + } + } +} + /** * Selenium tests for the Spark Web UI. */ @@ -52,31 +77,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B override def beforeAll(): Unit = { webDriver = new HtmlUnitDriver { - - getWebClient.setCssErrorHandler(new DefaultCssErrorHandler { - - private val cssWhiteList = List("bootstrap.min.css", "vis.min.css") - - private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith) - - override def warning(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.warning(e) - } - } - - override def fatalError(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.fatalError(e) - } - } - - override def error(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.error(e) - } - } - }) + getWebClient.setCssErrorHandler(new SparkUICssErrorHandler) } } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala index e674cd1263eb5..806240e6de458 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala @@ -19,7 +19,6 @@ package org.apache.spark.sql.hive.thriftserver import scala.util.Random -import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.openqa.selenium.WebDriver import org.openqa.selenium.htmlunit.HtmlUnitDriver @@ -27,9 +26,9 @@ import org.scalatest.{BeforeAndAfterAll, Matchers} import org.scalatest.concurrent.Eventually._ import org.scalatest.selenium.WebBrowser import org.scalatest.time.SpanSugar._ -import org.w3c.css.sac.CSSParseException import org.apache.spark.sql.hive.HiveContext +import org.apache.spark.ui.SparkUICssErrorHandler class UISeleniumSuite extends HiveThriftJdbcTest @@ -43,31 +42,7 @@ class UISeleniumSuite override def beforeAll(): Unit = { webDriver = new HtmlUnitDriver { - - getWebClient.setCssErrorHandler(new DefaultCssErrorHandler { - - private val cssWhiteList = List("bootstrap.min.css", "vis.min.css") - - private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith) - - override def warning(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.warning(e) - } - } - - override def fatalError(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.fatalError(e) - } - } - - override def error(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.error(e) - } - } - }) + getWebClient.setCssErrorHandler(new SparkUICssErrorHandler) } super.beforeAll() } diff --git a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala index e387d716c770b..a08578680cff9 100644 --- a/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala +++ b/streaming/src/test/scala/org/apache/spark/streaming/UISeleniumSuite.scala @@ -19,16 +19,15 @@ package org.apache.spark.streaming import scala.collection.mutable.Queue -import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler import org.openqa.selenium.WebDriver import org.openqa.selenium.htmlunit.HtmlUnitDriver import org.scalatest._ import org.scalatest.concurrent.Eventually._ import org.scalatest.selenium.WebBrowser import org.scalatest.time.SpanSugar._ -import org.w3c.css.sac.CSSParseException import org.apache.spark._ +import org.apache.spark.ui.SparkUICssErrorHandler /** * Selenium tests for the Spark Streaming Web UI. @@ -40,31 +39,7 @@ class UISeleniumSuite override def beforeAll(): Unit = { webDriver = new HtmlUnitDriver { - - getWebClient.setCssErrorHandler(new DefaultCssErrorHandler { - - private val cssWhiteList = List("bootstrap.min.css", "vis.min.css") - - private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith) - - override def warning(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.warning(e) - } - } - - override def fatalError(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.fatalError(e) - } - } - - override def error(e: CSSParseException): Unit = { - if (!isInWhileList(e.getURI)) { - super.error(e) - } - } - }) + getWebClient.setCssErrorHandler(new SparkUICssErrorHandler) } }