From 72d32662d4744440e286a639783fed8dcf6c3948 Mon Sep 17 00:00:00 2001 From: Yingyi Bu Date: Fri, 7 May 2021 18:36:28 +0800 Subject: [PATCH 1/7] [SPARK-35144][SQL] Migrate to transformWithPruning for object rules ### What changes were proposed in this pull request? Added the following TreePattern enums: - APPEND_COLUMNS - DESERIALIZE_TO_OBJECT - LAMBDA_VARIABLE - MAP_OBJECTS - SERIALIZE_FROM_OBJECT - PROJECT - TYPED_FILTER Added tree traversal pruning to the following rules dealing with objects: - EliminateSerialization - CombineTypedFilters - EliminateMapObjects - ObjectSerializerPruning ### Why are the changes needed? Reduce the number of tree traversals and hence improve the query compilation latency. ### How was this patch tested? Existing tests. Closes #32451 from sigmod/object. Authored-by: Yingyi Bu Signed-off-by: Gengliang Wang --- .../catalyst/expressions/objects/objects.scala | 6 +++++- .../spark/sql/catalyst/optimizer/objects.scala | 15 ++++++++++----- .../plans/logical/basicLogicalOperators.scala | 2 ++ .../spark/sql/catalyst/plans/logical/object.scala | 8 ++++++++ .../sql/catalyst/rules/RuleIdCollection.scala | 5 +++++ .../spark/sql/catalyst/trees/TreePatterns.scala | 7 +++++++ 6 files changed, 37 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index 469c89572aed8..40378a3a5f72d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -33,7 +33,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.catalyst.trees.TernaryLike -import org.apache.spark.sql.catalyst.trees.TreePattern.{NULL_CHECK, TreePattern} +import org.apache.spark.sql.catalyst.trees.TreePattern._ import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData, MapData} import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.types._ @@ -669,6 +669,8 @@ case class LambdaVariable( private val accessor: (InternalRow, Int) => Any = InternalRow.getAccessor(dataType, nullable) + final override val nodePatterns: Seq[TreePattern] = Seq(LAMBDA_VARIABLE) + // Interpreted execution of `LambdaVariable` always get the 0-index element from input row. override def eval(input: InternalRow): Any = { assert(input.numFields == 1, @@ -781,6 +783,8 @@ case class MapObjects private( override def second: Expression = lambdaFunction override def third: Expression = inputData + final override val nodePatterns: Seq[TreePattern] = Seq(MAP_OBJECTS) + // The data with UserDefinedType are actually stored with the data type of its sqlType. // When we want to apply MapObjects on it, we have to use it. lazy private val inputDataType = inputData.dataType match { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala index 97712a05b7c21..52544ff3e241d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.objects._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ +import org.apache.spark.sql.catalyst.trees.TreePattern._ import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, UserDefinedType} /* @@ -35,7 +36,8 @@ import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, Use * representation of data item. For example back to back map operations. */ object EliminateSerialization extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transform { + def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning( + _.containsAnyPattern(DESERIALIZE_TO_OBJECT, APPEND_COLUMNS, TYPED_FILTER), ruleId) { case d @ DeserializeToObject(_, _, s: SerializeFromObject) if d.outputObjAttr.dataType == s.inputObjAttr.dataType => // Adds an extra Project here, to preserve the output expr id of `DeserializeToObject`. @@ -72,7 +74,8 @@ object EliminateSerialization extends Rule[LogicalPlan] { * merging the filter functions into one conjunctive function. */ object CombineTypedFilters extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transform { + def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning( + _.containsPattern(TYPED_FILTER), ruleId) { case t1 @ TypedFilter(_, _, _, _, t2 @ TypedFilter(_, _, _, _, child)) if t1.deserializer.dataType == t2.deserializer.dataType => TypedFilter( @@ -108,7 +111,8 @@ object CombineTypedFilters extends Rule[LogicalPlan] { * 2. no custom collection class specified representation of data item. */ object EliminateMapObjects extends Rule[LogicalPlan] { - def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { + def apply(plan: LogicalPlan): LogicalPlan = plan.transformAllExpressionsWithPruning( + _.containsAllPatterns(MAP_OBJECTS, LAMBDA_VARIABLE), ruleId) { case MapObjects(_, LambdaVariable(_, _, false, _), inputData, None) => inputData } } @@ -207,7 +211,8 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] { } } - def apply(plan: LogicalPlan): LogicalPlan = plan transform { + def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning( + _.containsAllPatterns(SERIALIZE_FROM_OBJECT, PROJECT), ruleId) { case p @ Project(_, s: SerializeFromObject) => // Prunes individual serializer if it is not used at all by above projection. val usedRefs = p.references @@ -252,7 +257,7 @@ object ReassignLambdaVariableID extends Rule[LogicalPlan] { var hasNegativeIds = false var hasPositiveIds = false - plan.transformAllExpressions { + plan.transformAllExpressionsWithPruning(_.containsPattern(LAMBDA_VARIABLE), ruleId) { case lr: LambdaVariable if lr.id == 0 => throw new IllegalStateException("LambdaVariable should never has 0 as its ID.") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index f5e92fb2deb0a..1bd16661fddee 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -70,6 +70,8 @@ case class Project(projectList: Seq[NamedExpression], child: LogicalPlan) override def output: Seq[Attribute] = projectList.map(_.toAttribute) override def maxRows: Option[Long] = child.maxRows + final override val nodePatterns: Seq[TreePattern] = Seq(PROJECT) + override lazy val resolved: Boolean = { val hasSpecialExpressions = projectList.exists ( _.collect { case agg: AggregateExpression => agg diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala index 6d61a86ab5ef7..1f7eb67bf1726 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/object.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedDeserializer import org.apache.spark.sql.catalyst.encoders._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.objects.Invoke +import org.apache.spark.sql.catalyst.trees.TreePattern._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode} import org.apache.spark.sql.types._ @@ -80,6 +81,7 @@ case class DeserializeToObject( deserializer: Expression, outputObjAttr: Attribute, child: LogicalPlan) extends UnaryNode with ObjectProducer { + final override val nodePatterns: Seq[TreePattern] = Seq(DESERIALIZE_TO_OBJECT) override protected def withNewChildInternal(newChild: LogicalPlan): DeserializeToObject = copy(child = newChild) } @@ -94,6 +96,8 @@ case class SerializeFromObject( override def output: Seq[Attribute] = serializer.map(_.toAttribute) + final override val nodePatterns: Seq[TreePattern] = Seq(SERIALIZE_FROM_OBJECT) + override protected def withNewChildInternal(newChild: LogicalPlan): SerializeFromObject = copy(child = newChild) } @@ -256,6 +260,8 @@ case class TypedFilter( override def output: Seq[Attribute] = child.output + final override val nodePatterns: Seq[TreePattern] = Seq(TYPED_FILTER) + def withObjectProducerChild(obj: LogicalPlan): Filter = { assert(obj.output.length == 1) Filter(typedCondition(obj.output.head), obj) @@ -354,6 +360,8 @@ case class AppendColumns( override def output: Seq[Attribute] = child.output ++ newColumns + final override val nodePatterns: Seq[TreePattern] = Seq(APPEND_COLUMNS) + def newColumns: Seq[Attribute] = serializer.map(_.toAttribute) override protected def withNewChildInternal(newChild: LogicalPlan): AppendColumns = diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala index 3a1fa6a8e523d..62f09d02ea146 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala @@ -89,13 +89,17 @@ object RuleIdCollection { // Catalyst Optimizer rules "org.apache.spark.sql.catalyst.optimizer.BooleanSimplification" :: "org.apache.spark.sql.catalyst.optimizer.CombineConcats" :: + "org.apache.spark.sql.catalyst.optimizer.CombineTypedFilters" :: "org.apache.spark.sql.catalyst.optimizer.ConstantFolding" :: "org.apache.spark.sql.catalyst.optimizer.ConstantPropagation" :: "org.apache.spark.sql.catalyst.optimizer.CostBasedJoinReorder" :: + "org.apache.spark.sql.catalyst.optimizer.EliminateMapObjects" :: "org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin" :: + "org.apache.spark.sql.catalyst.optimizer.EliminateSerialization" :: "org.apache.spark.sql.catalyst.optimizer.LikeSimplification" :: "org.apache.spark.sql.catalyst.optimizer.LimitPushDownThroughWindow" :: "org.apache.spark.sql.catalyst.optimizer.NullPropagation" :: + "org.apache.spark.sql.catalyst.optimizer.ObjectSerializerPruning" :: "org.apache.spark.sql.catalyst.optimizer.OptimizeCsvJsonExprs" :: "org.apache.spark.sql.catalyst.optimizer.OptimizeIn" :: "org.apache.spark.sql.catalyst.optimizer.Optimizer$OptimizeSubqueries" :: @@ -105,6 +109,7 @@ object RuleIdCollection { "org.apache.spark.sql.catalyst.optimizer.PushExtraPredicateThroughJoin" :: "org.apache.spark.sql.catalyst.optimizer.PushFoldableIntoBranches" :: "org.apache.spark.sql.catalyst.optimizer.PushLeftSemiLeftAntiThroughJoin" :: + "org.apache.spark.sql.catalyst.optimizer.ReassignLambdaVariableID" :: "org.apache.spark.sql.catalyst.optimizer.RemoveDispensableExpressions" :: "org.apache.spark.sql.catalyst.optimizer.ReorderAssociativeOperator" :: "org.apache.spark.sql.catalyst.optimizer.ReorderJoin" :: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala index b44847cfef0e2..fb384cd946d3c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala @@ -25,6 +25,7 @@ object TreePattern extends Enumeration { // Expression patterns (alphabetically ordered) val AND_OR: Value = Value(0) val ATTRIBUTE_REFERENCE: Value = Value + val APPEND_COLUMNS: Value = Value val BINARY_ARITHMETIC: Value = Value val BINARY_COMPARISON: Value = Value val CASE_WHEN: Value = Value @@ -32,6 +33,7 @@ object TreePattern extends Enumeration { val CONCAT: Value = Value val COUNT: Value = Value val CREATE_NAMED_STRUCT: Value = Value + val DESERIALIZE_TO_OBJECT: Value = Value val DYNAMIC_PRUNING_SUBQUERY: Value = Value val EXISTS_SUBQUERY = Value val EXPRESSION_WITH_RANDOM_SEED: Value = Value @@ -41,12 +43,15 @@ object TreePattern extends Enumeration { val IN_SUBQUERY: Value = Value val INSET: Value = Value val JSON_TO_STRUCT: Value = Value + val LAMBDA_VARIABLE: Value = Value val LIKE_FAMLIY: Value = Value val LIST_SUBQUERY: Value = Value val LITERAL: Value = Value + val MAP_OBJECTS: Value = Value val NOT: Value = Value val NULL_CHECK: Value = Value val NULL_LITERAL: Value = Value + val SERIALIZE_FROM_OBJECT: Value = Value val OUTER_REFERENCE: Value = Value val PLAN_EXPRESSION: Value = Value val SCALAR_SUBQUERY: Value = Value @@ -66,5 +71,7 @@ object TreePattern extends Enumeration { val LOCAL_RELATION: Value = Value val NATURAL_LIKE_JOIN: Value = Value val OUTER_JOIN: Value = Value + val PROJECT: Value = Value + val TYPED_FILTER: Value = Value val WINDOW: Value = Value } From d3b92eec454380207f3d5b5c0974cdaf8aa78b68 Mon Sep 17 00:00:00 2001 From: beliefer Date: Fri, 7 May 2021 10:54:43 +0000 Subject: [PATCH 2/7] [SPARK-35021][SQL] Group exception messages in connector/catalog ### What changes were proposed in this pull request? This PR group exception messages in `sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog`. ### Why are the changes needed? It will largely help with standardization of error messages and its maintenance. ### Does this PR introduce _any_ user-facing change? No. Error messages remain unchanged. ### How was this patch tested? No new tests - pass all original tests to make sure it doesn't break any existing behavior. Closes #32377 from beliefer/SPARK-35021. Lead-authored-by: beliefer Co-authored-by: gengjiaan Co-authored-by: Jiaan Geng Signed-off-by: Wenchen Fan --- .../catalog/CatalogV2Implicits.scala | 36 +++++++--------- .../sql/connector/catalog/CatalogV2Util.scala | 5 +-- .../sql/connector/catalog/LookupCatalog.scala | 6 +-- .../sql/errors/QueryCompilationErrors.scala | 42 +++++++++++++++++-- 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala index cc41d8ca9007f..39642fd541706 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala @@ -17,12 +17,12 @@ package org.apache.spark.sql.connector.catalog -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.BucketSpec import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.util.quoteIfNeeded import org.apache.spark.sql.connector.expressions.{BucketTransform, IdentityTransform, LogicalExpressions, Transform} +import org.apache.spark.sql.errors.QueryCompilationErrors /** * Conversion helpers for working with v2 [[CatalogPlugin]]. @@ -39,8 +39,7 @@ private[sql] object CatalogV2Implicits { implicit class BucketSpecHelper(spec: BucketSpec) { def asTransform: BucketTransform = { if (spec.sortColumnNames.nonEmpty) { - throw new AnalysisException( - s"Cannot convert bucketing with sort columns to a transform: $spec") + throw QueryCompilationErrors.cannotConvertBucketWithSortColumnsToTransformError(spec) } val references = spec.bucketColumnNames.map(col => reference(Seq(col))) @@ -53,14 +52,13 @@ private[sql] object CatalogV2Implicits { val (idTransforms, nonIdTransforms) = transforms.partition(_.isInstanceOf[IdentityTransform]) if (nonIdTransforms.nonEmpty) { - throw new AnalysisException("Transforms cannot be converted to partition columns: " + - nonIdTransforms.map(_.describe).mkString(", ")) + throw QueryCompilationErrors.cannotConvertTransformsToPartitionColumnsError(nonIdTransforms) } idTransforms.map(_.asInstanceOf[IdentityTransform]).map(_.reference).map { ref => val parts = ref.fieldNames if (parts.size > 1) { - throw new AnalysisException(s"Cannot partition by nested column: $ref") + throw QueryCompilationErrors.cannotPartitionByNestedColumnError(ref) } else { parts(0) } @@ -73,15 +71,14 @@ private[sql] object CatalogV2Implicits { case tableCatalog: TableCatalog => tableCatalog case _ => - throw new AnalysisException(s"Cannot use catalog ${plugin.name}: not a TableCatalog") + throw QueryCompilationErrors.cannotUseCatalogError(plugin, "not a TableCatalog") } def asNamespaceCatalog: SupportsNamespaces = plugin match { case namespaceCatalog: SupportsNamespaces => namespaceCatalog case _ => - throw new AnalysisException( - s"Cannot use catalog ${plugin.name}: does not support namespaces") + throw QueryCompilationErrors.cannotUseCatalogError(plugin, "does not support namespaces") } def isFunctionCatalog: Boolean = plugin match { @@ -93,8 +90,7 @@ private[sql] object CatalogV2Implicits { case functionCatalog: FunctionCatalog => functionCatalog case _ => - throw new AnalysisException( - s"Cannot use catalog '${plugin.name}': not a FunctionCatalog") + throw QueryCompilationErrors.cannotUseCatalogError(plugin, "not a FunctionCatalog") } } @@ -128,22 +124,22 @@ private[sql] object CatalogV2Implicits { case ns if ns.isEmpty => TableIdentifier(ident.name) case Array(dbName) => TableIdentifier(ident.name, Some(dbName)) case _ => - throw new AnalysisException( - s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.") + throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError( + quoted, "TableIdentifier") } def asFunctionIdentifier: FunctionIdentifier = ident.namespace() match { case ns if ns.isEmpty => FunctionIdentifier(ident.name()) case Array(dbName) => FunctionIdentifier(ident.name(), Some(dbName)) case _ => - throw new AnalysisException( - s"$quoted is not a valid FunctionIdentifier as it has more than 2 name parts.") + throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError( + quoted, "FunctionIdentifier") } } implicit class MultipartIdentifierHelper(parts: Seq[String]) { if (parts.isEmpty) { - throw new AnalysisException("multi-part identifier cannot be empty.") + throw QueryCompilationErrors.emptyMultipartIdentifierError() } def asIdentifier: Identifier = Identifier.of(parts.init.toArray, parts.last) @@ -152,16 +148,16 @@ private[sql] object CatalogV2Implicits { case Seq(tblName) => TableIdentifier(tblName) case Seq(dbName, tblName) => TableIdentifier(tblName, Some(dbName)) case _ => - throw new AnalysisException( - s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.") + throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError( + quoted, "TableIdentifier") } def asFunctionIdentifier: FunctionIdentifier = parts match { case Seq(funcName) => FunctionIdentifier(funcName) case Seq(dbName, funcName) => FunctionIdentifier(funcName, Some(dbName)) case _ => - throw new AnalysisException( - s"$quoted is not a valid FunctionIdentifier as it has more than 2 name parts.") + throw QueryCompilationErrors.identifierHavingMoreThanTwoNamePartsError( + quoted, "FunctionIdentifier") } def quoted: String = parts.map(quoteIfNeeded).mkString(".") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala index 02db2293ec64a..a779e50a1f214 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala @@ -22,10 +22,10 @@ import java.util.Collections import scala.collection.JavaConverters._ -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{NamedRelation, NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, UnresolvedV2Relation} import org.apache.spark.sql.catalyst.plans.logical.{AlterTable, CreateTableAsSelectStatement, CreateTableStatement, ReplaceTableAsSelectStatement, ReplaceTableStatement, SerdeInfo} import org.apache.spark.sql.connector.catalog.TableChange._ +import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation import org.apache.spark.sql.types.{ArrayType, DataType, MapType, NullType, StructField, StructType} import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -386,8 +386,7 @@ private[sql] object CatalogV2Util { case _ => dt.isInstanceOf[NullType] } if (containsNullType(dt)) { - throw new AnalysisException( - s"Cannot create tables with ${NullType.simpleString} type.") + throw QueryCompilationErrors.cannotCreateTablesWithNullTypeError() } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala index dcd352267a178..06358590a1e46 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala @@ -18,8 +18,8 @@ package org.apache.spark.sql.connector.catalog import org.apache.spark.internal.Logging -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} /** @@ -192,11 +192,11 @@ private[sql] trait LookupCatalog extends Logging { ident.namespace match { case Array(db) => FunctionIdentifier(ident.name, Some(db)) case _ => - throw new AnalysisException(s"Unsupported function name '$ident'") + throw QueryCompilationErrors.unsupportedFunctionNameError(ident.toString) } } - case _ => throw new AnalysisException("function is only supported in v1 catalog") + case _ => throw QueryCompilationErrors.functionUnsupportedInV2CatalogError() } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 9cc649ee32f16..7626c339cb420 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -22,17 +22,18 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{FunctionIdentifier, QualifiedTableName, TableIdentifier} import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NamespaceAlreadyExistsException, NoSuchNamespaceException, NoSuchTableException, ResolvedNamespace, ResolvedTable, ResolvedView, TableAlreadyExistsException} -import org.apache.spark.sql.catalyst.catalog.{CatalogTable, InvalidUDFClassException} +import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable, InvalidUDFClassException} import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, CreateMap, Expression, GroupingID, NamedExpression, SpecifiedWindowFrame, WindowFrame, WindowFunction, WindowSpecDefinition} import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, SerdeInfo} import org.apache.spark.sql.catalyst.trees.TreeNode import org.apache.spark.sql.catalyst.util.{toPrettySQL, FailFastMode, ParseMode, PermissiveMode} -import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, Table, TableCapability, TableChange, V1Table} +import org.apache.spark.sql.connector.catalog.{CatalogPlugin, Identifier, NamespaceChange, Table, TableCapability, TableChange, V1Table} import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ +import org.apache.spark.sql.connector.expressions.{NamedReference, Transform} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.Filter import org.apache.spark.sql.streaming.OutputMode -import org.apache.spark.sql.types.{AbstractDataType, DataType, StructField, StructType} +import org.apache.spark.sql.types.{AbstractDataType, DataType, NullType, StructField, StructType} /** * Object for grouping error messages from exceptions thrown during query compilation. @@ -1355,4 +1356,39 @@ private[spark] object QueryCompilationErrors { def cannotUseIntervalTypeInTableSchemaError(): Throwable = { new AnalysisException("Cannot use interval type in the table schema.") } + + def cannotConvertBucketWithSortColumnsToTransformError(spec: BucketSpec): Throwable = { + new AnalysisException( + s"Cannot convert bucketing with sort columns to a transform: $spec") + } + + def cannotConvertTransformsToPartitionColumnsError(nonIdTransforms: Seq[Transform]): Throwable = { + new AnalysisException("Transforms cannot be converted to partition columns: " + + nonIdTransforms.map(_.describe).mkString(", ")) + } + + def cannotPartitionByNestedColumnError(reference: NamedReference): Throwable = { + new AnalysisException(s"Cannot partition by nested column: $reference") + } + + def cannotUseCatalogError(plugin: CatalogPlugin, msg: String): Throwable = { + new AnalysisException(s"Cannot use catalog ${plugin.name}: $msg") + } + + def identifierHavingMoreThanTwoNamePartsError( + quoted: String, identifier: String): Throwable = { + new AnalysisException(s"$quoted is not a valid $identifier as it has more than 2 name parts.") + } + + def emptyMultipartIdentifierError(): Throwable = { + new AnalysisException("multi-part identifier cannot be empty.") + } + + def cannotCreateTablesWithNullTypeError(): Throwable = { + new AnalysisException(s"Cannot create tables with ${NullType.simpleString} type.") + } + + def functionUnsupportedInV2CatalogError(): Throwable = { + new AnalysisException("function is only supported in v1 catalog") + } } From 2634dbac35c5e8d5216b38fd4256f5fd059f341f Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Fri, 7 May 2021 21:55:08 +0900 Subject: [PATCH 3/7] [SPARK-35175][BUILD] Add linter for JavaScript source files ### What changes were proposed in this pull request? This PR proposes to add linter for JavaScript source files. [ESLint](https://eslint.org/) seems to be a popular linter for JavaScript so I choose it. ### Why are the changes needed? Linter enables us to check style and keeps code clean. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manually run `dev/lint-js` (Node.js and npm are required). In this PR, mainly indentation style is also fixed an linter passes. Closes #32274 from sarutak/introduce-eslint. Authored-by: Kousuke Saruta Signed-off-by: Kousuke Saruta --- .gitignore | 4 + .../apache/spark/ui/static/executorspage.js | 1328 +++++------ .../spark/ui/static/historypage-common.js | 2 + .../org/apache/spark/ui/static/historypage.js | 328 +-- .../spark/ui/static/initialize-tooltips.js | 4 +- .../org/apache/spark/ui/static/log-view.js | 8 +- .../apache/spark/ui/static/spark-dag-viz.js | 8 +- .../org/apache/spark/ui/static/stagepage.js | 2076 +++++++++-------- .../apache/spark/ui/static/streaming-page.js | 492 ++-- .../ui/static/structured-streaming-page.js | 299 +-- .../org/apache/spark/ui/static/table.js | 130 +- .../apache/spark/ui/static/timeline-view.js | 5 +- .../org/apache/spark/ui/static/utils.js | 161 +- .../org/apache/spark/ui/static/webui.js | 35 +- dev/.rat-excludes | 1 + dev/eslint.json | 24 + dev/lint-js | 56 + dev/package-lock.json | 979 ++++++++ dev/package.json | 5 + 19 files changed, 3539 insertions(+), 2406 deletions(-) create mode 100644 dev/eslint.json create mode 100755 dev/lint-js create mode 100644 dev/package-lock.json create mode 100644 dev/package.json diff --git a/.gitignore b/.gitignore index c01f70b4b4bcf..3208b1e56bf63 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ sql/docs sql/site lib_managed/ lint-r-report.log +lint-js-report.log log/ logs/ out/ @@ -105,3 +106,6 @@ spark-warehouse/ # For SBT .jvmopts + +# For Node.js +node_modules diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index 9133ef87a8a8e..2055c8ff11882 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -15,114 +15,120 @@ * limitations under the License. */ +/* global $, Mustache, createRESTEndPointForExecutorsPage, createRESTEndPointForMiscellaneousProcess, */ +/* global createTemplateURI, formatBytes, formatDuration, formatLogsCells, getStandAloneAppId, */ +/* global jQuery, setDataTableDefaults */ + var threadDumpEnabled = false; +/* eslint-disable no-unused-vars */ function setThreadDumpEnabled(val) { - threadDumpEnabled = val; + threadDumpEnabled = val; } +/* eslint-enable no-unused-vars */ function getThreadDumpEnabled() { - return threadDumpEnabled; + return threadDumpEnabled; } function formatStatus(status, type, row) { - if (row.isExcluded) { - return "Excluded"; - } + if (row.isExcluded) { + return "Excluded"; + } - if (status) { - if (row.excludedInStages.length == 0) { - return "Active" - } - return "Active (Excluded in Stages: [" + row.excludedInStages.join(", ") + "])"; + if (status) { + if (row.excludedInStages.length == 0) { + return "Active" } - return "Dead" + return "Active (Excluded in Stages: [" + row.excludedInStages.join(", ") + "])"; + } + return "Dead" } function formatProcessStatus(activeStatus) { - if (activeStatus) { - return "Active" - } - return "Dead" + if (activeStatus) { + return "Active" + } + return "Dead" } function formatResourceCells(resources) { - var result = "" - var count = 0 - $.each(resources, function (name, resInfo) { - if (count > 0) { - result += ", " - } - result += name + ': [' + resInfo.addresses.join(", ") + ']' - count += 1 - }); - return result + var result = "" + var count = 0 + $.each(resources, function (name, resInfo) { + if (count > 0) { + result += ", " + } + result += name + ': [' + resInfo.addresses.join(", ") + ']'; + count += 1 + }); + return result } jQuery.extend(jQuery.fn.dataTableExt.oSort, { - "title-numeric-pre": function (a) { - var x = a.match(/title="*(-?[0-9\.]+)/)[1]; - return parseFloat(x); - }, - - "title-numeric-asc": function (a, b) { - return ((a < b) ? -1 : ((a > b) ? 1 : 0)); - }, - - "title-numeric-desc": function (a, b) { - return ((a < b) ? 1 : ((a > b) ? -1 : 0)); - } + "title-numeric-pre": function (a) { + var x = a.match(/title="*(-?[0-9.]+)/)[1]; + return parseFloat(x); + }, + + "title-numeric-asc": function (a, b) { + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, + + "title-numeric-desc": function (a, b) { + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } }); $(document).ajaxStop($.unblockUI); $(document).ajaxStart(function () { - $.blockUI({message: '

Loading Executors Page...

'}); + $.blockUI({message: '

Loading Executors Page...

'}); }); function logsExist(execs) { - return execs.some(function(exec) { - return !($.isEmptyObject(exec["executorLogs"])); - }); + return execs.some(function(exec) { + return !($.isEmptyObject(exec["executorLogs"])); + }); } // Determine Color Opacity from 0.5-1 // activeTasks range from 0 to maxTasks function activeTasksAlpha(activeTasks, maxTasks) { - return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1; + return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1; } function activeTasksStyle(activeTasks, maxTasks) { - return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : ""; + return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : ""; } // failedTasks range max at 10% failure, alpha max = 1 function failedTasksAlpha(failedTasks, totalTasks) { - return totalTasks > 0 ? - (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1; + return totalTasks > 0 ? + (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1; } function failedTasksStyle(failedTasks, totalTasks) { - return failedTasks > 0 ? - ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : ""; + return failedTasks > 0 ? + ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : ""; } // totalDuration range from 0 to 50% GC time, alpha max = 1 function totalDurationAlpha(totalGCTime, totalDuration) { - return totalDuration > 0 ? - (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1; + return totalDuration > 0 ? + (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1; } // When GCTimePercent is edited change ToolTips.TASK_TIME to match var GCTimePercent = 0.1; function totalDurationStyle(totalGCTime, totalDuration) { - // Red if GC time over GCTimePercent of total time - return (totalGCTime > GCTimePercent * totalDuration) ? - ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : ""; + // Red if GC time over GCTimePercent of total time + return (totalGCTime > GCTimePercent * totalDuration) ? + ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : ""; } function totalDurationColor(totalGCTime, totalDuration) { - return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black"; + return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black"; } var sumOptionalColumns = [3, 4]; @@ -131,622 +137,624 @@ var execDataTable; var sumDataTable; function reselectCheckboxesBasedOnTaskTableState() { - var allChecked = true; - if (typeof execDataTable !== "undefined") { - for (var k = 0; k < execOptionalColumns.length; k++) { - if (execDataTable.column(execOptionalColumns[k]).visible()) { - $("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true); - } else { - allChecked = false; - } - } - } - if (allChecked) { - $("#select-all-box").prop("checked", true); + var allChecked = true; + if (typeof execDataTable !== "undefined") { + for (var k = 0; k < execOptionalColumns.length; k++) { + if (execDataTable.column(execOptionalColumns[k]).visible()) { + $("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true); + } else { + allChecked = false; + } } + } + if (allChecked) { + $("#select-all-box").prop("checked", true); + } } $(document).ready(function () { - setDataTableDefaults(); - - var executorsSummary = $("#active-executors"); - - getStandAloneAppId(function (appId) { - - var endPoint = createRESTEndPointForExecutorsPage(appId); - $.getJSON(endPoint, function (response, status, jqXHR) { - var allExecCnt = 0; - var allRDDBlocks = 0; - var allMemoryUsed = 0; - var allMaxMemory = 0; - var allOnHeapMemoryUsed = 0; - var allOnHeapMaxMemory = 0; - var allOffHeapMemoryUsed = 0; - var allOffHeapMaxMemory = 0; - var allDiskUsed = 0; - var allTotalCores = 0; - var allMaxTasks = 0; - var allActiveTasks = 0; - var allFailedTasks = 0; - var allCompletedTasks = 0; - var allTotalTasks = 0; - var allTotalDuration = 0; - var allTotalGCTime = 0; - var allTotalInputBytes = 0; - var allTotalShuffleRead = 0; - var allTotalShuffleWrite = 0; - var allTotalExcluded = 0; - - var activeExecCnt = 0; - var activeRDDBlocks = 0; - var activeMemoryUsed = 0; - var activeMaxMemory = 0; - var activeOnHeapMemoryUsed = 0; - var activeOnHeapMaxMemory = 0; - var activeOffHeapMemoryUsed = 0; - var activeOffHeapMaxMemory = 0; - var activeDiskUsed = 0; - var activeTotalCores = 0; - var activeMaxTasks = 0; - var activeActiveTasks = 0; - var activeFailedTasks = 0; - var activeCompletedTasks = 0; - var activeTotalTasks = 0; - var activeTotalDuration = 0; - var activeTotalGCTime = 0; - var activeTotalInputBytes = 0; - var activeTotalShuffleRead = 0; - var activeTotalShuffleWrite = 0; - var activeTotalExcluded = 0; - - var deadExecCnt = 0; - var deadRDDBlocks = 0; - var deadMemoryUsed = 0; - var deadMaxMemory = 0; - var deadOnHeapMemoryUsed = 0; - var deadOnHeapMaxMemory = 0; - var deadOffHeapMemoryUsed = 0; - var deadOffHeapMaxMemory = 0; - var deadDiskUsed = 0; - var deadTotalCores = 0; - var deadMaxTasks = 0; - var deadActiveTasks = 0; - var deadFailedTasks = 0; - var deadCompletedTasks = 0; - var deadTotalTasks = 0; - var deadTotalDuration = 0; - var deadTotalGCTime = 0; - var deadTotalInputBytes = 0; - var deadTotalShuffleRead = 0; - var deadTotalShuffleWrite = 0; - var deadTotalExcluded = 0; - - response.forEach(function (exec) { - var memoryMetrics = { - usedOnHeapStorageMemory: 0, - usedOffHeapStorageMemory: 0, - totalOnHeapStorageMemory: 0, - totalOffHeapStorageMemory: 0 - }; - - exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics; - }); - - response.forEach(function (exec) { - allExecCnt += 1; - allRDDBlocks += exec.rddBlocks; - allMemoryUsed += exec.memoryUsed; - allMaxMemory += exec.maxMemory; - allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory; - allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory; - allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory; - allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory; - allDiskUsed += exec.diskUsed; - allTotalCores += exec.totalCores; - allMaxTasks += exec.maxTasks; - allActiveTasks += exec.activeTasks; - allFailedTasks += exec.failedTasks; - allCompletedTasks += exec.completedTasks; - allTotalTasks += exec.totalTasks; - allTotalDuration += exec.totalDuration; - allTotalGCTime += exec.totalGCTime; - allTotalInputBytes += exec.totalInputBytes; - allTotalShuffleRead += exec.totalShuffleRead; - allTotalShuffleWrite += exec.totalShuffleWrite; - allTotalExcluded += exec.isExcluded ? 1 : 0; - if (exec.isActive) { - activeExecCnt += 1; - activeRDDBlocks += exec.rddBlocks; - activeMemoryUsed += exec.memoryUsed; - activeMaxMemory += exec.maxMemory; - activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory; - activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory; - activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory; - activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory; - activeDiskUsed += exec.diskUsed; - activeTotalCores += exec.totalCores; - activeMaxTasks += exec.maxTasks; - activeActiveTasks += exec.activeTasks; - activeFailedTasks += exec.failedTasks; - activeCompletedTasks += exec.completedTasks; - activeTotalTasks += exec.totalTasks; - activeTotalDuration += exec.totalDuration; - activeTotalGCTime += exec.totalGCTime; - activeTotalInputBytes += exec.totalInputBytes; - activeTotalShuffleRead += exec.totalShuffleRead; - activeTotalShuffleWrite += exec.totalShuffleWrite; - activeTotalExcluded += exec.isExcluded ? 1 : 0; + setDataTableDefaults(); + + var executorsSummary = $("#active-executors"); + + getStandAloneAppId(function (appId) { + + var endPoint = createRESTEndPointForExecutorsPage(appId); + $.getJSON(endPoint, function (response, _ignored_status, _ignored_jqXHR) { + var allExecCnt = 0; + var allRDDBlocks = 0; + var allMemoryUsed = 0; + var allMaxMemory = 0; + var allOnHeapMemoryUsed = 0; + var allOnHeapMaxMemory = 0; + var allOffHeapMemoryUsed = 0; + var allOffHeapMaxMemory = 0; + var allDiskUsed = 0; + var allTotalCores = 0; + var allMaxTasks = 0; + var allActiveTasks = 0; + var allFailedTasks = 0; + var allCompletedTasks = 0; + var allTotalTasks = 0; + var allTotalDuration = 0; + var allTotalGCTime = 0; + var allTotalInputBytes = 0; + var allTotalShuffleRead = 0; + var allTotalShuffleWrite = 0; + var allTotalExcluded = 0; + + var activeExecCnt = 0; + var activeRDDBlocks = 0; + var activeMemoryUsed = 0; + var activeMaxMemory = 0; + var activeOnHeapMemoryUsed = 0; + var activeOnHeapMaxMemory = 0; + var activeOffHeapMemoryUsed = 0; + var activeOffHeapMaxMemory = 0; + var activeDiskUsed = 0; + var activeTotalCores = 0; + var activeMaxTasks = 0; + var activeActiveTasks = 0; + var activeFailedTasks = 0; + var activeCompletedTasks = 0; + var activeTotalTasks = 0; + var activeTotalDuration = 0; + var activeTotalGCTime = 0; + var activeTotalInputBytes = 0; + var activeTotalShuffleRead = 0; + var activeTotalShuffleWrite = 0; + var activeTotalExcluded = 0; + + var deadExecCnt = 0; + var deadRDDBlocks = 0; + var deadMemoryUsed = 0; + var deadMaxMemory = 0; + var deadOnHeapMemoryUsed = 0; + var deadOnHeapMaxMemory = 0; + var deadOffHeapMemoryUsed = 0; + var deadOffHeapMaxMemory = 0; + var deadDiskUsed = 0; + var deadTotalCores = 0; + var deadMaxTasks = 0; + var deadActiveTasks = 0; + var deadFailedTasks = 0; + var deadCompletedTasks = 0; + var deadTotalTasks = 0; + var deadTotalDuration = 0; + var deadTotalGCTime = 0; + var deadTotalInputBytes = 0; + var deadTotalShuffleRead = 0; + var deadTotalShuffleWrite = 0; + var deadTotalExcluded = 0; + + response.forEach(function (exec) { + var memoryMetrics = { + usedOnHeapStorageMemory: 0, + usedOffHeapStorageMemory: 0, + totalOnHeapStorageMemory: 0, + totalOffHeapStorageMemory: 0 + }; + + // TODO: Replace hasOwnProperty with prototype.hasOwnProperty after we find it's safe to do. + /* eslint-disable no-prototype-builtins */ + exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics; + }); + + response.forEach(function (exec) { + allExecCnt += 1; + allRDDBlocks += exec.rddBlocks; + allMemoryUsed += exec.memoryUsed; + allMaxMemory += exec.maxMemory; + allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory; + allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory; + allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory; + allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory; + allDiskUsed += exec.diskUsed; + allTotalCores += exec.totalCores; + allMaxTasks += exec.maxTasks; + allActiveTasks += exec.activeTasks; + allFailedTasks += exec.failedTasks; + allCompletedTasks += exec.completedTasks; + allTotalTasks += exec.totalTasks; + allTotalDuration += exec.totalDuration; + allTotalGCTime += exec.totalGCTime; + allTotalInputBytes += exec.totalInputBytes; + allTotalShuffleRead += exec.totalShuffleRead; + allTotalShuffleWrite += exec.totalShuffleWrite; + allTotalExcluded += exec.isExcluded ? 1 : 0; + if (exec.isActive) { + activeExecCnt += 1; + activeRDDBlocks += exec.rddBlocks; + activeMemoryUsed += exec.memoryUsed; + activeMaxMemory += exec.maxMemory; + activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory; + activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory; + activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory; + activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory; + activeDiskUsed += exec.diskUsed; + activeTotalCores += exec.totalCores; + activeMaxTasks += exec.maxTasks; + activeActiveTasks += exec.activeTasks; + activeFailedTasks += exec.failedTasks; + activeCompletedTasks += exec.completedTasks; + activeTotalTasks += exec.totalTasks; + activeTotalDuration += exec.totalDuration; + activeTotalGCTime += exec.totalGCTime; + activeTotalInputBytes += exec.totalInputBytes; + activeTotalShuffleRead += exec.totalShuffleRead; + activeTotalShuffleWrite += exec.totalShuffleWrite; + activeTotalExcluded += exec.isExcluded ? 1 : 0; + } else { + deadExecCnt += 1; + deadRDDBlocks += exec.rddBlocks; + deadMemoryUsed += exec.memoryUsed; + deadMaxMemory += exec.maxMemory; + deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory; + deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory; + deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory; + deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory; + deadDiskUsed += exec.diskUsed; + deadTotalCores += exec.totalCores; + deadMaxTasks += exec.maxTasks; + deadActiveTasks += exec.activeTasks; + deadFailedTasks += exec.failedTasks; + deadCompletedTasks += exec.completedTasks; + deadTotalTasks += exec.totalTasks; + deadTotalDuration += exec.totalDuration; + deadTotalGCTime += exec.totalGCTime; + deadTotalInputBytes += exec.totalInputBytes; + deadTotalShuffleRead += exec.totalShuffleRead; + deadTotalShuffleWrite += exec.totalShuffleWrite; + deadTotalExcluded += exec.isExcluded ? 1 : 0; // todo - TEST BACKWARDS compatibility history? + } + }); + + var totalSummary = { + "execCnt": ( "Total(" + allExecCnt + ")"), + "allRDDBlocks": allRDDBlocks, + "allMemoryUsed": allMemoryUsed, + "allMaxMemory": allMaxMemory, + "allOnHeapMemoryUsed": allOnHeapMemoryUsed, + "allOnHeapMaxMemory": allOnHeapMaxMemory, + "allOffHeapMemoryUsed": allOffHeapMemoryUsed, + "allOffHeapMaxMemory": allOffHeapMaxMemory, + "allDiskUsed": allDiskUsed, + "allTotalCores": allTotalCores, + "allMaxTasks": allMaxTasks, + "allActiveTasks": allActiveTasks, + "allFailedTasks": allFailedTasks, + "allCompletedTasks": allCompletedTasks, + "allTotalTasks": allTotalTasks, + "allTotalDuration": allTotalDuration, + "allTotalGCTime": allTotalGCTime, + "allTotalInputBytes": allTotalInputBytes, + "allTotalShuffleRead": allTotalShuffleRead, + "allTotalShuffleWrite": allTotalShuffleWrite, + "allTotalExcluded": allTotalExcluded + }; + var activeSummary = { + "execCnt": ( "Active(" + activeExecCnt + ")"), + "allRDDBlocks": activeRDDBlocks, + "allMemoryUsed": activeMemoryUsed, + "allMaxMemory": activeMaxMemory, + "allOnHeapMemoryUsed": activeOnHeapMemoryUsed, + "allOnHeapMaxMemory": activeOnHeapMaxMemory, + "allOffHeapMemoryUsed": activeOffHeapMemoryUsed, + "allOffHeapMaxMemory": activeOffHeapMaxMemory, + "allDiskUsed": activeDiskUsed, + "allTotalCores": activeTotalCores, + "allMaxTasks": activeMaxTasks, + "allActiveTasks": activeActiveTasks, + "allFailedTasks": activeFailedTasks, + "allCompletedTasks": activeCompletedTasks, + "allTotalTasks": activeTotalTasks, + "allTotalDuration": activeTotalDuration, + "allTotalGCTime": activeTotalGCTime, + "allTotalInputBytes": activeTotalInputBytes, + "allTotalShuffleRead": activeTotalShuffleRead, + "allTotalShuffleWrite": activeTotalShuffleWrite, + "allTotalExcluded": activeTotalExcluded + }; + var deadSummary = { + "execCnt": ( "Dead(" + deadExecCnt + ")" ), + "allRDDBlocks": deadRDDBlocks, + "allMemoryUsed": deadMemoryUsed, + "allMaxMemory": deadMaxMemory, + "allOnHeapMemoryUsed": deadOnHeapMemoryUsed, + "allOnHeapMaxMemory": deadOnHeapMaxMemory, + "allOffHeapMemoryUsed": deadOffHeapMemoryUsed, + "allOffHeapMaxMemory": deadOffHeapMaxMemory, + "allDiskUsed": deadDiskUsed, + "allTotalCores": deadTotalCores, + "allMaxTasks": deadMaxTasks, + "allActiveTasks": deadActiveTasks, + "allFailedTasks": deadFailedTasks, + "allCompletedTasks": deadCompletedTasks, + "allTotalTasks": deadTotalTasks, + "allTotalDuration": deadTotalDuration, + "allTotalGCTime": deadTotalGCTime, + "allTotalInputBytes": deadTotalInputBytes, + "allTotalShuffleRead": deadTotalShuffleRead, + "allTotalShuffleWrite": deadTotalShuffleWrite, + "allTotalExcluded": deadTotalExcluded + }; + + var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; + $.get(createTemplateURI(appId, "executorspage"), function (template) { + + executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data)); + var selector = "#active-executors-table"; + var conf = { + "data": response, + "columns": [ + { + data: function (row, type) { + return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id; + } + }, + {data: 'hostPort'}, + { + data: 'isActive', + render: function (data, type, row) { + return formatStatus (data, type, row); + } + }, + {data: 'rddBlocks'}, + { + data: function (row, type) { + if (type !== 'display') + return row.memoryUsed; + else + return (formatBytes(row.memoryUsed, type) + ' / ' + + formatBytes(row.maxMemory, type)); + } + }, + { + data: function (row, type) { + if (type !== 'display') + return row.memoryMetrics.usedOnHeapStorageMemory; + else + return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' + + formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type)); + } + }, + { + data: function (row, type) { + if (type !== 'display') + return row.memoryMetrics.usedOffHeapStorageMemory; + else + return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' + + formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type)); + } + }, + { + data: function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics; + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.JVMHeapMemory; + else + return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type)); } else { - deadExecCnt += 1; - deadRDDBlocks += exec.rddBlocks; - deadMemoryUsed += exec.memoryUsed; - deadMaxMemory += exec.maxMemory; - deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory; - deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory; - deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory; - deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory; - deadDiskUsed += exec.diskUsed; - deadTotalCores += exec.totalCores; - deadMaxTasks += exec.maxTasks; - deadActiveTasks += exec.activeTasks; - deadFailedTasks += exec.failedTasks; - deadCompletedTasks += exec.completedTasks; - deadTotalTasks += exec.totalTasks; - deadTotalDuration += exec.totalDuration; - deadTotalGCTime += exec.totalGCTime; - deadTotalInputBytes += exec.totalInputBytes; - deadTotalShuffleRead += exec.totalShuffleRead; - deadTotalShuffleWrite += exec.totalShuffleWrite; - deadTotalExcluded += exec.isExcluded ? 1 : 0; // todo - TEST BACKWARDS compatibility history? + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } } - }); - - var totalSummary = { - "execCnt": ( "Total(" + allExecCnt + ")"), - "allRDDBlocks": allRDDBlocks, - "allMemoryUsed": allMemoryUsed, - "allMaxMemory": allMaxMemory, - "allOnHeapMemoryUsed": allOnHeapMemoryUsed, - "allOnHeapMaxMemory": allOnHeapMaxMemory, - "allOffHeapMemoryUsed": allOffHeapMemoryUsed, - "allOffHeapMaxMemory": allOffHeapMaxMemory, - "allDiskUsed": allDiskUsed, - "allTotalCores": allTotalCores, - "allMaxTasks": allMaxTasks, - "allActiveTasks": allActiveTasks, - "allFailedTasks": allFailedTasks, - "allCompletedTasks": allCompletedTasks, - "allTotalTasks": allTotalTasks, - "allTotalDuration": allTotalDuration, - "allTotalGCTime": allTotalGCTime, - "allTotalInputBytes": allTotalInputBytes, - "allTotalShuffleRead": allTotalShuffleRead, - "allTotalShuffleWrite": allTotalShuffleWrite, - "allTotalExcluded": allTotalExcluded - }; - var activeSummary = { - "execCnt": ( "Active(" + activeExecCnt + ")"), - "allRDDBlocks": activeRDDBlocks, - "allMemoryUsed": activeMemoryUsed, - "allMaxMemory": activeMaxMemory, - "allOnHeapMemoryUsed": activeOnHeapMemoryUsed, - "allOnHeapMaxMemory": activeOnHeapMaxMemory, - "allOffHeapMemoryUsed": activeOffHeapMemoryUsed, - "allOffHeapMaxMemory": activeOffHeapMaxMemory, - "allDiskUsed": activeDiskUsed, - "allTotalCores": activeTotalCores, - "allMaxTasks": activeMaxTasks, - "allActiveTasks": activeActiveTasks, - "allFailedTasks": activeFailedTasks, - "allCompletedTasks": activeCompletedTasks, - "allTotalTasks": activeTotalTasks, - "allTotalDuration": activeTotalDuration, - "allTotalGCTime": activeTotalGCTime, - "allTotalInputBytes": activeTotalInputBytes, - "allTotalShuffleRead": activeTotalShuffleRead, - "allTotalShuffleWrite": activeTotalShuffleWrite, - "allTotalExcluded": activeTotalExcluded - }; - var deadSummary = { - "execCnt": ( "Dead(" + deadExecCnt + ")" ), - "allRDDBlocks": deadRDDBlocks, - "allMemoryUsed": deadMemoryUsed, - "allMaxMemory": deadMaxMemory, - "allOnHeapMemoryUsed": deadOnHeapMemoryUsed, - "allOnHeapMaxMemory": deadOnHeapMaxMemory, - "allOffHeapMemoryUsed": deadOffHeapMemoryUsed, - "allOffHeapMaxMemory": deadOffHeapMaxMemory, - "allDiskUsed": deadDiskUsed, - "allTotalCores": deadTotalCores, - "allMaxTasks": deadMaxTasks, - "allActiveTasks": deadActiveTasks, - "allFailedTasks": deadFailedTasks, - "allCompletedTasks": deadCompletedTasks, - "allTotalTasks": deadTotalTasks, - "allTotalDuration": deadTotalDuration, - "allTotalGCTime": deadTotalGCTime, - "allTotalInputBytes": deadTotalInputBytes, - "allTotalShuffleRead": deadTotalShuffleRead, - "allTotalShuffleWrite": deadTotalShuffleWrite, - "allTotalExcluded": deadTotalExcluded + } + }, + { + data: function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics; + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.OnHeapExecutionMemory; + else + return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type)); + } else { + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } + } + } + }, + { + data: function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics; + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.OnHeapStorageMemory; + else + return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type)); + } else { + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } + } + } + }, + { + data: function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics; + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.DirectPoolMemory; + else + return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.MappedPoolMemory, type)); + } else { + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } + } + } + }, + {data: 'diskUsed', render: formatBytes}, + {data: 'totalCores'}, + {name: 'resourcesCol', data: 'resources', render: formatResourceCells, orderable: false}, + {name: 'resourceProfileIdCol', data: 'resourceProfileId'}, + { + data: 'activeTasks', + "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks)); + } + } + }, + { + data: 'failedTasks', + "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks)); + } + } + }, + {data: 'completedTasks'}, + {data: 'totalTasks'}, + { + data: function (row, type) { + return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration + }, + "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) { + if (oData.totalDuration > 0) { + $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration)); + $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration)); + } + } + }, + {data: 'totalInputBytes', render: formatBytes}, + {data: 'totalShuffleRead', render: formatBytes}, + {data: 'totalShuffleWrite', render: formatBytes}, + {name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells}, + { + name: 'threadDumpCol', + data: 'id', render: function (data, type) { + return type === 'display' ? ("Thread Dump" ) : data; + } + } + ], + "order": [[0, "asc"]], + "columnDefs": [ + {"visible": false, "targets": 5}, + {"visible": false, "targets": 6}, + {"visible": false, "targets": 7}, + {"visible": false, "targets": 8}, + {"visible": false, "targets": 9}, + {"visible": false, "targets": 10}, + {"visible": false, "targets": 13}, + {"visible": false, "targets": 14} + ], + "deferRender": true + }; + + execDataTable = $(selector).DataTable(conf); + execDataTable.column('executorLogsCol:name').visible(logsExist(response)); + execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled()); + $('#active-executors [data-toggle="tooltip"]').tooltip(); + + // This section should be visible once API gives the response. + $('.active-process-container').hide(); + var endPoint = createRESTEndPointForMiscellaneousProcess(appId); + $.getJSON(endPoint, function( response, _ignored_status, _ignored_jqXHR ) { + if (response.length) { + var processSummaryResponse = response; + var processSummaryConf = { + "data": processSummaryResponse, + "columns": [{ + data: "id" + }, + { + data: "hostPort" + }, + { + data: function(row) { + return formatProcessStatus(row.isActive); + } + }, + { + data: "totalCores" + }, + { + data: "processLogs", + render: formatLogsCells + }, + ], + "deferRender": true, + "order": [ + [0, "asc"] + ], + "bAutoWidth": false, + "oLanguage": { + "sEmptyTable": "No data to show yet" + } }; + $("#active-process-table").DataTable(processSummaryConf); + $('.active-process-container').show() + } + }); - var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; - $.get(createTemplateURI(appId, "executorspage"), function (template) { - - executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data)); - var selector = "#active-executors-table"; - var conf = { - "data": response, - "columns": [ - { - data: function (row, type) { - return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id; - } - }, - {data: 'hostPort'}, - { - data: 'isActive', - render: function (data, type, row) { - return formatStatus (data, type, row); - } - }, - {data: 'rddBlocks'}, - { - data: function (row, type) { - if (type !== 'display') - return row.memoryUsed; - else - return (formatBytes(row.memoryUsed, type) + ' / ' + - formatBytes(row.maxMemory, type)); - } - }, - { - data: function (row, type) { - if (type !== 'display') - return row.memoryMetrics.usedOnHeapStorageMemory; - else - return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' + - formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type)); - } - }, - { - data: function (row, type) { - if (type !== 'display') - return row.memoryMetrics.usedOffHeapStorageMemory; - else - return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' + - formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type)); - } - }, - { - data: function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics; - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.JVMHeapMemory; - else - return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - } - }, - { - data: function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics; - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.OnHeapExecutionMemory; - else - return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - } - }, - { - data: function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics; - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.OnHeapStorageMemory; - else - return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - } - }, - { - data: function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics; - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.DirectPoolMemory; - else - return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.MappedPoolMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - } - }, - {data: 'diskUsed', render: formatBytes}, - {data: 'totalCores'}, - {name: 'resourcesCol', data: 'resources', render: formatResourceCells, orderable: false}, - {name: 'resourceProfileIdCol', data: 'resourceProfileId'}, - { - data: 'activeTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks)); - } - } - }, - { - data: 'failedTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks)); - } - } - }, - {data: 'completedTasks'}, - {data: 'totalTasks'}, - { - data: function (row, type) { - return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration - }, - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (oData.totalDuration > 0) { - $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration)); - $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration)); - } - } - }, - {data: 'totalInputBytes', render: formatBytes}, - {data: 'totalShuffleRead', render: formatBytes}, - {data: 'totalShuffleWrite', render: formatBytes}, - {name: 'executorLogsCol', data: 'executorLogs', render: formatLogsCells}, - { - name: 'threadDumpCol', - data: 'id', render: function (data, type) { - return type === 'display' ? ("Thread Dump" ) : data; - } - } - ], - "order": [[0, "asc"]], - "columnDefs": [ - {"visible": false, "targets": 5}, - {"visible": false, "targets": 6}, - {"visible": false, "targets": 7}, - {"visible": false, "targets": 8}, - {"visible": false, "targets": 9}, - {"visible": false, "targets": 10}, - {"visible": false, "targets": 13}, - {"visible": false, "targets": 14} - ], - "deferRender": true - }; - - execDataTable = $(selector).DataTable(conf); - execDataTable.column('executorLogsCol:name').visible(logsExist(response)); - execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled()); - $('#active-executors [data-toggle="tooltip"]').tooltip(); - - // This section should be visible once API gives the response. - $('.active-process-container').hide() - var endPoint = createRESTEndPointForMiscellaneousProcess(appId); - $.getJSON(endPoint, function( response, status, jqXHR ) { - if (response.length) { - var processSummaryResponse = response; - var processSummaryConf = { - "data": processSummaryResponse, - "columns": [{ - data: "id" - }, - { - data: "hostPort" - }, - { - data: function(row) { - return formatProcessStatus(row.isActive); - } - }, - { - data: "totalCores" - }, - { - data: "processLogs", - render: formatLogsCells - }, - ], - "deferRender": true, - "order": [ - [0, "asc"] - ], - "bAutoWidth": false, - "oLanguage": { - "sEmptyTable": "No data to show yet" - } - }; - $("#active-process-table").DataTable(processSummaryConf); - $('.active-process-container').show() - } - }); - - var sumSelector = "#summary-execs-table"; - var sumConf = { - "data": [activeSummary, deadSummary, totalSummary], - "columns": [ - { - data: 'execCnt', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - $(nTd).css('font-weight', 'bold'); - } - }, - {data: 'allRDDBlocks'}, - { - data: function (row, type) { - if (type !== 'display') - return row.allMemoryUsed - else - return (formatBytes(row.allMemoryUsed, type) + ' / ' + - formatBytes(row.allMaxMemory, type)); - } - }, - { - data: function (row, type) { - if (type !== 'display') - return row.allOnHeapMemoryUsed; - else - return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' + - formatBytes(row.allOnHeapMaxMemory, type)); - } - }, - { - data: function (row, type) { - if (type !== 'display') - return row.allOffHeapMemoryUsed; - else - return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' + - formatBytes(row.allOffHeapMaxMemory, type)); - } - }, - {data: 'allDiskUsed', render: formatBytes}, - {data: 'allTotalCores'}, - { - data: 'allActiveTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks)); - } - } - }, - { - data: 'allFailedTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks)); - } - } - }, - {data: 'allCompletedTasks'}, - {data: 'allTotalTasks'}, - { - data: function (row, type) { - return type === 'display' ? (formatDuration(row.allTotalDuration) + ' (' + formatDuration(row.allTotalGCTime) + ')') : row.allTotalDuration - }, - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (oData.allTotalDuration > 0) { - $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration)); - $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration)); - } - } - }, - {data: 'allTotalInputBytes', render: formatBytes}, - {data: 'allTotalShuffleRead', render: formatBytes}, - {data: 'allTotalShuffleWrite', render: formatBytes}, - {data: 'allTotalExcluded'} - ], - "paging": false, - "searching": false, - "info": false, - "columnDefs": [ - {"visible": false, "targets": 3}, - {"visible": false, "targets": 4} - ] - - }; - - sumDataTable = $(sumSelector).DataTable(sumConf); - $('#execSummary [data-toggle="tooltip"]').tooltip(); - - $("#showAdditionalMetrics").append( - "" + - "
" + - "
Select All
" + - "
On Heap Memory
" + - "
Off Heap Memory
" + - "
Peak JVM Memory OnHeap / OffHeap
" + - "
Peak Execution Memory OnHeap / OffHeap
" + - "
Peak Storage Memory OnHeap / OffHeap
" + - "
Peak Pool Memory Direct / Mapped
" + - "
Resources
" + - "
Resource Profile Id
" + - "
"); - - reselectCheckboxesBasedOnTaskTableState(); - - $("#additionalMetrics").click(function() { - $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed"); - $("#toggle-metrics").toggleClass("d-none"); - if (window.localStorage) { - window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class')); - } - }); - - $(".toggle-vis").on("click", function() { - var thisBox = $(this); - if (thisBox.is("#select-all-box")) { - var sumColumn = sumDataTable.columns(sumOptionalColumns); - var execColumn = execDataTable.columns(execOptionalColumns); - if (thisBox.is(":checked")) { - $(".toggle-vis").prop("checked", true); - sumColumn.visible(true); - execColumn.visible(true); - } else { - $(".toggle-vis").prop("checked", false); - sumColumn.visible(false); - execColumn.visible(false); - } - } else { - var execColIdx = thisBox.attr("data-exec-col-idx"); - var execCol = execDataTable.column(execColIdx); - execCol.visible(!execCol.visible()); - var sumColIdx = thisBox.attr("data-sum-col-idx"); - if (sumColIdx) { - var sumCol = sumDataTable.column(sumColIdx); - sumCol.visible(!sumCol.visible()); - } - } - }); - - if (window.localStorage) { - if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null && - window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) { - $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed"); - $("#toggle-metrics").toggleClass("d-none"); - } + var sumSelector = "#summary-execs-table"; + var sumConf = { + "data": [activeSummary, deadSummary, totalSummary], + "columns": [ + { + data: 'execCnt', + "fnCreatedCell": function (nTd, _ignored_sData, _ignored_oData, _ignored_iRow, _ignored_iCol) { + $(nTd).css('font-weight', 'bold'); + } + }, + {data: 'allRDDBlocks'}, + { + data: function (row, type) { + if (type !== 'display') + return row.allMemoryUsed; + else + return (formatBytes(row.allMemoryUsed, type) + ' / ' + + formatBytes(row.allMaxMemory, type)); + } + }, + { + data: function (row, type) { + if (type !== 'display') + return row.allOnHeapMemoryUsed; + else + return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' + + formatBytes(row.allOnHeapMaxMemory, type)); + } + }, + { + data: function (row, type) { + if (type !== 'display') + return row.allOffHeapMemoryUsed; + else + return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' + + formatBytes(row.allOffHeapMaxMemory, type)); + } + }, + {data: 'allDiskUsed', render: formatBytes}, + {data: 'allTotalCores'}, + { + data: 'allActiveTasks', + "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks)); + } + } + }, + { + data: 'allFailedTasks', + "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks)); + } + } + }, + {data: 'allCompletedTasks'}, + {data: 'allTotalTasks'}, + { + data: function (row, type) { + return type === 'display' ? (formatDuration(row.allTotalDuration) + ' (' + formatDuration(row.allTotalGCTime) + ')') : row.allTotalDuration + }, + "fnCreatedCell": function (nTd, sData, oData, _ignored_iRow, _ignored_iCol) { + if (oData.allTotalDuration > 0) { + $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration)); + $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration)); } - }); + } + }, + {data: 'allTotalInputBytes', render: formatBytes}, + {data: 'allTotalShuffleRead', render: formatBytes}, + {data: 'allTotalShuffleWrite', render: formatBytes}, + {data: 'allTotalExcluded'} + ], + "paging": false, + "searching": false, + "info": false, + "columnDefs": [ + {"visible": false, "targets": 3}, + {"visible": false, "targets": 4} + ] + + }; + + sumDataTable = $(sumSelector).DataTable(sumConf); + $('#execSummary [data-toggle="tooltip"]').tooltip(); + + $("#showAdditionalMetrics").append( + "" + + "
" + + "
Select All
" + + "
On Heap Memory
" + + "
Off Heap Memory
" + + "
Peak JVM Memory OnHeap / OffHeap
" + + "
Peak Execution Memory OnHeap / OffHeap
" + + "
Peak Storage Memory OnHeap / OffHeap
" + + "
Peak Pool Memory Direct / Mapped
" + + "
Resources
" + + "
Resource Profile Id
" + + "
"); + + reselectCheckboxesBasedOnTaskTableState(); + + $("#additionalMetrics").click(function() { + $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed"); + $("#toggle-metrics").toggleClass("d-none"); + if (window.localStorage) { + window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class')); + } + }); + + $(".toggle-vis").on("click", function() { + var thisBox = $(this); + if (thisBox.is("#select-all-box")) { + var sumColumn = sumDataTable.columns(sumOptionalColumns); + var execColumn = execDataTable.columns(execOptionalColumns); + if (thisBox.is(":checked")) { + $(".toggle-vis").prop("checked", true); + sumColumn.visible(true); + execColumn.visible(true); + } else { + $(".toggle-vis").prop("checked", false); + sumColumn.visible(false); + execColumn.visible(false); + } + } else { + var execColIdx = thisBox.attr("data-exec-col-idx"); + var execCol = execDataTable.column(execColIdx); + execCol.visible(!execCol.visible()); + var sumColIdx = thisBox.attr("data-sum-col-idx"); + if (sumColIdx) { + var sumCol = sumDataTable.column(sumColIdx); + sumCol.visible(!sumCol.visible()); + } + } }); + + if (window.localStorage) { + if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null && + window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) { + $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed"); + $("#toggle-metrics").toggleClass("d-none"); + } + } + }); }); + }); }); diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js b/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js index 4cfe46ec914ae..cd8cf098ef1c0 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-common.js @@ -15,6 +15,8 @@ * limitations under the License. */ +/* global $, formatTimeMillis, getTimeZone */ + $(document).ready(function() { if ($('#last-updated').length) { var lastUpdatedMillis = Number($('#last-updated').text()); diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index aa542a733fe75..b334bceb5a039 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -15,11 +15,15 @@ * limitations under the License. */ +/* global $, Mustache, formatDuration, formatTimeMillis, jQuery, uiRoot */ + var appLimit = -1; +/* eslint-disable no-unused-vars */ function setAppLimit(val) { - appLimit = val; + appLimit = val; } +/* eslint-enable no-unused-vars*/ function makeIdNumeric(id) { var strs = id.split("_"); @@ -30,8 +34,8 @@ function makeIdNumeric(id) { var resl = strs[0] + "_" + strs[1] + "_"; var diff = 10 - appSeqNum.length; while (diff > 0) { - resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters - diff--; + resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters + diff--; } resl += appSeqNum; return resl; @@ -39,7 +43,7 @@ function makeIdNumeric(id) { function getParameterByName(name, searchString) { var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"), - results = regex.exec(searchString); + results = regex.exec(searchString); return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); } @@ -56,183 +60,185 @@ function getColumnIndex(columns, columnName) { } jQuery.extend( jQuery.fn.dataTableExt.oSort, { - "title-numeric-pre": function ( a ) { - var x = a.match(/title="*(-?[0-9\.]+)/)[1]; - return parseFloat( x ); - }, + "title-numeric-pre": function ( a ) { + var x = a.match(/title="*(-?[0-9.]+)/)[1]; + return parseFloat( x ); + }, - "title-numeric-asc": function ( a, b ) { - return ((a < b) ? -1 : ((a > b) ? 1 : 0)); - }, + "title-numeric-asc": function ( a, b ) { + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, - "title-numeric-desc": function ( a, b ) { - return ((a < b) ? 1 : ((a > b) ? -1 : 0)); - } -} ); + "title-numeric-desc": function ( a, b ) { + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } +}); jQuery.extend( jQuery.fn.dataTableExt.oSort, { - "appid-numeric-pre": function ( a ) { - var x = a.match(/title="*(-?[0-9a-zA-Z\-\_]+)/)[1]; - return makeIdNumeric(x); - }, + "appid-numeric-pre": function ( a ) { + var x = a.match(/title="*(-?[0-9a-zA-Z\-_]+)/)[1]; + return makeIdNumeric(x); + }, - "appid-numeric-asc": function ( a, b ) { - return ((a < b) ? -1 : ((a > b) ? 1 : 0)); - }, + "appid-numeric-asc": function ( a, b ) { + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, - "appid-numeric-desc": function ( a, b ) { - return ((a < b) ? 1 : ((a > b) ? -1 : 0)); - } -} ); + "appid-numeric-desc": function ( a, b ) { + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } +}); jQuery.extend( jQuery.fn.dataTableExt.ofnSearch, { - "appid-numeric": function ( a ) { - return a.replace(/[\r\n]/g, " ").replace(/<.*?>/g, ""); - } -} ); + "appid-numeric": function ( a ) { + return a.replace(/[\r\n]/g, " ").replace(/<.*?>/g, ""); + } +}); $(document).ajaxStop($.unblockUI); $(document).ajaxStart(function(){ - $.blockUI({ message: '

Loading history summary...

'}); + $.blockUI({ message: '

Loading history summary...

'}); }); $(document).ready(function() { - $.extend( $.fn.dataTable.defaults, { - stateSave: true, - lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]], - pageLength: 20 - }); + $.extend( $.fn.dataTable.defaults, { + stateSave: true, + lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]], + pageLength: 20 + }); + + var historySummary = $("#history-summary"); + var searchString = window.location.search; + var requestedIncomplete = getParameterByName("showIncomplete", searchString); + requestedIncomplete = (requestedIncomplete == "true" ? true : false); + + var appParams = { + limit: appLimit, + status: (requestedIncomplete ? "running" : "completed") + }; + + $.getJSON(uiRoot + "/api/v1/applications", appParams, function(response, _ignored_status, _ignored_jqXHR) { + var array = []; + var hasMultipleAttempts = false; + for (var i in response) { + var app = response[i]; + if (app["attempts"][0]["completed"] == requestedIncomplete) { + continue; // if we want to show for Incomplete, we skip the completed apps; otherwise skip incomplete ones. + } + var version = "Unknown" + if (app["attempts"].length > 0) { + version = app["attempts"][0]["appSparkVersion"] + } + var id = app["id"]; + var name = app["name"]; + if (app["attempts"].length > 1) { + hasMultipleAttempts = true; + } - var historySummary = $("#history-summary"); - var searchString = window.location.search; - var requestedIncomplete = getParameterByName("showIncomplete", searchString); - requestedIncomplete = (requestedIncomplete == "true" ? true : false); + // TODO: Replace hasOwnProperty with prototype.hasOwnProperty after we find it's safe to do. + /* eslint-disable no-prototype-builtins */ + for (var j in app["attempts"]) { + var attempt = app["attempts"][j]; + attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]); + attempt["endTime"] = formatTimeMillis(attempt["endTimeEpoch"]); + attempt["lastUpdated"] = formatTimeMillis(attempt["lastUpdatedEpoch"]); + attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" + + (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs"; + attempt["durationMillisec"] = attempt["duration"]; + attempt["duration"] = formatDuration(attempt["duration"]); + attempt["id"] = id; + attempt["name"] = name; + attempt["version"] = version; + attempt["attemptUrl"] = uiRoot + "/history/" + id + "/" + + (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "jobs/"; + array.push(attempt); + } + /* eslint-enable no-prototype-builtins */ + } + if(array.length < 20) { + $.fn.dataTable.defaults.paging = false; + } - var appParams = { - limit: appLimit, - status: (requestedIncomplete ? "running" : "completed") + var data = { + "uiroot": uiRoot, + "applications": array, + "hasMultipleAttempts": hasMultipleAttempts, + "showCompletedColumns": !requestedIncomplete, }; - $.getJSON(uiRoot + "/api/v1/applications", appParams, function(response,status,jqXHR) { - var array = []; - var hasMultipleAttempts = false; - for (var i in response) { - var app = response[i]; - if (app["attempts"][0]["completed"] == requestedIncomplete) { - continue; // if we want to show for Incomplete, we skip the completed apps; otherwise skip incomplete ones. - } - var version = "Unknown" - if (app["attempts"].length > 0) { - version = app["attempts"][0]["appSparkVersion"] - } - var id = app["id"]; - var name = app["name"]; - if (app["attempts"].length > 1) { - hasMultipleAttempts = true; - } - - for (var j in app["attempts"]) { - var attempt = app["attempts"][j]; - attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]); - attempt["endTime"] = formatTimeMillis(attempt["endTimeEpoch"]); - attempt["lastUpdated"] = formatTimeMillis(attempt["lastUpdatedEpoch"]); - attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" + - (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs"; - attempt["durationMillisec"] = attempt["duration"]; - attempt["duration"] = formatDuration(attempt["duration"]); - attempt["id"] = id; - attempt["name"] = name; - attempt["version"] = version; - attempt["attemptUrl"] = uiRoot + "/history/" + id + "/" + - (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "jobs/"; - - array.push(attempt); - } - } - if(array.length < 20) { - $.fn.dataTable.defaults.paging = false; - } - - var data = { - "uiroot": uiRoot, - "applications": array, - "hasMultipleAttempts": hasMultipleAttempts, - "showCompletedColumns": !requestedIncomplete, + $.get(uiRoot + "/static/historypage-template.html", function(template) { + var sibling = historySummary.prev(); + historySummary.detach(); + var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); + var attemptIdColumnName = 'attemptId'; + var startedColumnName = 'started'; + var completedColumnName = 'completed'; + var durationColumnName = 'duration'; + var conf = { + "data": array, + "columns": [ + {name: 'version', data: 'version' }, + { + name: 'appId', + type: "appid-numeric", + data: 'id', + render: (id, type, row) => `${id}` + }, + {name: 'appName', data: 'name' }, + { + name: attemptIdColumnName, + data: 'attemptId', + render: (attemptId, type, row) => (attemptId ? `${attemptId}` : '') + }, + {name: startedColumnName, data: 'startTime' }, + {name: completedColumnName, data: 'endTime' }, + {name: durationColumnName, type: "title-numeric", data: 'duration' }, + {name: 'user', data: 'sparkUser' }, + {name: 'lastUpdated', data: 'lastUpdated' }, + { + name: 'eventLog', + data: 'log', + render: (log, _ignored_type, _ignored_row) => `Download` + }, + ], + "aoColumnDefs": [ + { + aTargets: [0, 1, 2], + fnCreatedCell: (nTd, _ignored_sData, _ignored_oData, _ignored_iRow, _ignored_iCol) => { + if (hasMultipleAttempts) { + $(nTd).css('background-color', '#fff'); + } + } + }, + ], + "autoWidth": false, + "deferRender": true }; - $.get(uiRoot + "/static/historypage-template.html", function(template) { - var sibling = historySummary.prev(); - historySummary.detach(); - var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); - var attemptIdColumnName = 'attemptId'; - var startedColumnName = 'started'; - var completedColumnName = 'completed'; - var durationColumnName = 'duration'; - var conf = { - "data": array, - "columns": [ - {name: 'version', data: 'version' }, - { - name: 'appId', - type: "appid-numeric", - data: 'id', - render: (id, type, row) => `${id}` - }, - {name: 'appName', data: 'name' }, - { - name: attemptIdColumnName, - data: 'attemptId', - render: (attemptId, type, row) => (attemptId ? `${attemptId}` : '') - }, - {name: startedColumnName, data: 'startTime' }, - {name: completedColumnName, data: 'endTime' }, - {name: durationColumnName, type: "title-numeric", data: 'duration' }, - {name: 'user', data: 'sparkUser' }, - {name: 'lastUpdated', data: 'lastUpdated' }, - { - name: 'eventLog', - data: 'log', - render: (log, type, row) => `Download` - }, - ], - "aoColumnDefs": [ - { - aTargets: [0, 1, 2], - fnCreatedCell: (nTd, sData, oData, iRow, iCol) => { - if (hasMultipleAttempts) { - $(nTd).css('background-color', '#fff'); - } - } - }, - ], - "autoWidth": false, - "deferRender": true - }; - - if (hasMultipleAttempts) { - conf.rowsGroup = [ - 'appId:name', - 'version:name', - 'appName:name' - ]; - } else { - conf.columns = removeColumnByName(conf.columns, attemptIdColumnName); - } - - var defaultSortColumn = completedColumnName; - if (requestedIncomplete) { - defaultSortColumn = startedColumnName; - conf.columns = removeColumnByName(conf.columns, completedColumnName); - conf.columns = removeColumnByName(conf.columns, durationColumnName); - } - conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]]; - conf.columnDefs = [ - {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]} + if (hasMultipleAttempts) { + conf.rowsGroup = [ + 'appId:name', + 'version:name', + 'appName:name' ]; - historySummary.append(apps); - apps.DataTable(conf); - sibling.after(historySummary); - $('#history-summary [data-toggle="tooltip"]').tooltip(); - }); + } else { + conf.columns = removeColumnByName(conf.columns, attemptIdColumnName); + } + + var defaultSortColumn = completedColumnName; + if (requestedIncomplete) { + defaultSortColumn = startedColumnName; + conf.columns = removeColumnByName(conf.columns, completedColumnName); + conf.columns = removeColumnByName(conf.columns, durationColumnName); + } + conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]]; + conf.columnDefs = [ + {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]} + ]; + historySummary.append(apps); + apps.DataTable(conf); + sibling.after(historySummary); + $('#history-summary [data-toggle="tooltip"]').tooltip(); }); + }); }); diff --git a/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js b/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js index 70f355dfb49cb..b273a19783f8f 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js +++ b/core/src/main/resources/org/apache/spark/ui/static/initialize-tooltips.js @@ -15,7 +15,9 @@ * limitations under the License. */ +/* global $ */ + $(document).ready(function(){ - $("[data-toggle=tooltip]").tooltip({container: 'body'}); + $("[data-toggle=tooltip]").tooltip({container: 'body'}); }); diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js index b5c43e5788bc3..1a4f9254742ff 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/log-view.js +++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js @@ -15,6 +15,8 @@ * limitations under the License. */ +/* global $ */ + var baseParams; var curLogLength; @@ -59,11 +61,12 @@ function getRESTEndPoint() { var words = document.baseURI.split('/'); var ind = words.indexOf("proxy"); if (ind > 0) { - return words.slice(0, ind + 2).join('/') + "/log"; + return words.slice(0, ind + 2).join('/') + "/log"; } return "/log" } +/* eslint-disable no-unused-vars */ function loadMore() { var offset = Math.max(startByte - byteLength, 0); var moreByteLength = Math.min(byteLength, startByte); @@ -139,4 +142,5 @@ function initLogPage(params, logLen, start, end, totLogLen, defaultLen) { if (startByte == 0) { disableMoreButton(); } -} \ No newline at end of file +} +/* eslint-enable no-unused-vars */ \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js index 48a3c93cce914..6a0cd012146ec 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js +++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js @@ -51,6 +51,8 @@ * since it was forked (commit 101503833a8ce5fe369547f6addf3e71172ce10b). */ +/* global $, appBasePath, d3, dagreD3, graphlibDot, uiRoot */ + var VizConstants = { svgMarginX: 16, svgMarginY: 16, @@ -166,7 +168,7 @@ function renderDagViz(forJob) { } // Find cached RDDs and mark them as such - metadataContainer().selectAll(".cached-rdd").each(function(v) { + metadataContainer().selectAll(".cached-rdd").each(function(_ignored_v) { var rddId = d3.select(this).text().trim(); var nodeId = VizConstants.nodePrefix + rddId; svg.selectAll("g." + nodeId).classed("cached", true); @@ -180,7 +182,7 @@ function renderDagViz(forJob) { svg.selectAll("g[id=" + stageClusterId + "] g." + opClusterId).classed("barrier", true) }); - metadataContainer().selectAll(".indeterminate-rdd").each(function(v) { + metadataContainer().selectAll(".indeterminate-rdd").each(function(_ignored_v) { var rddId = d3.select(this).text().trim(); var nodeId = VizConstants.nodePrefix + rddId; svg.selectAll("g." + nodeId).classed("indeterminate", true); @@ -275,7 +277,7 @@ function renderDagVizForJob(svgContainer) { // If there are any incoming edges into this graph, keep track of them to render // them separately later. Note that we cannot draw them now because we need to // put these edges in a separate container that is on top of all stage graphs. - metadata.selectAll(".incoming-edge").each(function(v) { + metadata.selectAll(".incoming-edge").each(function(_ignored_v) { var edge = d3.select(this).text().trim().split(","); // e.g. 3,4 => [3, 4] crossStageEdges.push(edge); }); diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js index 8b32fe7d3e20d..fea595d0a6216 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js @@ -15,284 +15,291 @@ * limitations under the License. */ +/* global $, ConvertDurationString, Mustache, createRESTEndPointForExecutorsPage */ +/* global createTemplateURI, formatBytes, formatDate, formatDuration, formatLogsCells */ +/* global getStandAloneAppId, setDataTableDefaults, uiRoot */ + var shouldBlockUI = true; $(document).ajaxStop(function () { - if (shouldBlockUI) { - $.unblockUI(); - shouldBlockUI = false; - } + if (shouldBlockUI) { + $.unblockUI(); + shouldBlockUI = false; + } }); $(document).ajaxStart(function () { - if (shouldBlockUI) { - $.blockUI({message: '

Loading Stage Page...

'}); - } + if (shouldBlockUI) { + $.blockUI({message: '

Loading Stage Page...

'}); + } }); $.extend( $.fn.dataTable.ext.type.order, { - "duration-pre": ConvertDurationString, - - "duration-asc": function ( a, b ) { - a = ConvertDurationString( a ); - b = ConvertDurationString( b ); - return ((a < b) ? -1 : ((a > b) ? 1 : 0)); - }, - - "duration-desc": function ( a, b ) { - a = ConvertDurationString( a ); - b = ConvertDurationString( b ); - return ((a < b) ? 1 : ((a > b) ? -1 : 0)); - }, - - "size-pre": function (data) { - var floatValue = parseFloat(data) - return isNaN(floatValue) ? 0 : floatValue; - }, - - "size-asc": function (a, b) { - a = parseFloat(a); - b = parseFloat(b); - return ((a < b) ? -1 : ((a > b) ? 1 : 0)); - }, - - "size-desc": function (a, b) { - a = parseFloat(a); - b = parseFloat(b); - return ((a < b) ? 1 : ((a > b) ? -1 : 0)); - } -} ); + "duration-pre": ConvertDurationString, + + "duration-asc": function ( a, b ) { + a = ConvertDurationString( a ); + b = ConvertDurationString( b ); + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, + + "duration-desc": function ( a, b ) { + a = ConvertDurationString( a ); + b = ConvertDurationString( b ); + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + }, + + "size-pre": function (data) { + var floatValue = parseFloat(data) + return isNaN(floatValue) ? 0 : floatValue; + }, + + "size-asc": function (a, b) { + a = parseFloat(a); + b = parseFloat(b); + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, + + "size-desc": function (a, b) { + a = parseFloat(a); + b = parseFloat(b); + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } +}); // This function will only parse the URL under certain format // e.g. (history) https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1 // e.g. (proxy) https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1 function stageEndPoint(appId) { - var queryString = document.baseURI.split('?'); - var words = document.baseURI.split('/'); - var indexOfProxy = words.indexOf("proxy"); - var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1]; - if (indexOfProxy > 0) { - var appId = words[indexOfProxy + 1]; - var newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/'); - return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId; - } - var indexOfHistory = words.indexOf("history"); - if (indexOfHistory > 0) { - var appId = words[indexOfHistory + 1]; - var appAttemptId = words[indexOfHistory + 2]; - var newBaseURI = words.slice(0, words.indexOf("history")).join('/'); - if (isNaN(appAttemptId) || appAttemptId == "0") { - return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId; - } else { - return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId; - } + var queryString = document.baseURI.split('?'); + var words = document.baseURI.split('/'); + var indexOfProxy = words.indexOf("proxy"); + var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1]; + var newBaseURI; + if (indexOfProxy > 0) { + appId = words[indexOfProxy + 1]; + newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/'); + return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId; + } + var indexOfHistory = words.indexOf("history"); + if (indexOfHistory > 0) { + appId = words[indexOfHistory + 1]; + var appAttemptId = words[indexOfHistory + 2]; + newBaseURI = words.slice(0, words.indexOf("history")).join('/'); + if (isNaN(appAttemptId) || appAttemptId == "0") { + return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId; + } else { + return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId; } - return uiRoot + "/api/v1/applications/" + appId + "/stages/" + stageId; + } + return uiRoot + "/api/v1/applications/" + appId + "/stages/" + stageId; } function getColumnNameForTaskMetricSummary(columnKey) { - switch(columnKey) { - case "executorRunTime": - return "Duration"; + switch(columnKey) { + case "executorRunTime": + return "Duration"; - case "jvmGcTime": - return "GC Time"; + case "jvmGcTime": + return "GC Time"; - case "gettingResultTime": - return "Getting Result Time"; + case "gettingResultTime": + return "Getting Result Time"; - case "inputMetrics": - return "Input Size / Records"; + case "inputMetrics": + return "Input Size / Records"; - case "outputMetrics": - return "Output Size / Records"; + case "outputMetrics": + return "Output Size / Records"; - case "peakExecutionMemory": - return "Peak Execution Memory"; + case "peakExecutionMemory": + return "Peak Execution Memory"; - case "resultSerializationTime": - return "Result Serialization Time"; + case "resultSerializationTime": + return "Result Serialization Time"; - case "schedulerDelay": - return "Scheduler Delay"; + case "schedulerDelay": + return "Scheduler Delay"; - case "diskBytesSpilled": - return "Spill (disk)"; + case "diskBytesSpilled": + return "Spill (disk)"; - case "memoryBytesSpilled": - return "Spill (memory)"; + case "memoryBytesSpilled": + return "Spill (memory)"; - case "shuffleReadMetrics": - return "Shuffle Read Size / Records"; + case "shuffleReadMetrics": + return "Shuffle Read Size / Records"; - case "shuffleWriteMetrics": - return "Shuffle Write Size / Records"; + case "shuffleWriteMetrics": + return "Shuffle Write Size / Records"; - case "executorDeserializeTime": - return "Task Deserialization Time"; + case "executorDeserializeTime": + return "Task Deserialization Time"; - case "shuffleReadBlockedTime": - return "Shuffle Read Blocked Time"; + case "shuffleReadBlockedTime": + return "Shuffle Read Blocked Time"; - case "shuffleRemoteReads": - return "Shuffle Remote Reads"; + case "shuffleRemoteReads": + return "Shuffle Remote Reads"; - case "shuffleWriteTime": - return "Shuffle Write Time"; + case "shuffleWriteTime": + return "Shuffle Write Time"; - default: - return "NA"; - } + default: + return "NA"; + } } function displayRowsForSummaryMetricsTable(row, type, columnIndex) { - switch(row.columnKey) { - case 'inputMetrics': - var str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " + - row.data.recordsRead[columnIndex]; - return str; - - case 'outputMetrics': - var str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " + - row.data.recordsWritten[columnIndex]; - return str; - - case 'shuffleReadMetrics': - var str = formatBytes(row.data.readBytes[columnIndex], type) + " / " + - row.data.readRecords[columnIndex]; - return str; - - case 'shuffleReadBlockedTime': - var str = formatDuration(row.data.fetchWaitTime[columnIndex]); - return str; - - case 'shuffleRemoteReads': - var str = formatBytes(row.data.remoteBytesRead[columnIndex], type); - return str; - - case 'shuffleWriteMetrics': - var str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " + - row.data.writeRecords[columnIndex]; - return str; - - case 'shuffleWriteTime': - var str = formatDuration(row.data.writeTime[columnIndex] / 1000000.0); - return str; - - default: - return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled' - || row.columnKey == 'diskBytesSpilled') ? formatBytes( - row.data[columnIndex], type) : (formatDuration(row.data[columnIndex])); - - } + var str; + switch(row.columnKey) { + case 'inputMetrics': + str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " + + row.data.recordsRead[columnIndex]; + return str; + + case 'outputMetrics': + str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " + + row.data.recordsWritten[columnIndex]; + return str; + + case 'shuffleReadMetrics': + str = formatBytes(row.data.readBytes[columnIndex], type) + " / " + + row.data.readRecords[columnIndex]; + return str; + + case 'shuffleReadBlockedTime': + str = formatDuration(row.data.fetchWaitTime[columnIndex]); + return str; + + case 'shuffleRemoteReads': + str = formatBytes(row.data.remoteBytesRead[columnIndex], type); + return str; + + case 'shuffleWriteMetrics': + str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " + + row.data.writeRecords[columnIndex]; + return str; + + case 'shuffleWriteTime': + str = formatDuration(row.data.writeTime[columnIndex] / 1000000.0); + return str; + + default: + return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled' + || row.columnKey == 'diskBytesSpilled') ? formatBytes( + row.data[columnIndex], type) : (formatDuration(row.data[columnIndex])); + + } } function createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTable) { - var taskMetricsTable = "#summary-metrics-table"; - if ($.fn.dataTable.isDataTable(taskMetricsTable)) { - taskSummaryMetricsDataTable.clear().draw(); - taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw(); - } else { - var taskConf = { - "data": taskSummaryMetricsTable, - "columns": [ - {data : 'metric'}, - // Min - { - data: function (row, type) { - return displayRowsForSummaryMetricsTable(row, type, 0); - } - }, - // 25th percentile - { - data: function (row, type) { - return displayRowsForSummaryMetricsTable(row, type, 1); - } - }, - // Median - { - data: function (row, type) { - return displayRowsForSummaryMetricsTable(row, type, 2); - } - }, - // 75th percentile - { - data: function (row, type) { - return displayRowsForSummaryMetricsTable(row, type, 3); - } - }, - // Max - { - data: function (row, type) { - return displayRowsForSummaryMetricsTable(row, type, 4); - } - } - ], - "columnDefs": [ - { "type": "duration", "targets": 1 }, - { "type": "duration", "targets": 2 }, - { "type": "duration", "targets": 3 }, - { "type": "duration", "targets": 4 }, - { "type": "duration", "targets": 5 } - ], - "paging": false, - "searching": false, - "order": [[0, "asc"]], - "bSort": false, - "bAutoWidth": false, - "oLanguage": { - "sEmptyTable": "No tasks have reported metrics yet" - } - }; - taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf); - } - taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice(); + var taskMetricsTable = "#summary-metrics-table"; + if ($.fn.dataTable.isDataTable(taskMetricsTable)) { + taskSummaryMetricsDataTable.clear().draw(); + taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw(); + } else { + var taskConf = { + "data": taskSummaryMetricsTable, + "columns": [ + {data : 'metric'}, + // Min + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 0); + } + }, + // 25th percentile + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 1); + } + }, + // Median + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 2); + } + }, + // 75th percentile + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 3); + } + }, + // Max + { + data: function (row, type) { + return displayRowsForSummaryMetricsTable(row, type, 4); + } + } + ], + "columnDefs": [ + { "type": "duration", "targets": 1 }, + { "type": "duration", "targets": 2 }, + { "type": "duration", "targets": 3 }, + { "type": "duration", "targets": 4 }, + { "type": "duration", "targets": 5 } + ], + "paging": false, + "searching": false, + "order": [[0, "asc"]], + "bSort": false, + "bAutoWidth": false, + "oLanguage": { + "sEmptyTable": "No tasks have reported metrics yet" + } + }; + taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf); + } + taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice(); } function createRowMetadataForColumn(colKey, data, checkboxId) { var row = { - "metric": getColumnNameForTaskMetricSummary(colKey), - "data": data, - "checkboxId": checkboxId, - "columnKey": colKey + "metric": getColumnNameForTaskMetricSummary(colKey), + "data": data, + "checkboxId": checkboxId, + "columnKey": colKey }; return row; } function reselectCheckboxesBasedOnTaskTableState() { - var taskSummaryHasSelected = false; - var executorSummaryHasSelected = false; - var allTaskSummaryChecked = true; - var allExecutorSummaryChecked = true; - var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice(); - if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) { - for (var k = 0; k < optionalColumns.length; k++) { - if (taskTableSelector.column(optionalColumns[k]).visible()) { - taskSummaryHasSelected = true; - $("#box-"+optionalColumns[k]).prop('checked', true); - taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]); - taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice(); - } else { - allTaskSummaryChecked = false; - } - } - createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray); + var taskSummaryHasSelected = false; + var executorSummaryHasSelected = false; + var allTaskSummaryChecked = true; + var allExecutorSummaryChecked = true; + var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice(); + var k; + if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) { + for (k = 0; k < optionalColumns.length; k++) { + if (taskTableSelector.column(optionalColumns[k]).visible()) { + taskSummaryHasSelected = true; + $("#box-"+optionalColumns[k]).prop('checked', true); + taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]); + taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice(); + } else { + allTaskSummaryChecked = false; + } } - - if (typeof executorSummaryTableSelector !== 'undefined') { - for (var k = 0; k < executorOptionalColumns.length; k++) { - if (executorSummaryTableSelector.column(executorOptionalColumns[k]).visible()) { - executorSummaryHasSelected = true; - $("#executor-box-"+executorOptionalColumns[k]).prop('checked', true); - } else { - allExecutorSummaryChecked = false; - } - } + createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray); + } + + if (typeof executorSummaryTableSelector !== 'undefined') { + for (k = 0; k < executorOptionalColumns.length; k++) { + if (executorSummaryTableSelector.column(executorOptionalColumns[k]).visible()) { + executorSummaryHasSelected = true; + $("#executor-box-"+executorOptionalColumns[k]).prop('checked', true); + } else { + allExecutorSummaryChecked = false; + } } + } - if ((taskSummaryHasSelected || executorSummaryHasSelected) && allTaskSummaryChecked && allExecutorSummaryChecked) { - $("#box-0").prop('checked', true); - } + if ((taskSummaryHasSelected || executorSummaryHasSelected) && allTaskSummaryChecked && allExecutorSummaryChecked) { + $("#box-0").prop('checked', true); + } } function getStageAttemptId() { @@ -301,7 +308,7 @@ function getStageAttemptId() { // We are using regex here to extract the stage attempt id as there might be certain url's with format // like /proxy/application_1539986433979_27115/stages/stage/?id=0&attempt=0#tasksTitle var stgAttemptId = words[1].split("&").filter( - word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex); + word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex); return stgAttemptId; } @@ -315,826 +322,833 @@ var executorOptionalColumns = [15, 16, 17, 18]; var executorSummaryTableSelector; $(document).ready(function () { - setDataTableDefaults(); - - $("#showAdditionalMetrics").append( - "" + - "
" + - "
Select All
" + - "
Scheduler Delay
" + - "
Task Deserialization Time
" + - "
Shuffle Read Blocked Time
" + - "
Shuffle Remote Reads
" + - "
Shuffle Write Time
" + - "
Result Serialization Time
" + - "
Getting Result Time
" + - "
Peak Execution Memory
" + - "
Peak JVM Memory OnHeap / OffHeap
" + - "
Peak Execution Memory OnHeap / OffHeap
" + - "
Peak Storage Memory OnHeap / OffHeap
" + - "
Peak Pool Memory Direct / Mapped
" + - "
"); - - $('#scheduler_delay').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " + - "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results."); - $('#task_deserialization_time').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task."); - $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines."); - $('#shuffle_remote_reads').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. "); - $('#shuffle_write_time').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Time that the task spent writing shuffle data."); - $('#result_serialization_time').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver."); - $('#getting_result_time').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task."); - $('#peak_execution_memory').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Execution memory refers to the memory used by internal data structures created during " + - "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " + - "should be approximately the sum of the peak sizes across all such data structures created " + - "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " + - "external sort."); - $('[data-toggle="tooltip"]').tooltip(); - var tasksSummary = $("#parent-container"); - getStandAloneAppId(function (appId) { - // rendering the UI page - $.get(createTemplateURI(appId, "stagespage"), function(template) { - tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html())); - - $("#additionalMetrics").click(function(){ - $("#arrowtoggle1").toggleClass("arrow-open arrow-closed"); - $("#toggle-metrics").toggleClass("d-none"); - if (window.localStorage) { - window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class')); - } + setDataTableDefaults(); + + $("#showAdditionalMetrics").append( + "" + + "
" + + "
Select All
" + + "
Scheduler Delay
" + + "
Task Deserialization Time
" + + "
Shuffle Read Blocked Time
" + + "
Shuffle Remote Reads
" + + "
Shuffle Write Time
" + + "
Result Serialization Time
" + + "
Getting Result Time
" + + "
Peak Execution Memory
" + + "
Peak JVM Memory OnHeap / OffHeap
" + + "
Peak Execution Memory OnHeap / OffHeap
" + + "
Peak Storage Memory OnHeap / OffHeap
" + + "
Peak Pool Memory Direct / Mapped
" + + "
"); + + $('#scheduler_delay').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " + + "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results."); + $('#task_deserialization_time').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task."); + $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines."); + $('#shuffle_remote_reads').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. "); + $('#shuffle_write_time').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Time that the task spent writing shuffle data."); + $('#result_serialization_time').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver."); + $('#getting_result_time').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task."); + $('#peak_execution_memory').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Execution memory refers to the memory used by internal data structures created during " + + "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " + + "should be approximately the sum of the peak sizes across all such data structures created " + + "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " + + "external sort."); + $('[data-toggle="tooltip"]').tooltip(); + var tasksSummary = $("#parent-container"); + getStandAloneAppId(function (appId) { + // rendering the UI page + $.get(createTemplateURI(appId, "stagespage"), function(template) { + tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html())); + + $("#additionalMetrics").click(function(){ + $("#arrowtoggle1").toggleClass("arrow-open arrow-closed"); + $("#toggle-metrics").toggleClass("d-none"); + if (window.localStorage) { + window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class')); + } + }); + + $("#aggregatedMetrics").click(function(){ + $("#arrowtoggle2").toggleClass("arrow-open arrow-closed"); + $("#toggle-aggregatedMetrics").toggleClass("d-none"); + if (window.localStorage) { + window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class')); + } + }); + + var endPoint = stageEndPoint(appId); + var stageAttemptId = getStageAttemptId(); + $.getJSON(endPoint + "/" + stageAttemptId, function(response, _ignored_status, _ignored_jqXHR) { + + var responseBody = response; + var dataToShow = {}; + dataToShow.showInputData = responseBody.inputBytes > 0; + dataToShow.showOutputData = responseBody.outputBytes > 0; + dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0; + dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0; + dataToShow.showBytesSpilledData = + (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0); + + var columnIndicesToRemove = []; + if (!dataToShow.showShuffleReadData) { + $('#shuffle_read_blocked_time').remove(); + $('#shuffle_remote_reads').remove(); + columnIndicesToRemove.push(2); + columnIndicesToRemove.push(3); + } + + if (!dataToShow.showShuffleWriteData) { + $('#shuffle_write_time').remove(); + columnIndicesToRemove.push(7); + } + + if (columnIndicesToRemove.length > 0) { + columnIndicesToRemove.sort(function(a, b) { return b - a; }); + columnIndicesToRemove.forEach(function(idx) { + optionalColumns.splice(idx, 1); }); + } + + // prepare data for executor summary table + var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary); + $.getJSON(createRESTEndPointForExecutorsPage(appId), + function(executorSummaryResponse, _ignored_status, _ignored_jqXHR) { + var executorDetailsMap = {}; + executorSummaryResponse.forEach(function (executorDetail) { + executorDetailsMap[executorDetail.id] = executorDetail; + }); + + var executorSummaryTable = []; + stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) { + var executorSummary = responseBody.executorSummary[columnKeyIndex]; + var executorDetail = executorDetailsMap[columnKeyIndex.toString()]; + executorSummary.id = columnKeyIndex; + executorSummary.executorLogs = {}; + executorSummary.hostPort = "CANNOT FIND ADDRESS"; + + if (executorDetail) { + if (executorDetail["executorLogs"]) { + responseBody.executorSummary[columnKeyIndex].executorLogs = + executorDetail["executorLogs"]; + } + if (executorDetail["hostPort"]) { + responseBody.executorSummary[columnKeyIndex].hostPort = + executorDetail["hostPort"]; + } + } + executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]); + }); + // building task aggregated metrics by executor table + var executorSummaryConf = { + "data": executorSummaryTable, + "columns": [ + {data : "id"}, + {data : "executorLogs", render: formatLogsCells}, + {data : "hostPort"}, + { + data : function (row, type) { + return type === 'display' ? formatDuration(row.taskTime) : row.taskTime; + } + }, + { + data : function (row, type) { + var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks; + return type === 'display' ? totaltasks : totaltasks.toString(); + } + }, + {data : "failedTasks"}, + {data : "killedTasks"}, + {data : "succeededTasks"}, + {data : "isExcludedForStage"}, + { + data : function (row, type) { + return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : ""; + } + }, + { + data : function (row, type) { + return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : ""; + } + }, + { + data : function (row, type) { + return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : ""; + } + }, + { + data : function (row, type) { + return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : ""; + } + }, + { + data : function (row, type) { + return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : ""; + } + }, + { + data : function (row, type) { + return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : ""; + } + }, + { + data : function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics; + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.JVMHeapMemory; + else + return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type)); + } else { + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } + } - $("#aggregatedMetrics").click(function(){ - $("#arrowtoggle2").toggleClass("arrow-open arrow-closed"); - $("#toggle-aggregatedMetrics").toggleClass("d-none"); - if (window.localStorage) { - window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class')); + } + }, + { + data : function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.OnHeapExecutionMemory; + else + return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type)); + } else { + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } + } + } + }, + { + data : function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.OnHeapStorageMemory; + else + return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type)); + } else { + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } + } + } + }, + { + data : function (row, type) { + var peakMemoryMetrics = row.peakMemoryMetrics + if (typeof peakMemoryMetrics !== 'undefined') { + if (type !== 'display') + return peakMemoryMetrics.DirectPoolMemory; + else + return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' + + formatBytes(peakMemoryMetrics.MappedPoolMemory, type)); + } else { + if (type !== 'display') { + return 0; + } else { + return '0.0 B / 0.0 B'; + } + } + } + } + ], + "columnDefs": [ + // SPARK-35087 [type:size] means String with structures like : 'size / records', + // they should be sorted as numerical-order instead of lexicographical-order by default. + // The targets: $id represents column id which comes from stagespage-template.html + // #summary-executor-table.If the relative position of the columns in the table + // #summary-executor-table has changed,please be careful to adjust the column index here + // Input Size / Records + {"type": "size", "targets": 9}, + // Output Size / Records + {"type": "size", "targets": 10}, + // Shuffle Read Size / Records + {"type": "size", "targets": 11}, + // Shuffle Write Size / Records + {"type": "size", "targets": 12}, + // Peak JVM Memory OnHeap / OffHeap + {"visible": false, "targets": 15}, + // Peak Execution Memory OnHeap / OffHeap + {"visible": false, "targets": 16}, + // Peak Storage Memory OnHeap / OffHeap + {"visible": false, "targets": 17}, + // Peak Pool Memory Direct / Mapped + {"visible": false, "targets": 18} + ], + "deferRender": true, + "order": [[0, "asc"]], + "bAutoWidth": false, + "oLanguage": { + "sEmptyTable": "No data to show yet" } + }; + executorSummaryTableSelector = + $("#summary-executor-table").DataTable(executorSummaryConf); + $('#parent-container [data-toggle="tooltip"]').tooltip(); + + executorSummaryTableSelector.column(9).visible(dataToShow.showInputData); + if (dataToShow.showInputData) { + $('#executor-summary-input').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Bytes and records read from Hadoop or from Spark storage."); + $('#executor-summary-input').tooltip(true); + } + executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData); + if (dataToShow.showOutputData) { + $('#executor-summary-output').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Bytes and records written to Hadoop."); + $('#executor-summary-output').tooltip(true); + } + executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData); + if (dataToShow.showShuffleReadData) { + $('#executor-summary-shuffle-read').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors)."); + $('#executor-summary-shuffle-read').tooltip(true); + } + executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData); + if (dataToShow.showShuffleWriteData) { + $('#executor-summary-shuffle-write').attr("data-toggle", "tooltip") + .attr("data-placement", "top") + .attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage."); + $('#executor-summary-shuffle-write').tooltip(true); + } + executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData); + executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData); }); - var endPoint = stageEndPoint(appId); - var stageAttemptId = getStageAttemptId(); - $.getJSON(endPoint + "/" + stageAttemptId, function(response, status, jqXHR) { - - var responseBody = response; - var dataToShow = {}; - dataToShow.showInputData = responseBody.inputBytes > 0; - dataToShow.showOutputData = responseBody.outputBytes > 0; - dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0; - dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0; - dataToShow.showBytesSpilledData = - (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0); - - var columnIndicesToRemove = []; - if (!dataToShow.showShuffleReadData) { - $('#shuffle_read_blocked_time').remove(); - $('#shuffle_remote_reads').remove(); - columnIndicesToRemove.push(2); - columnIndicesToRemove.push(3); - } + // prepare data for accumulatorUpdates + var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate => + !(accumUpdate.name).toString().includes("internal.")); + + var quantiles = "0,0.25,0.5,0.75,1.0"; + $.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles, + function(taskMetricsResponse, _ignored_status, _ignored_jqXHR) { + var taskMetricKeys = Object.keys(taskMetricsResponse); + taskMetricKeys.forEach(function (columnKey) { + var row; + var row1; + var row2; + var row3; + switch(columnKey) { + case "shuffleReadMetrics": + row1 = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 3); + row2 = createRowMetadataForColumn( + "shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13); + row3 = createRowMetadataForColumn( + "shuffleRemoteReads", taskMetricsResponse[columnKey], 14); + if (dataToShow.showShuffleReadData) { + taskSummaryMetricsTableArray.push(row1); + taskSummaryMetricsTableArray.push(row2); + taskSummaryMetricsTableArray.push(row3); + } + break; + + case "schedulerDelay": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 11); + taskSummaryMetricsTableArray.push(row); + break; + + case "executorDeserializeTime": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 12); + taskSummaryMetricsTableArray.push(row); + break; + + case "resultSerializationTime": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 15); + taskSummaryMetricsTableArray.push(row); + break; + + case "gettingResultTime": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 16); + taskSummaryMetricsTableArray.push(row); + break; + + case "peakExecutionMemory": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 17); + taskSummaryMetricsTableArray.push(row); + break; + + case "inputMetrics": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 1); + if (dataToShow.showInputData) { + taskSummaryMetricsTableArray.push(row); + } + break; - if (!dataToShow.showShuffleWriteData) { - $('#shuffle_write_time').remove(); - columnIndicesToRemove.push(7); - } + case "outputMetrics": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 2); + if (dataToShow.showOutputData) { + taskSummaryMetricsTableArray.push(row); + } + break; + + case "shuffleWriteMetrics": + row1 = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 4); + row2 = createRowMetadataForColumn( + "shuffleWriteTime", taskMetricsResponse[columnKey], 21); + if (dataToShow.showShuffleWriteData) { + taskSummaryMetricsTableArray.push(row1); + taskSummaryMetricsTableArray.push(row2); + } + break; - if (columnIndicesToRemove.length > 0) { - columnIndicesToRemove.sort(function(a, b) { return b - a; }); - columnIndicesToRemove.forEach(function(idx) { - optionalColumns.splice(idx, 1); - }); - } + case "diskBytesSpilled": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 5); + if (dataToShow.showBytesSpilledData) { + taskSummaryMetricsTableArray.push(row); + } + break; - // prepare data for executor summary table - var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary); - $.getJSON(createRESTEndPointForExecutorsPage(appId), - function(executorSummaryResponse, status, jqXHR) { - var executorDetailsMap = {}; - executorSummaryResponse.forEach(function (executorDetail) { - executorDetailsMap[executorDetail.id] = executorDetail; - }); - - var executorSummaryTable = []; - stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) { - var executorSummary = responseBody.executorSummary[columnKeyIndex]; - var executorDetail = executorDetailsMap[columnKeyIndex.toString()]; - executorSummary.id = columnKeyIndex; - executorSummary.executorLogs = {}; - executorSummary.hostPort = "CANNOT FIND ADDRESS"; - - if (executorDetail) { - if (executorDetail["executorLogs"]) { - responseBody.executorSummary[columnKeyIndex].executorLogs = - executorDetail["executorLogs"]; - } - if (executorDetail["hostPort"]) { - responseBody.executorSummary[columnKeyIndex].hostPort = - executorDetail["hostPort"]; - } - } - executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]); - }); - // building task aggregated metrics by executor table - var executorSummaryConf = { - "data": executorSummaryTable, - "columns": [ - {data : "id"}, - {data : "executorLogs", render: formatLogsCells}, - {data : "hostPort"}, - { - data : function (row, type) { - return type === 'display' ? formatDuration(row.taskTime) : row.taskTime; - } - }, - { - data : function (row, type) { - var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks; - return type === 'display' ? totaltasks : totaltasks.toString(); - } - }, - {data : "failedTasks"}, - {data : "killedTasks"}, - {data : "succeededTasks"}, - {data : "isExcludedForStage"}, - { - data : function (row, type) { - return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : ""; - } - }, - { - data : function (row, type) { - return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : ""; - } - }, - { - data : function (row, type) { - return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : ""; - } - }, - { - data : function (row, type) { - return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : ""; - } - }, - { - data : function (row, type) { - return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : ""; - } - }, - { - data : function (row, type) { - return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : ""; - } - }, - { - data : function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics; - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.JVMHeapMemory; - else - return (formatBytes(peakMemoryMetrics.JVMHeapMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.JVMOffHeapMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - - } - }, - { - data : function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.OnHeapExecutionMemory; - else - return (formatBytes(peakMemoryMetrics.OnHeapExecutionMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.OffHeapExecutionMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - } - }, - { - data : function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.OnHeapStorageMemory; - else - return (formatBytes(peakMemoryMetrics.OnHeapStorageMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.OffHeapStorageMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - } - }, - { - data : function (row, type) { - var peakMemoryMetrics = row.peakMemoryMetrics - if (typeof peakMemoryMetrics !== 'undefined') { - if (type !== 'display') - return peakMemoryMetrics.DirectPoolMemory; - else - return (formatBytes(peakMemoryMetrics.DirectPoolMemory, type) + ' / ' + - formatBytes(peakMemoryMetrics.MappedPoolMemory, type)); - } else { - if (type !== 'display') { - return 0; - } else { - return '0.0 B / 0.0 B'; - } - } - } - } - ], - "columnDefs": [ - // SPARK-35087 [type:size] means String with structures like : 'size / records', - // they should be sorted as numerical-order instead of lexicographical-order by default. - // The targets: $id represents column id which comes from stagespage-template.html - // #summary-executor-table.If the relative position of the columns in the table - // #summary-executor-table has changed,please be careful to adjust the column index here - // Input Size / Records - {"type": "size", "targets": 9}, - // Output Size / Records - {"type": "size", "targets": 10}, - // Shuffle Read Size / Records - {"type": "size", "targets": 11}, - // Shuffle Write Size / Records - {"type": "size", "targets": 12}, - // Peak JVM Memory OnHeap / OffHeap - {"visible": false, "targets": 15}, - // Peak Execution Memory OnHeap / OffHeap - {"visible": false, "targets": 16}, - // Peak Storage Memory OnHeap / OffHeap - {"visible": false, "targets": 17}, - // Peak Pool Memory Direct / Mapped - {"visible": false, "targets": 18} - ], - "deferRender": true, - "order": [[0, "asc"]], - "bAutoWidth": false, - "oLanguage": { - "sEmptyTable": "No data to show yet" - } - }; - executorSummaryTableSelector = - $("#summary-executor-table").DataTable(executorSummaryConf); - $('#parent-container [data-toggle="tooltip"]').tooltip(); - - executorSummaryTableSelector.column(9).visible(dataToShow.showInputData); - if (dataToShow.showInputData) { - $('#executor-summary-input').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Bytes and records read from Hadoop or from Spark storage."); - $('#executor-summary-input').tooltip(true); + case "memoryBytesSpilled": + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 6); + if (dataToShow.showBytesSpilledData) { + taskSummaryMetricsTableArray.push(row); + } + break; + + default: + if (getColumnNameForTaskMetricSummary(columnKey) != "NA") { + row = createRowMetadataForColumn( + columnKey, taskMetricsResponse[columnKey], 0); + taskSummaryMetricsTableArray.push(row); + } + break; + } + }); + var taskSummaryMetricsTableFilteredArray = + taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11); + taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice(); + reselectCheckboxesBasedOnTaskTableState(); + }); + + // building accumulator update table + var accumulatorConf = { + "data": accumulatorTable, + "columns": [ + {data : "id"}, + {data : "name"}, + {data : "value"} + ], + "paging": false, + "searching": false, + "order": [[0, "asc"]], + "bAutoWidth": false + }; + $("#accumulator-table").DataTable(accumulatorConf); + + // building tasks table that uses server side functionality + var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks + + responseBody.numKilledTasks + responseBody.numFailedTasks; + var taskTable = "#active-tasks-table"; + var taskConf = { + "serverSide": true, + "paging": true, + "info": true, + "processing": true, + "lengthMenu": [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]], + "orderMulti": false, + "bAutoWidth": false, + "ajax": { + "url": endPoint + "/" + stageAttemptId + "/taskTable", + "data": function (data) { + var columnIndexToSort = 0; + var columnNameToSort = "Index"; + if (data.order[0].column && data.order[0].column != "") { + columnIndexToSort = parseInt(data.order[0].column); + columnNameToSort = data.columns[columnIndexToSort].name; + } + delete data.columns; + data.numTasks = totalTasksToShow; + data.columnIndexToSort = columnIndexToSort; + data.columnNameToSort = columnNameToSort; + if (data.length === -1) { + data.length = totalTasksToShow; + } + }, + "dataSrc": function (jsons) { + var jsonStr = JSON.stringify(jsons); + var tasksToShow = JSON.parse(jsonStr); + return tasksToShow.aaData; + }, + "error": function (_ignored_jqXHR, _ignored_textStatus, _ignored_errorThrown) { + alert("Unable to connect to the server. Looks like the Spark " + + "application must have ended. Please Switch to the history UI."); + $("#active-tasks-table_processing").css("display","none"); + } + }, + "columns": [ + { + data: function (row, type) { + return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index; + }, + name: "Index" + }, + {data : "taskId", name: "ID"}, + {data : "attempt", name: "Attempt"}, + {data : "status", name: "Status"}, + {data : "taskLocality", name: "Locality Level"}, + {data : "executorId", name: "Executor ID"}, + {data : "host", name: "Host"}, + {data : "executorLogs", name: "Logs", render: formatLogsCells}, + {data : "launchTime", name: "Launch Time", render: formatDate}, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.executorRunTime) { + return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime; + } else { + return ""; } - executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData); - if (dataToShow.showOutputData) { - $('#executor-summary-output').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Bytes and records written to Hadoop."); - $('#executor-summary-output').tooltip(true); + }, + name: "Duration" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.jvmGcTime) { + return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime; + } else { + return ""; } - executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData); - if (dataToShow.showShuffleReadData) { - $('#executor-summary-shuffle-read').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors)."); - $('#executor-summary-shuffle-read').tooltip(true); + }, + name: "GC Time" + }, + { + data : function (row, type) { + if (row.schedulerDelay) { + return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay; + } else { + return ""; } - executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData); - if (dataToShow.showShuffleWriteData) { - $('#executor-summary-shuffle-write').attr("data-toggle", "tooltip") - .attr("data-placement", "top") - .attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage."); - $('#executor-summary-shuffle-write').tooltip(true); + }, + name: "Scheduler Delay" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) { + return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime; + } else { + return ""; } - executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData); - executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData); - }); - - // prepare data for accumulatorUpdates - var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate => - !(accumUpdate.name).toString().includes("internal.")); - - var quantiles = "0,0.25,0.5,0.75,1.0"; - $.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles, - function(taskMetricsResponse, status, jqXHR) { - var taskMetricKeys = Object.keys(taskMetricsResponse); - taskMetricKeys.forEach(function (columnKey) { - switch(columnKey) { - case "shuffleReadMetrics": - var row1 = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 3); - var row2 = createRowMetadataForColumn( - "shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13); - var row3 = createRowMetadataForColumn( - "shuffleRemoteReads", taskMetricsResponse[columnKey], 14); - if (dataToShow.showShuffleReadData) { - taskSummaryMetricsTableArray.push(row1); - taskSummaryMetricsTableArray.push(row2); - taskSummaryMetricsTableArray.push(row3); - } - break; - - case "schedulerDelay": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 11); - taskSummaryMetricsTableArray.push(row); - break; - - case "executorDeserializeTime": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 12); - taskSummaryMetricsTableArray.push(row); - break; - - case "resultSerializationTime": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 15); - taskSummaryMetricsTableArray.push(row); - break; - - case "gettingResultTime": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 16); - taskSummaryMetricsTableArray.push(row); - break; - - case "peakExecutionMemory": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 17); - taskSummaryMetricsTableArray.push(row); - break; - - case "inputMetrics": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 1); - if (dataToShow.showInputData) { - taskSummaryMetricsTableArray.push(row); - } - break; - - case "outputMetrics": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 2); - if (dataToShow.showOutputData) { - taskSummaryMetricsTableArray.push(row); - } - break; - - case "shuffleWriteMetrics": - var row1 = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 4); - var row2 = createRowMetadataForColumn( - "shuffleWriteTime", taskMetricsResponse[columnKey], 21); - if (dataToShow.showShuffleWriteData) { - taskSummaryMetricsTableArray.push(row1); - taskSummaryMetricsTableArray.push(row2); - } - break; - - case "diskBytesSpilled": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 5); - if (dataToShow.showBytesSpilledData) { - taskSummaryMetricsTableArray.push(row); - } - break; - - case "memoryBytesSpilled": - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 6); - if (dataToShow.showBytesSpilledData) { - taskSummaryMetricsTableArray.push(row); - } - break; - - default: - if (getColumnNameForTaskMetricSummary(columnKey) != "NA") { - var row = createRowMetadataForColumn( - columnKey, taskMetricsResponse[columnKey], 0); - taskSummaryMetricsTableArray.push(row); - } - break; - } - }); - var taskSummaryMetricsTableFilteredArray = - taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11); - taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice(); - reselectCheckboxesBasedOnTaskTableState(); - }); - - // building accumulator update table - var accumulatorConf = { - "data": accumulatorTable, - "columns": [ - {data : "id"}, - {data : "name"}, - {data : "value"} - ], - "paging": false, - "searching": false, - "order": [[0, "asc"]], - "bAutoWidth": false - }; - $("#accumulator-table").DataTable(accumulatorConf); - - // building tasks table that uses server side functionality - var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks + - responseBody.numKilledTasks + responseBody.numFailedTasks; - var taskTable = "#active-tasks-table"; - var taskConf = { - "serverSide": true, - "paging": true, - "info": true, - "processing": true, - "lengthMenu": [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]], - "orderMulti": false, - "bAutoWidth": false, - "ajax": { - "url": endPoint + "/" + stageAttemptId + "/taskTable", - "data": function (data) { - var columnIndexToSort = 0; - var columnNameToSort = "Index"; - if (data.order[0].column && data.order[0].column != "") { - columnIndexToSort = parseInt(data.order[0].column); - columnNameToSort = data.columns[columnIndexToSort].name; - } - delete data.columns; - data.numTasks = totalTasksToShow; - data.columnIndexToSort = columnIndexToSort; - data.columnNameToSort = columnNameToSort; - if (data.length === -1) { - data.length = totalTasksToShow; - } - }, - "dataSrc": function (jsons) { - var jsonStr = JSON.stringify(jsons); - var tasksToShow = JSON.parse(jsonStr); - return tasksToShow.aaData; - }, - "error": function (jqXHR, textStatus, errorThrown) { - alert("Unable to connect to the server. Looks like the Spark " + - "application must have ended. Please Switch to the history UI."); - $("#active-tasks-table_processing").css("display","none"); - } - }, - "columns": [ - {data: function (row, type) { - return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index; - }, - name: "Index" - }, - {data : "taskId", name: "ID"}, - {data : "attempt", name: "Attempt"}, - {data : "status", name: "Status"}, - {data : "taskLocality", name: "Locality Level"}, - {data : "executorId", name: "Executor ID"}, - {data : "host", name: "Host"}, - {data : "executorLogs", name: "Logs", render: formatLogsCells}, - {data : "launchTime", name: "Launch Time", render: formatDate}, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.executorRunTime) { - return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime; - } else { - return ""; - } - }, - name: "Duration" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.jvmGcTime) { - return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime; - } else { - return ""; - } - }, - name: "GC Time" - }, - { - data : function (row, type) { - if (row.schedulerDelay) { - return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay; - } else { - return ""; - } - }, - name: "Scheduler Delay" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) { - return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime; - } else { - return ""; - } - }, - name: "Task Deserialization Time" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) { - return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime; - } else { - return ""; - } - }, - name: "Shuffle Read Blocked Time" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) { - return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead; - } else { - return ""; - } - }, - name: "Shuffle Remote Reads" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.resultSerializationTime) { - return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime; - } else { - return ""; - } - }, - name: "Result Serialization Time" - }, - { - data : function (row, type) { - if (row.gettingResultTime) { - return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime; - } else { - return ""; - } - }, - name: "Getting Result Time" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) { - return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory; - } else { - return ""; - } - }, - name: "Peak Execution Memory" - }, - { - data : function (row, type) { - if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) { - var allAccums = ""; - row.accumulatorUpdates.forEach(function(accumulator) { - allAccums += accumulator.name + ': ' + accumulator.update + "
"; - }); - return allAccums; - } else { - return ""; - } - }, - name: "Accumulators" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) { - if (type === 'display') { - return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead; - } else { - return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead; - } - } else { - return ""; - } - }, - name: "Input Size / Records" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) { - if (type === 'display') { - return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten; - } else { - return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten; - } - } else { - return ""; - } - }, - name: "Output Size / Records" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) { - return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000.0) : row.taskMetrics.shuffleWriteMetrics.writeTime; - } else { - return ""; - } - }, - name: "Shuffle Write Time" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) { - if (type === 'display') { - return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten; - } else { - return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten; - } - } else { - return ""; - } - }, - name: "Shuffle Write Size / Records" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics && - (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0 || row.taskMetrics.shuffleReadMetrics.remoteBytesRead > 0)) { - var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead); - if (type === 'display') { - return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead; - } else { - return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead; - } - } else { - return ""; - } - }, - name: "Shuffle Read Size / Records" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) { - return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled; - } else { - return ""; - } - }, - name: "Spill (Memory)" - }, - { - data : function (row, type) { - if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) { - return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled; - } else { - return ""; - } - }, - name: "Spill (Disk)" - }, - { - data : function (row, type) { - var msg = row.errorMessage; - if (typeof msg === 'undefined') { - return ""; - } else { - var indexOfLineSeparator = msg.indexOf("\n"); - var formHead = indexOfLineSeparator > 0 ? msg.substring(0, indexOfLineSeparator) : (msg.length > 100 ? msg.substring(0, 100) : msg); - var form = "+details"; - var formMsg = "
" + row.errorMessage + "
"; - return formHead + form + formMsg; - } - }, - name: "Errors" - } - ], - "columnDefs": [ - { "visible": false, "targets": 11 }, - { "visible": false, "targets": 12 }, - { "visible": false, "targets": 13 }, - { "visible": false, "targets": 14 }, - { "visible": false, "targets": 15 }, - { "visible": false, "targets": 16 }, - { "visible": false, "targets": 17 }, - { "visible": false, "targets": 18 }, - { "visible": false, "targets": 21 } - ], - "deferRender": true - }; - taskTableSelector = $(taskTable).DataTable(taskConf); - $('#active-tasks-table_filter input').unbind(); - var searchEvent; - $('#active-tasks-table_filter input').bind('keyup', function(e) { - if (typeof searchEvent !== 'undefined') { - window.clearTimeout(searchEvent); + }, + name: "Task Deserialization Time" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) { + return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime; + } else { + return ""; + } + }, + name: "Shuffle Read Blocked Time" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) { + return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead; + } else { + return ""; + } + }, + name: "Shuffle Remote Reads" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.resultSerializationTime) { + return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime; + } else { + return ""; + } + }, + name: "Result Serialization Time" + }, + { + data : function (row, type) { + if (row.gettingResultTime) { + return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime; + } else { + return ""; + } + }, + name: "Getting Result Time" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) { + return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory; + } else { + return ""; + } + }, + name: "Peak Execution Memory" + }, + { + data : function (row, _ignored_type) { + if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) { + var allAccums = ""; + row.accumulatorUpdates.forEach(function(accumulator) { + allAccums += accumulator.name + ': ' + accumulator.update + "
"; + }); + return allAccums; + } else { + return ""; + } + }, + name: "Accumulators" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) { + if (type === 'display') { + return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead; + } else { + return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead; } - var value = this.value; - searchEvent = window.setTimeout(function(){ - taskTableSelector.search( value ).draw();}, 500); - }); - reselectCheckboxesBasedOnTaskTableState(); - - // hide or show columns dynamically event - $('input.toggle-vis').on('click', function(e){ - // Get the column - var para = $(this).attr('data-column'); - if (para == "0") { - var allColumns = taskTableSelector.columns(optionalColumns); - var executorAllColumns = executorSummaryTableSelector.columns(executorOptionalColumns); - if ($(this).is(":checked")) { - $(".toggle-vis").prop('checked', true); - allColumns.visible(true); - executorAllColumns.visible(true); - createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray); - } else { - $(".toggle-vis").prop('checked', false); - allColumns.visible(false); - executorAllColumns.visible(false); - var taskSummaryMetricsTableFilteredArray = - taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11); - createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray); - } - } else { - var dataMetricsType = $(this).attr("data-metrics-type"); - if (dataMetricsType === 'task') { - var column = taskTableSelector.column(para); - // Toggle the visibility - column.visible(!column.visible()); - var taskSummaryMetricsTableFilteredArray = []; - if ($(this).is(":checked")) { - taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]); - taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice(); - } else { - taskSummaryMetricsTableFilteredArray = - taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para); - } - createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray); - } - if (dataMetricsType === "executor") { - var column = executorSummaryTableSelector.column(para); - column.visible(!column.visible()); - } - } - }); - - // title number and toggle list - $("#summaryMetricsTitle").html("Summary Metrics for " + "" + responseBody.numCompleteTasks + " Completed Tasks" + ""); - $("#tasksTitle").html("Tasks (" + totalTasksToShow + ")"); - - // hide or show the accumulate update table - if (accumulatorTable.length == 0) { - $("#accumulator-update-table").hide(); } else { - taskTableSelector.column(18).visible(true); - $("#accumulator-update-table").show(); + return ""; } - // Showing relevant stage data depending on stage type for task table and executor - // summary table - taskTableSelector.column(19).visible(dataToShow.showInputData); - taskTableSelector.column(20).visible(dataToShow.showOutputData); - taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData); - taskTableSelector.column(23).visible(dataToShow.showShuffleReadData); - taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData); - taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData); - - if (window.localStorage) { - if (window.localStorage.getItem("arrowtoggle1class") !== null && - window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) { - $("#arrowtoggle1").toggleClass("arrow-open arrow-closed"); - $("#toggle-metrics").toggleClass("d-none"); - } - if (window.localStorage.getItem("arrowtoggle2class") !== null && - window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) { - $("#arrowtoggle2").toggleClass("arrow-open arrow-closed"); - $("#toggle-aggregatedMetrics").toggleClass("d-none"); - } + }, + name: "Input Size / Records" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) { + if (type === 'display') { + return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten; + } else { + return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten; + } + } else { + return ""; } - }); + }, + name: "Output Size / Records" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) { + return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000.0) : row.taskMetrics.shuffleWriteMetrics.writeTime; + } else { + return ""; + } + }, + name: "Shuffle Write Time" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) { + if (type === 'display') { + return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten; + } else { + return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten; + } + } else { + return ""; + } + }, + name: "Shuffle Write Size / Records" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics && + (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0 || row.taskMetrics.shuffleReadMetrics.remoteBytesRead > 0)) { + var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead); + if (type === 'display') { + return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead; + } else { + return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead; + } + } else { + return ""; + } + }, + name: "Shuffle Read Size / Records" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) { + return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled; + } else { + return ""; + } + }, + name: "Spill (Memory)" + }, + { + data : function (row, type) { + if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) { + return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled; + } else { + return ""; + } + }, + name: "Spill (Disk)" + }, + { + data : function (row, _ignored_type) { + var msg = row.errorMessage; + if (typeof msg === 'undefined') { + return ""; + } else { + var indexOfLineSeparator = msg.indexOf("\n"); + var formHead = indexOfLineSeparator > 0 ? msg.substring(0, indexOfLineSeparator) : (msg.length > 100 ? msg.substring(0, 100) : msg); + var form = "+details"; + var formMsg = "
" + row.errorMessage + "
"; + return formHead + form + formMsg; + } + }, + name: "Errors" + } + ], + "columnDefs": [ + { "visible": false, "targets": 11 }, + { "visible": false, "targets": 12 }, + { "visible": false, "targets": 13 }, + { "visible": false, "targets": 14 }, + { "visible": false, "targets": 15 }, + { "visible": false, "targets": 16 }, + { "visible": false, "targets": 17 }, + { "visible": false, "targets": 18 }, + { "visible": false, "targets": 21 } + ], + "deferRender": true + }; + taskTableSelector = $(taskTable).DataTable(taskConf); + $('#active-tasks-table_filter input').unbind(); + var searchEvent; + $('#active-tasks-table_filter input').bind('keyup', function(_ignored_e) { + if (typeof searchEvent !== 'undefined') { + window.clearTimeout(searchEvent); + } + var value = this.value; + searchEvent = window.setTimeout(function(){ + taskTableSelector.search( value ).draw();}, 500); }); + reselectCheckboxesBasedOnTaskTableState(); + + // hide or show columns dynamically event + $('input.toggle-vis').on('click', function(_ignored_e){ + var taskSummaryMetricsTableFilteredArray; + // Get the column + var para = $(this).attr('data-column'); + if (para == "0") { + var allColumns = taskTableSelector.columns(optionalColumns); + var executorAllColumns = executorSummaryTableSelector.columns(executorOptionalColumns); + if ($(this).is(":checked")) { + $(".toggle-vis").prop('checked', true); + allColumns.visible(true); + executorAllColumns.visible(true); + createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray); + } else { + $(".toggle-vis").prop('checked', false); + allColumns.visible(false); + executorAllColumns.visible(false); + taskSummaryMetricsTableFilteredArray = + taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11); + createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray); + } + } else { + var dataMetricsType = $(this).attr("data-metrics-type"); + var column; + if (dataMetricsType === 'task') { + column = taskTableSelector.column(para); + // Toggle the visibility + column.visible(!column.visible()); + taskSummaryMetricsTableFilteredArray = []; + if ($(this).is(":checked")) { + taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]); + taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice(); + } else { + taskSummaryMetricsTableFilteredArray = + taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para); + } + createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray); + } + if (dataMetricsType === "executor") { + column = executorSummaryTableSelector.column(para); + column.visible(!column.visible()); + } + } + }); + + // title number and toggle list + $("#summaryMetricsTitle").html("Summary Metrics for " + "" + responseBody.numCompleteTasks + " Completed Tasks" + ""); + $("#tasksTitle").html("Tasks (" + totalTasksToShow + ")"); + + // hide or show the accumulate update table + if (accumulatorTable.length == 0) { + $("#accumulator-update-table").hide(); + } else { + taskTableSelector.column(18).visible(true); + $("#accumulator-update-table").show(); + } + // Showing relevant stage data depending on stage type for task table and executor + // summary table + taskTableSelector.column(19).visible(dataToShow.showInputData); + taskTableSelector.column(20).visible(dataToShow.showOutputData); + taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData); + taskTableSelector.column(23).visible(dataToShow.showShuffleReadData); + taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData); + taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData); + + if (window.localStorage) { + if (window.localStorage.getItem("arrowtoggle1class") !== null && + window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) { + $("#arrowtoggle1").toggleClass("arrow-open arrow-closed"); + $("#toggle-metrics").toggleClass("d-none"); + } + if (window.localStorage.getItem("arrowtoggle2class") !== null && + window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) { + $("#arrowtoggle2").toggleClass("arrow-open arrow-closed"); + $("#toggle-aggregatedMetrics").toggleClass("d-none"); + } + } + }); }); + }); }); diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js index 6d4c8d94b4288..9f366025a8b55 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js @@ -15,6 +15,7 @@ * limitations under the License. */ +/* global $, d3, timeFormat, timeTipStrings */ // timeFormat: StreamingPage.scala will generate a global "timeFormat" dictionary to store the time // and its formatted string. Because we cannot specify a timezone in JavaScript, to make sure the @@ -37,87 +38,90 @@ var onClickTimeline = function() {}; // Show a tooltip "text" for "node" function showBootstrapTooltip(node, text) { - $(node).tooltip({title: text, trigger: "manual", container: "body"}); - $(node).tooltip("show"); + $(node).tooltip({title: text, trigger: "manual", container: "body"}); + $(node).tooltip("show"); } // Hide the tooltip for "node" function hideBootstrapTooltip(node) { - $(node).tooltip("dispose"); + $(node).tooltip("dispose"); } +/* eslint-disable no-unused-vars */ // Return the function to scroll to the corresponding // row on clicking a point of batch in the timeline. function getOnClickTimelineFunction() { - // If the user click one point in the graphs, jump to the batch row and highlight it. And - // recovery the batch row after 3 seconds if necessary. - // We need to remember the last clicked batch so that we can recovery it. - var lastClickedBatch = null; - var lastTimeout = null; - - return function(d) { - var batchSelector = $("#batch-" + d.x); - // If there is a corresponding batch row, scroll down to it and highlight it. - if (batchSelector.length > 0) { - if (lastTimeout != null) { - window.clearTimeout(lastTimeout); - } - if (lastClickedBatch != null) { - clearBatchRow(lastClickedBatch); - lastClickedBatch = null; - } - lastClickedBatch = d.x; - highlightBatchRow(lastClickedBatch); - lastTimeout = window.setTimeout(function () { - lastTimeout = null; - if (lastClickedBatch != null) { - clearBatchRow(lastClickedBatch); - lastClickedBatch = null; - } - }, 3000); // Clean up after 3 seconds - - var topOffset = batchSelector.offset().top - 15; - if (topOffset < 0) { - topOffset = 0; - } - $('html,body').animate({scrollTop: topOffset}, 200); + // If the user click one point in the graphs, jump to the batch row and highlight it. And + // recovery the batch row after 3 seconds if necessary. + // We need to remember the last clicked batch so that we can recovery it. + var lastClickedBatch = null; + var lastTimeout = null; + + return function(d) { + var batchSelector = $("#batch-" + d.x); + // If there is a corresponding batch row, scroll down to it and highlight it. + if (batchSelector.length > 0) { + if (lastTimeout != null) { + window.clearTimeout(lastTimeout); + } + if (lastClickedBatch != null) { + clearBatchRow(lastClickedBatch); + lastClickedBatch = null; + } + lastClickedBatch = d.x; + highlightBatchRow(lastClickedBatch); + lastTimeout = window.setTimeout(function () { + lastTimeout = null; + if (lastClickedBatch != null) { + clearBatchRow(lastClickedBatch); + lastClickedBatch = null; } + }, 3000); // Clean up after 3 seconds + + var topOffset = batchSelector.offset().top - 15; + if (topOffset < 0) { + topOffset = 0; + } + $('html,body').animate({scrollTop: topOffset}, 200); } + } } // Register a timeline graph. All timeline graphs should be register before calling any // "drawTimeline" so that we can determine the max margin left for all timeline graphs. function registerTimeline(minY, maxY) { - var numOfChars = yValueFormat(maxY).length; - // A least width for "maxY" in the graph - var pxForMaxY = numOfChars * 8 + 10; - // Make sure we have enough space to show the ticks in the y axis of timeline - maxMarginLeftForTimeline = pxForMaxY > maxMarginLeftForTimeline? pxForMaxY : maxMarginLeftForTimeline; + var numOfChars = yValueFormat(maxY).length; + // A least width for "maxY" in the graph + var pxForMaxY = numOfChars * 8 + 10; + // Make sure we have enough space to show the ticks in the y axis of timeline + maxMarginLeftForTimeline = pxForMaxY > maxMarginLeftForTimeline? pxForMaxY : maxMarginLeftForTimeline; } // Register a histogram graph. All histogram graphs should be register before calling any // "drawHistogram" so that we can determine the max X value for histograms. function registerHistogram(values, minY, maxY) { - var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values); - // d.x is the y values while d.y is the x values - var maxX = d3.max(data, function(d) { return d.y; }); - maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram; + var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values); + // d.x is the y values while d.y is the x values + var maxX = d3.max(data, function(d) { return d.y; }); + maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram; } +/* eslint-enable no-unused-vars */ // Draw a line between (x1, y1) and (x2, y2) function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) { - var line = d3.svg.line() - .x(function(d) { return xFunc(d.x); }) - .y(function(d) { return yFunc(d.y); }); - var data = [{x: x1, y: y1}, {x: x2, y: y2}]; - svg.append("path") - .datum(data) - .style("stroke-dasharray", ("6, 6")) - .style("stroke", "lightblue") - .attr("class", "line") - .attr("d", line); + var line = d3.svg.line() + .x(function(d) { return xFunc(d.x); }) + .y(function(d) { return yFunc(d.y); }); + var data = [{x: x1, y: y1}, {x: x2, y: y2}]; + svg.append("path") + .datum(data) + .style("stroke-dasharray", ("6, 6")) + .style("stroke", "lightblue") + .attr("class", "line") + .attr("d", line); } +/* eslint-disable no-unused-vars */ /** * @param id the `id` used in the html `div` tag * @param data the data for the timeline graph @@ -129,108 +133,105 @@ function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) { * @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph */ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) { - // Hide the right border of "". We cannot use "css" directly, or "sorttable.js" will override them. - d3.select(d3.select(id).node().parentNode) - .style("padding", "8px 0 8px 8px") - .style("border-right", "0px solid white"); - - var margin = {top: 20, right: 27, bottom: 30, left: maxMarginLeftForTimeline}; - var width = 500 - margin.left - margin.right; - var height = 150 - margin.top - margin.bottom; - - var x = d3.scale.linear().domain([minX, maxX]).range([0, width]); - var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); - - var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) { - var formattedDate = timeFormat[d]; - var dotIndex = formattedDate.indexOf('.'); - if (dotIndex >= 0) { - // Remove milliseconds - return formattedDate.substring(0, dotIndex); - } else { - return formattedDate; - } - }); - var formatYValue = d3.format(",.2f"); - var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue); - - var line = d3.svg.line() - .x(function(d) { return x(d.x); }) - .y(function(d) { return y(d.y); }); - - var svg = d3.select(id).append("svg") - .attr("width", width + margin.left + margin.right) - .attr("height", height + margin.top + margin.bottom) - .append("g") - .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - - // Only show the first and last time in the graph - xAxis.tickValues(x.domain()); - - svg.append("g") - .attr("class", "x axis") - .attr("transform", "translate(0," + height + ")") - .call(xAxis); - - svg.append("g") - .attr("class", "y axis") - .call(yAxis) - .append("text") - .attr("transform", "translate(0," + unitLabelYOffset + ")") - .text(unitY); - - - if (batchInterval && batchInterval <= maxY) { - drawLine(svg, x, y, minX, batchInterval, maxX, batchInterval); - } - - svg.append("path") - .datum(data) - .attr("class", "line") - .attr("d", line); - - // If the user click one point in the graphs, jump to the batch row and highlight it. And - // recovery the batch row after 3 seconds if necessary. - // We need to remember the last clicked batch so that we can recovery it. - var lastClickedBatch = null; - var lastTimeout = null; - - function isFailedBatch(batchTime) { - return $("#batch-" + batchTime).attr("isFailed") == "true"; + // Hide the right border of "". We cannot use "css" directly, or "sorttable.js" will override them. + d3.select(d3.select(id).node().parentNode) + .style("padding", "8px 0 8px 8px") + .style("border-right", "0px solid white"); + var margin = {top: 20, right: 27, bottom: 30, left: maxMarginLeftForTimeline}; + var width = 500 - margin.left - margin.right; + var height = 150 - margin.top - margin.bottom; + + var x = d3.scale.linear().domain([minX, maxX]).range([0, width]); + var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); + + var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) { + var formattedDate = timeFormat[d]; + var dotIndex = formattedDate.indexOf('.'); + if (dotIndex >= 0) { + // Remove milliseconds + return formattedDate.substring(0, dotIndex); + } else { + return formattedDate; } - - // Add points to the line. However, we make it invisible at first. But when the user moves mouse - // over a point, it will be displayed with its detail. - svg.selectAll(".point") - .data(data) - .enter().append("circle") - .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) // white and opacity = 0 make it invisible - .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) - .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";}) - .style("cursor", "pointer") - .attr("cx", function(d) { return x(d.x); }) - .attr("cy", function(d) { return y(d.y); }) - .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";}) - .on('mouseover', function(d) { - var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x]; - showBootstrapTooltip(d3.select(this).node(), tip); - // show the point - d3.select(this) - .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";}) - .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";}) - .attr("opacity", "1") - .attr("r", "3"); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - // hide the point - d3.select(this) - .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) - .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) - .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";}) - .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";}); - }) - .on("click", onClickTimeline); + }); + var formatYValue = d3.format(",.2f"); + var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue); + + var line = d3.svg.line() + .x(function(d) { return x(d.x); }) + .y(function(d) { return y(d.y); }); + + var svg = d3.select(id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + // Only show the first and last time in the graph + xAxis.tickValues(x.domain()); + + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(xAxis); + + svg.append("g") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "translate(0," + unitLabelYOffset + ")") + .text(unitY); + + if (batchInterval && batchInterval <= maxY) { + drawLine(svg, x, y, minX, batchInterval, maxX, batchInterval); + } + + svg.append("path") + .datum(data) + .attr("class", "line") + .attr("d", line); + // If the user click one point in the graphs, jump to the batch row and highlight it. And + // recovery the batch row after 3 seconds if necessary. + // We need to remember the last clicked batch so that we can recovery it. + var lastClickedBatch = null; + var lastTimeout = null; + + function isFailedBatch(batchTime) { + return $("#batch-" + batchTime).attr("isFailed") == "true"; + } + + // Add points to the line. However, we make it invisible at first. But when the user moves mouse + // over a point, it will be displayed with its detail. + svg.selectAll(".point") + .data(data) + .enter().append("circle") + .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) // white and opacity = 0 make it invisible + .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) + .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";}) + .style("cursor", "pointer") + .attr("cx", function(d) { return x(d.x); }) + .attr("cy", function(d) { return y(d.y); }) + .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";}) + .on('mouseover', function(d) { + var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x]; + showBootstrapTooltip(d3.select(this).node(), tip); + // show the point + d3.select(this) + .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";}) + .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "steelblue";}) + .attr("opacity", "1") + .attr("r", "3"); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + // hide the point + d3.select(this) + .attr("stroke", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) + .attr("fill", function(d) { return isFailedBatch(d.x) ? "red" : "white";}) + .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";}) + .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";}); + }) + .on("click", onClickTimeline); } /** @@ -242,105 +243,106 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) { * @param batchInterval if "batchInterval" is specified, we will draw a line for "batchInterval" in the graph */ function drawHistogram(id, values, minY, maxY, unitY, batchInterval) { - // Hide the left border of "". We cannot use "css" directly, or "sorttable.js" will override them. - d3.select(d3.select(id).node().parentNode) - .style("padding", "8px 8px 8px 0") - .style("border-left", "0px solid white"); - - var margin = {top: 20, right: 30, bottom: 30, left: 10}; - var width = 350 - margin.left - margin.right; - var height = 150 - margin.top - margin.bottom; - - var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width - 50]); - var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); - - var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5); - var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; }); - - var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values); - - var svg = d3.select(id).append("svg") - .attr("width", width + margin.left + margin.right) - .attr("height", height + margin.top + margin.bottom) - .append("g") - .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - - if (batchInterval && batchInterval <= maxY) { - drawLine(svg, x, y, 0, batchInterval, maxXForHistogram, batchInterval); - } + // Hide the left border of "". We cannot use "css" directly, or "sorttable.js" will override them. + d3.select(d3.select(id).node().parentNode) + .style("padding", "8px 8px 8px 0") + .style("border-left", "0px solid white"); + + var margin = {top: 20, right: 30, bottom: 30, left: 10}; + var width = 350 - margin.left - margin.right; + var height = 150 - margin.top - margin.bottom; + + var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width - 50]); + var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]); + + var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5); + var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; }); + + var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values); + + var svg = d3.select(id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + if (batchInterval && batchInterval <= maxY) { + drawLine(svg, x, y, 0, batchInterval, maxXForHistogram, batchInterval); + } + + svg.append("g") + .attr("class", "x axis") + .call(xAxis) + .append("text") + .attr("transform", "translate(" + (margin.left + width - 45) + ", " + unitLabelYOffset + ")") + .text("#batches"); + + svg.append("g") + .attr("class", "y axis") + .call(yAxis); + + svg.selectAll(".bar") + .data(data) + .enter() + .append("g") + .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx)) + ")";}) + .attr("class", "bar").append("rect") + .attr("width", function(d) { return x(d.y); }) + .attr("height", function(d) { return height - y(d.dx); }) + .on('mouseover', function(d) { + var percent = yValueFormat(d.y * 100.0 / values.length) + "%"; + var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY; + showBootstrapTooltip(d3.select(this).node(), tip); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + }); - svg.append("g") - .attr("class", "x axis") - .call(xAxis) - .append("text") - .attr("transform", "translate(" + (margin.left + width - 45) + ", " + unitLabelYOffset + ")") - .text("#batches"); - - svg.append("g") - .attr("class", "y axis") - .call(yAxis); - - svg.selectAll(".bar") - .data(data) - .enter() - .append("g") - .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx)) + ")";}) - .attr("class", "bar").append("rect") - .attr("width", function(d) { return x(d.y); }) - .attr("height", function(d) { return height - y(d.dx); }) - .on('mouseover', function(d) { - var percent = yValueFormat(d.y * 100.0 / values.length) + "%"; - var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY; - showBootstrapTooltip(d3.select(this).node(), tip); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - }); - - if (batchInterval && batchInterval <= maxY) { - // Add the "stable" text to the graph below the batch interval line. - var stableXOffset = x(maxXForHistogram) - 20; - var stableYOffset = y(batchInterval) + 15; - svg.append("text") - .style("fill", "lightblue") - .attr("class", "stable-text") - .attr("text-anchor", "middle") - .attr("transform", "translate(" + stableXOffset + "," + stableYOffset + ")") - .text("stable") - .on('mouseover', function(d) { - var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")"; - showBootstrapTooltip(d3.select(this).node(), tip); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - }); - } + if (batchInterval && batchInterval <= maxY) { + // Add the "stable" text to the graph below the batch interval line. + var stableXOffset = x(maxXForHistogram) - 20; + var stableYOffset = y(batchInterval) + 15; + svg.append("text") + .style("fill", "lightblue") + .attr("class", "stable-text") + .attr("text-anchor", "middle") + .attr("transform", "translate(" + stableXOffset + "," + stableYOffset + ")") + .text("stable") + .on('mouseover', function(d) { + var tip = "Processing Time <= Batch Interval (" + yValueFormat(batchInterval) +" " + unitY +")"; + showBootstrapTooltip(d3.select(this).node(), tip); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + }); + } } +/* eslint-enable no-unused-vars */ $(function() { - var status = window.localStorage && window.localStorage.getItem("show-streams-detail") == "true"; - - $("span.expand-input-rate").click(function() { - status = !status; - $("#inputs-table").toggle('collapsed'); - // Toggle the class of the arrow between open and closed - $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed'); - if (window.localStorage) { - window.localStorage.setItem("show-streams-detail", "" + status); - } - }); - - if (status) { - $("#inputs-table").toggle('collapsed'); - // Toggle the class of the arrow between open and closed - $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed'); + var status = window.localStorage && window.localStorage.getItem("show-streams-detail") == "true"; + + $("span.expand-input-rate").click(function() { + status = !status; + $("#inputs-table").toggle('collapsed'); + // Toggle the class of the arrow between open and closed + $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed'); + if (window.localStorage) { + window.localStorage.setItem("show-streams-detail", "" + status); } + }); + + if (status) { + $("#inputs-table").toggle('collapsed'); + // Toggle the class of the arrow between open and closed + $(this).find('.expand-input-rate-arrow').toggleClass('arrow-open').toggleClass('arrow-closed'); + } }); function highlightBatchRow(batch) { - $("#batch-" + batch).parent().addClass("batch-table-cell-highlight"); + $("#batch-" + batch).parent().addClass("batch-table-cell-highlight"); } function clearBatchRow(batch) { - $("#batch-" + batch).parent().removeClass("batch-table-cell-highlight"); + $("#batch-" + batch).parent().removeClass("batch-table-cell-highlight"); } diff --git a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js index c92226b408b6c..9701f5a57e170 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js +++ b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js @@ -15,157 +15,158 @@ * limitations under the License. */ +/* global d3, formattedTimeTipStrings, formattedTimeToValues, hideBootstrapTooltip, maxMarginLeftForTimeline, showBootstrapTooltip, unitLabelYOffset */ // pre-define some colors for legends. var colorPool = ["#F8C471", "#F39C12", "#B9770E", "#73C6B6", "#16A085", "#117A65", "#B2BABB", "#7F8C8D", "#616A6B"]; +/* eslint-disable no-unused-vars */ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) { - d3.select(d3.select(id).node().parentNode) - .style("padding", "8px 0 8px 8px") - .style("border-right", "0px solid white"); - - // Setup svg using Bostock's margin convention - var margin = {top: 20, right: 40, bottom: 30, left: maxMarginLeftForTimeline}; - var width = 850 - margin.left - margin.right; - var height = 300 - margin.top - margin.bottom; - - var svg = d3.select(id) - .append("svg") - .attr("width", width + margin.left + margin.right) - .attr("height", height + margin.top + margin.bottom) - .append("g") - .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - - var data = values; - - var parse = d3.time.format("%H:%M:%S.%L").parse; - - // Transpose the data into layers - var dataset = d3.layout.stack()(labels.map(function(fruit) { - return data.map(function(d) { - return {_x: d.x, x: parse(d.x), y: +d[fruit]}; - }); - })); - - - // Set x, y and colors - var x = d3.scale.ordinal() - .domain(dataset[0].map(function(d) { return d.x; })) - .rangeRoundBands([10, width-10], 0.02); - - var y = d3.scale.linear() - .domain([0, d3.max(dataset, function(d) { return d3.max(d, function(d) { return d.y0 + d.y; }); })]) - .range([height, 0]); - - var colors = colorPool.slice(0, labels.length) - - // Define and draw axes - var yAxis = d3.svg.axis() - .scale(y) - .orient("left") - .ticks(7) - .tickFormat( function(d) { return d } ); - - var xAxis = d3.svg.axis() - .scale(x) - .orient("bottom") - .tickFormat(d3.time.format("%H:%M:%S.%L")); - - // Only show the first and last time in the graph - var xline = [] - xline.push(x.domain()[0]) - xline.push(x.domain()[x.domain().length - 1]) - xAxis.tickValues(xline); - - svg.append("g") - .attr("class", "y axis") - .call(yAxis) - .append("text") - .attr("transform", "translate(0," + unitLabelYOffset + ")") - .text("ms"); - - svg.append("g") - .attr("class", "x axis") - .attr("transform", "translate(0," + height + ")") - .call(xAxis); - - // Create groups for each series, rects for each segment - var groups = svg.selectAll("g.cost") - .data(dataset) - .enter().append("g") - .attr("class", "cost") - .style("fill", function(d, i) { return colors[i]; }); - - var rect = groups.selectAll("rect") - .data(function(d) { return d; }) - .enter() - .append("rect") - .attr("x", function(d) { return x(d.x); }) - .attr("y", function(d) { return y(d.y0 + d.y); }) - .attr("height", function(d) { return y(d.y0) - y(d.y0 + d.y); }) - .attr("width", x.rangeBand()) - .on('mouseover', function(d) { - var tip = ''; - var idx = 0; - var _values = formattedTimeToValues[d._x]; - _values.forEach(function (k) { - tip += labels[idx] + ': ' + k + ' '; - idx += 1; - }); - tip += " at " + formattedTimeTipStrings[d._x]; - showBootstrapTooltip(d3.select(this).node(), tip); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - }) - .on("mousemove", function(d) { - var xPosition = d3.mouse(this)[0] - 15; - var yPosition = d3.mouse(this)[1] - 25; - tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")"); - tooltip.select("text").text(d.y); - }); - - - // Draw legend - var legend = svg.selectAll(".legend") - .data(colors) - .enter().append("g") - .attr("class", "legend") - .attr("transform", function(d, i) { return "translate(30," + i * 19 + ")"; }); - - legend.append("rect") - .attr("x", width - 20) - .attr("width", 18) - .attr("height", 18) - .style("fill", function(d, i) {return colors.slice().reverse()[i];}) - .on('mouseover', function(d, i) { - var len = labels.length - showBootstrapTooltip(d3.select(this).node(), labels[len - 1 - i]); - }) - .on('mouseout', function() { - hideBootstrapTooltip(d3.select(this).node()); - }) - .on("mousemove", function(d) { - var xPosition = d3.mouse(this)[0] - 15; - var yPosition = d3.mouse(this)[1] - 25; - tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")"); - tooltip.select("text").text(d.y); - }); - - // Prep the tooltip bits, initial display is hidden - var tooltip = svg.append("g") - .attr("class", "tooltip") - .style("display", "none"); - - tooltip.append("rect") - .attr("width", 30) - .attr("height", 20) - .attr("fill", "white") - .style("opacity", 0.5); - - tooltip.append("text") - .attr("x", 15) - .attr("dy", "1.2em") - .style("text-anchor", "middle") - .attr("font-size", "12px") - .attr("font-weight", "bold"); + d3.select(d3.select(id).node().parentNode) + .style("padding", "8px 0 8px 8px") + .style("border-right", "0px solid white"); + + // Setup svg using Bostock's margin convention + var margin = {top: 20, right: 40, bottom: 30, left: maxMarginLeftForTimeline}; + var width = 850 - margin.left - margin.right; + var height = 300 - margin.top - margin.bottom; + + var svg = d3.select(id) + .append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + var data = values; + + var parse = d3.time.format("%H:%M:%S.%L").parse; + + // Transpose the data into layers + var dataset = d3.layout.stack()(labels.map(function(fruit) { + return data.map(function(d) { + return {_x: d.x, x: parse(d.x), y: +d[fruit]}; + }); + })); + + // Set x, y and colors + var x = d3.scale.ordinal() + .domain(dataset[0].map(function(d) { return d.x; })) + .rangeRoundBands([10, width-10], 0.02); + + var y = d3.scale.linear() + .domain([0, d3.max(dataset, function(d) { return d3.max(d, function(d) { return d.y0 + d.y; }); })]) + .range([height, 0]); + + var colors = colorPool.slice(0, labels.length); + + // Define and draw axes + var yAxis = d3.svg.axis() + .scale(y) + .orient("left") + .ticks(7) + .tickFormat( function(d) { return d } ); + + var xAxis = d3.svg.axis() + .scale(x) + .orient("bottom") + .tickFormat(d3.time.format("%H:%M:%S.%L")); + + // Only show the first and last time in the graph + var xline = []; + xline.push(x.domain()[0]); + xline.push(x.domain()[x.domain().length - 1]); + xAxis.tickValues(xline); + + svg.append("g") + .attr("class", "y axis") + .call(yAxis) + .append("text") + .attr("transform", "translate(0," + unitLabelYOffset + ")") + .text("ms"); + + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(xAxis); + + // Create groups for each series, rects for each segment + var groups = svg.selectAll("g.cost") + .data(dataset) + .enter().append("g") + .attr("class", "cost") + .style("fill", function(d, i) { return colors[i]; }); + + var rect = groups.selectAll("rect") + .data(function(d) { return d; }) + .enter() + .append("rect") + .attr("x", function(d) { return x(d.x); }) + .attr("y", function(d) { return y(d.y0 + d.y); }) + .attr("height", function(d) { return y(d.y0) - y(d.y0 + d.y); }) + .attr("width", x.rangeBand()) + .on('mouseover', function(d) { + var tip = ''; + var idx = 0; + var _values = formattedTimeToValues[d._x]; + _values.forEach(function (k) { + tip += labels[idx] + ': ' + k + ' '; + idx += 1; + }); + tip += " at " + formattedTimeTipStrings[d._x]; + showBootstrapTooltip(d3.select(this).node(), tip); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + }) + .on("mousemove", function(d) { + var xPosition = d3.mouse(this)[0] - 15; + var yPosition = d3.mouse(this)[1] - 25; + tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")"); + tooltip.select("text").text(d.y); + }); + + // Draw legend + var legend = svg.selectAll(".legend") + .data(colors) + .enter().append("g") + .attr("class", "legend") + .attr("transform", function(d, i) { return "translate(30," + i * 19 + ")"; }); + + legend.append("rect") + .attr("x", width - 20) + .attr("width", 18) + .attr("height", 18) + .style("fill", function(d, i) {return colors.slice().reverse()[i];}) + .on('mouseover', function(d, i) { + var len = labels.length; + showBootstrapTooltip(d3.select(this).node(), labels[len - 1 - i]); + }) + .on('mouseout', function() { + hideBootstrapTooltip(d3.select(this).node()); + }) + .on("mousemove", function(d) { + var xPosition = d3.mouse(this)[0] - 15; + var yPosition = d3.mouse(this)[1] - 25; + tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")"); + tooltip.select("text").text(d.y); + }); + + // Prep the tooltip bits, initial display is hidden + var tooltip = svg.append("g") + .attr("class", "tooltip") + .style("display", "none"); + + tooltip.append("rect") + .attr("width", 30) + .attr("height", 20) + .attr("fill", "white") + .style("opacity", 0.5); + + tooltip.append("text") + .attr("x", 15) + .attr("dy", "1.2em") + .style("text-anchor", "middle") + .attr("font-size", "12px") + .attr("font-weight", "bold"); } +/* eslint-enable no-unused-vars */ \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/table.js b/core/src/main/resources/org/apache/spark/ui/static/table.js index 32b7a6522d050..0203748cf7dbc 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/table.js +++ b/core/src/main/resources/org/apache/spark/ui/static/table.js @@ -15,6 +15,8 @@ * limitations under the License. */ +/* global $ */ +/* eslint-disable no-unused-vars */ /* Adds background colors to stripe table rows in the summary table (on the stage page). This is * necessary (instead of using css or the table striping provided by bootstrap) because the summary * table has hidden rows. @@ -22,84 +24,88 @@ * An ID selector (rather than a class selector) is used to ensure this runs quickly even on pages * with thousands of task rows (ID selectors are much faster than class selectors). */ function stripeSummaryTable() { - $("#task-summary-table").find("tr:not(:hidden)").each(function (index) { - if (index % 2 == 1) { - $(this).css("background-color", "#f9f9f9"); - } else { - $(this).css("background-color", "#ffffff"); - } - }); + $("#task-summary-table").find("tr:not(:hidden)").each(function (index) { + if (index % 2 == 1) { + $(this).css("background-color", "#f9f9f9"); + } else { + $(this).css("background-color", "#ffffff"); + } + }); } +/* eslint-enable no-unused-vars */ function toggleThreadStackTrace(threadId, forceAdd) { - var stackTrace = $("#" + threadId + "_stacktrace") - if (stackTrace.length == 0) { - var stackTraceText = $('#' + threadId + "_td_stacktrace").html() - var threadCell = $("#thread_" + threadId + "_tr") - threadCell.after("
" +
-            stackTraceText +  "
") - } else { - if (!forceAdd) { - stackTrace.remove() - } + var stackTrace = $("#" + threadId + "_stacktrace"); + if (stackTrace.length == 0) { + var stackTraceText = $('#' + threadId + "_td_stacktrace").html(); + var threadCell = $("#thread_" + threadId + "_tr"); + threadCell.after("
" +
+      stackTraceText +  "
") + } else { + if (!forceAdd) { + stackTrace.remove() } + } } +/* eslint-disable no-unused-vars */ function expandAllThreadStackTrace(toggleButton) { - $('.accordion-heading').each(function() { - //get thread ID - if (!$(this).hasClass("d-none")) { - var trId = $(this).attr('id').match(/thread_([0-9]+)_tr/m)[1] - toggleThreadStackTrace(trId, true) - } - }) - if (toggleButton) { - $('.expandbutton').toggleClass('d-none') + $('.accordion-heading').each(function() { + //get thread ID + if (!$(this).hasClass("d-none")) { + var trId = $(this).attr('id').match(/thread_([0-9]+)_tr/m)[1]; + toggleThreadStackTrace(trId, true) } + }); + if (toggleButton) { + $('.expandbutton').toggleClass('d-none') + } } +/* eslint-enable no-unused-vars */ function collapseAllThreadStackTrace(toggleButton) { - $('.accordion-body').each(function() { - $(this).remove() - }) - if (toggleButton) { - $('.expandbutton').toggleClass('d-none'); - } + $('.accordion-body').each(function() { + $(this).remove() + }); + if (toggleButton) { + $('.expandbutton').toggleClass('d-none'); + } } - +/* eslint-disable no-unused-vars */ // inOrOut - true: over, false: out function onMouseOverAndOut(threadId) { - $("#" + threadId + "_td_id").toggleClass("threaddump-td-mouseover"); - $("#" + threadId + "_td_name").toggleClass("threaddump-td-mouseover"); - $("#" + threadId + "_td_state").toggleClass("threaddump-td-mouseover"); - $("#" + threadId + "_td_locking").toggleClass("threaddump-td-mouseover"); + $("#" + threadId + "_td_id").toggleClass("threaddump-td-mouseover"); + $("#" + threadId + "_td_name").toggleClass("threaddump-td-mouseover"); + $("#" + threadId + "_td_state").toggleClass("threaddump-td-mouseover"); + $("#" + threadId + "_td_locking").toggleClass("threaddump-td-mouseover"); } function onSearchStringChange() { - var searchString = $('#search').val().toLowerCase(); - //remove the stacktrace - collapseAllThreadStackTrace(false) - if (searchString.length == 0) { - $('tr').each(function() { - $(this).removeClass('d-none') - }) - } else { - $('tr').each(function(){ - if($(this).attr('id') && $(this).attr('id').match(/thread_[0-9]+_tr/) ) { - var children = $(this).children() - var found = false - for (var i = 0; i < children.length; i++) { - if (children.eq(i).text().toLowerCase().indexOf(searchString) >= 0) { - found = true - } - } - if (found) { - $(this).removeClass('d-none') - } else { - $(this).addClass('d-none') - } - } - }); - } + var searchString = $('#search').val().toLowerCase(); + //remove the stacktrace + collapseAllThreadStackTrace(false); + if (searchString.length == 0) { + $('tr').each(function() { + $(this).removeClass('d-none') + }) + } else { + $('tr').each(function(){ + if($(this).attr('id') && $(this).attr('id').match(/thread_[0-9]+_tr/) ) { + var children = $(this).children(); + var found = false; + for (var i = 0; i < children.length; i++) { + if (children.eq(i).text().toLowerCase().indexOf(searchString) >= 0) { + found = true; + } + } + if (found) { + $(this).removeClass('d-none') + } else { + $(this).addClass('d-none') + } + } + }); + } } +/* eslint-enable no-unused-vars */ diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js index 220b76a0f1b27..274848d5e62b5 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js +++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js @@ -15,6 +15,8 @@ * limitations under the License. */ +/* global $, vis, uiRoot, appBasePath */ +/* eslint-disable no-unused-vars */ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) { var groups = new vis.DataSet(groupArray); var items = new vis.DataSet(eventObjArray); @@ -249,7 +251,7 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma var visibilityState = status ? "" : "none"; $("#task-assignment-timeline").css("display", visibilityState); - // Switch the class of the arrow from open to closed. + // Switch the class of the arrow from open to closed. $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-open"); $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-closed"); @@ -280,6 +282,7 @@ function setupExecutorEventAction() { ); }); } +/* eslint-enable no-unused-vars */ function setupZoomable(id, timeline) { $(id + ' > input[type="checkbox"]').click(function() { diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js index 0026850654447..a2d4f5568cb26 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/utils.js +++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js @@ -15,42 +15,46 @@ * limitations under the License. */ +/* global $, uiRoot */ +/* eslint-disable no-unused-vars */ // this function works exactly the same as UIUtils.formatDuration function formatDuration(milliseconds) { - if (milliseconds < 100) { - return parseInt(milliseconds).toFixed(1) + " ms"; - } - var seconds = milliseconds * 1.0 / 1000; - if (seconds < 1) { - return seconds.toFixed(1) + " s"; - } - if (seconds < 60) { - return seconds.toFixed(0) + " s"; - } - var minutes = seconds / 60; - if (minutes < 10) { - return minutes.toFixed(1) + " min"; - } else if (minutes < 60) { - return minutes.toFixed(0) + " min"; - } - var hours = minutes / 60; - return hours.toFixed(1) + " h"; + if (milliseconds < 100) { + return parseInt(milliseconds).toFixed(1) + " ms"; + } + var seconds = milliseconds * 1.0 / 1000; + if (seconds < 1) { + return seconds.toFixed(1) + " s"; + } + if (seconds < 60) { + return seconds.toFixed(0) + " s"; + } + var minutes = seconds / 60; + if (minutes < 10) { + return minutes.toFixed(1) + " min"; + } else if (minutes < 60) { + return minutes.toFixed(0) + " min"; + } + var hours = minutes / 60; + return hours.toFixed(1) + " h"; } function formatBytes(bytes, type) { - if (type !== 'display') return bytes; - if (bytes <= 0) return '0.0 B'; - var k = 1024; - var dm = 1; - var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; - var i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; + if (type !== 'display') return bytes; + if (bytes <= 0) return '0.0 B'; + var k = 1024; + var dm = 1; + var sizes = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } +/* eslint-enable no-unused-vars */ function padZeroes(num) { return ("0" + num).slice(-2); } +/* eslint-disable no-unused-vars */ function formatTimeMillis(timeMillis) { if (timeMillis <= 0) { return "-"; @@ -59,16 +63,18 @@ function formatTimeMillis(timeMillis) { return formatDateString(dt); } } +/* eslint-enable no-unused-vars */ function formatDateString(dt) { - return dt.getFullYear() + "-" + - padZeroes(dt.getMonth() + 1) + "-" + - padZeroes(dt.getDate()) + " " + - padZeroes(dt.getHours()) + ":" + - padZeroes(dt.getMinutes()) + ":" + - padZeroes(dt.getSeconds()); + return dt.getFullYear() + "-" + + padZeroes(dt.getMonth() + 1) + "-" + + padZeroes(dt.getDate()) + " " + + padZeroes(dt.getHours()) + ":" + + padZeroes(dt.getMinutes()) + ":" + + padZeroes(dt.getSeconds()); } +/* eslint-disable no-unused-vars */ function getTimeZone() { try { return Intl.DateTimeFormat().resolvedOptions().timeZone; @@ -92,14 +98,15 @@ function formatLogsCells(execLogs, type) { function getStandAloneAppId(cb) { var words = document.baseURI.split('/'); var ind = words.indexOf("proxy"); + var appId; if (ind > 0) { - var appId = words[ind + 1]; + appId = words[ind + 1]; cb(appId); return; } ind = words.indexOf("history"); if (ind > 0) { - var appId = words[ind + 1]; + appId = words[ind + 1]; cb(appId); return; } @@ -119,7 +126,7 @@ function getStandAloneAppId(cb) { // It will convert the string into integer for correct ordering function ConvertDurationString(data) { data = data.toString(); - var units = data.replace(/[\d\.]/g, '' ) + var units = data.replace(/[\d.]/g, '' ) .replace(' ', '') .toLowerCase(); var multiplier = 1; @@ -143,13 +150,14 @@ function ConvertDurationString(data) { function createTemplateURI(appId, templateName) { var words = document.baseURI.split('/'); var ind = words.indexOf("proxy"); + var baseURI; if (ind > 0) { - var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/' + templateName + '-template.html'; + baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/' + templateName + '-template.html'; return baseURI; } ind = words.indexOf("history"); if(ind > 0) { - var baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html'; + baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html'; return baseURI; } return uiRoot + "/static/" + templateName + "-template.html"; @@ -159,7 +167,7 @@ function setDataTableDefaults() { $.extend($.fn.dataTable.defaults, { stateSave: true, stateSaveParams: function(_, data) { - data.search.search = ""; + data.search.search = ""; }, lengthMenu: [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]], pageLength: 20 @@ -169,51 +177,54 @@ function setDataTableDefaults() { function formatDate(date) { if (date <= 0) return "-"; else { - var dt = new Date(date.replace("GMT", "Z")); - return formatDateString(dt); + var dt = new Date(date.replace("GMT", "Z")); + return formatDateString(dt); } } function createRESTEndPointForExecutorsPage(appId) { - var words = document.baseURI.split('/'); - var ind = words.indexOf("proxy"); - if (ind > 0) { - var appId = words[ind + 1]; - var newBaseURI = words.slice(0, ind + 2).join('/'); - return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors"; - } - ind = words.indexOf("history"); - if (ind > 0) { - var appId = words[ind + 1]; - var attemptId = words[ind + 2]; - var newBaseURI = words.slice(0, ind).join('/'); - if (isNaN(attemptId)) { - return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors"; - } else { - return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors"; - } + var words = document.baseURI.split('/'); + var ind = words.indexOf("proxy"); + var newBaseURI; + if (ind > 0) { + appId = words[ind + 1]; + newBaseURI = words.slice(0, ind + 2).join('/'); + return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors"; + } + ind = words.indexOf("history"); + if (ind > 0) { + appId = words[ind + 1]; + var attemptId = words[ind + 2]; + newBaseURI = words.slice(0, ind).join('/'); + if (isNaN(attemptId)) { + return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors"; + } else { + return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors"; } - return uiRoot + "/api/v1/applications/" + appId + "/allexecutors"; + } + return uiRoot + "/api/v1/applications/" + appId + "/allexecutors"; } function createRESTEndPointForMiscellaneousProcess(appId) { - var words = document.baseURI.split('/'); - var ind = words.indexOf("proxy"); - if (ind > 0) { - var appId = words[ind + 1]; - var newBaseURI = words.slice(0, ind + 2).join('/'); - return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess"; - } - ind = words.indexOf("history"); - if (ind > 0) { - var appId = words[ind + 1]; - var attemptId = words[ind + 2]; - var newBaseURI = words.slice(0, ind).join('/'); - if (isNaN(attemptId)) { - return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess"; - } else { - return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allmiscellaneousprocess"; - } + var words = document.baseURI.split('/'); + var ind = words.indexOf("proxy"); + var newBaseURI; + if (ind > 0) { + appId = words[ind + 1]; + newBaseURI = words.slice(0, ind + 2).join('/'); + return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess"; + } + ind = words.indexOf("history"); + if (ind > 0) { + appId = words[ind + 1]; + var attemptId = words[ind + 2]; + newBaseURI = words.slice(0, ind).join('/'); + if (isNaN(attemptId)) { + return newBaseURI + "/api/v1/applications/" + appId + "/allmiscellaneousprocess"; + } else { + return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allmiscellaneousprocess"; } - return uiRoot + "/api/v1/applications/" + appId + "/allmiscellaneousprocess"; + } + return uiRoot + "/api/v1/applications/" + appId + "/allmiscellaneousprocess"; } +/* eslint-enable no-unused-vars */ \ No newline at end of file diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js index bb3725650c667..c149f2d84337e 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/webui.js +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js @@ -15,16 +15,19 @@ * limitations under the License. */ +/* global $ */ +/* eslint-disable no-unused-vars */ var uiRoot = ""; var appBasePath = ""; function setUIRoot(val) { - uiRoot = val; + uiRoot = val; } function setAppBasePath(path) { - appBasePath = path; + appBasePath = path; } +/* eslint-enable no-unused-vars */ function collapseTablePageLoad(name, table){ if (window.localStorage.getItem(name) == "true") { @@ -35,20 +38,20 @@ function collapseTablePageLoad(name, table){ } function collapseTable(thisName, table){ - var status = window.localStorage.getItem(thisName) == "true"; - status = !status; + var status = window.localStorage.getItem(thisName) == "true"; + status = !status; - var thisClass = '.' + thisName; + var thisClass = '.' + thisName; - // Expand the list of additional metrics. - var tableDiv = $(thisClass).parent().find('.' + table); - $(tableDiv).toggleClass('collapsed'); + // Expand the list of additional metrics. + var tableDiv = $(thisClass).parent().find('.' + table); + $(tableDiv).toggleClass('collapsed'); - // Switch the class of the arrow from open to closed. - $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-open'); - $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-closed'); + // Switch the class of the arrow from open to closed. + $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-open'); + $(thisClass).find('.collapse-table-arrow').toggleClass('arrow-closed'); - window.localStorage.setItem(thisName, "" + status); + window.localStorage.setItem(thisName, "" + status); } // Add a call to collapseTablePageLoad() on each collapsible table @@ -101,8 +104,8 @@ $(function() { }); $(function() { - // Show/hide full job description on click event. - $(".description-input").click(function() { - $(this).toggleClass("description-input-full"); - }); + // Show/hide full job description on click event. + $(".description-input").click(function() { + $(this).toggleClass("description-input-full"); + }); }); diff --git a/dev/.rat-excludes b/dev/.rat-excludes index 167cf224f92c2..6c809f43418cd 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -134,3 +134,4 @@ flights_tiny.txt.1 over1k over10k exported_table/* +node_modules \ No newline at end of file diff --git a/dev/eslint.json b/dev/eslint.json new file mode 100644 index 0000000000000..ee1fd3dcc6e71 --- /dev/null +++ b/dev/eslint.json @@ -0,0 +1,24 @@ +{ + "env": { + "browser": true, + "es6": true + }, + "extends": "eslint:recommended", + "rules": { + "indent": [ + "error", + 2, + { + "SwitchCase": 1, + "MemberExpression": "off" + } + ], + "no-unused-vars": ["error", {"argsIgnorePattern": "^_ignored_.*"}] + }, + "ignorePatterns": [ + "*.min.js", + "sorttable.js", + "jquery.mustache.js", + "dataTables.rowsGroup.js" + ] +} diff --git a/dev/lint-js b/dev/lint-js new file mode 100755 index 0000000000000..ce06e282192a0 --- /dev/null +++ b/dev/lint-js @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o pipefail + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)" +LINT_JS_REPORT_FILE_NAME="$SPARK_ROOT_DIR/dev/lint-js-report.log" +LINT_TARGET_FILES=( + "$SPARK_ROOT_DIR/core/src/main/resources/org/apache/spark/ui/static/" + "$SPARK_ROOT_DIR/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/" + "$SPARK_ROOT_DIR/docs/js" +) + +if ! type "npm" > /dev/null; then + echo "ERROR: You should install npm" + exit 1 +fi + +if ! type "npx" > /dev/null; then + echo "ERROR: You should install npx" + exit 1 +fi + +cd $SCRIPT_DIR + +if ! npm ls eslint > /dev/null; then + npm ci eslint +fi + +npx eslint -c "$SPARK_ROOT_DIR/dev/eslint.json" $LINT_TARGET_FILES | tee "$LINT_JS_REPORT_FILE_NAME" +lint_status=$? + +if [ "$lint_status" = "0" ] ; then + echo "lint-js checks passed." +else + echo "lint-js checks failed." +fi + +exit "$lint_status" diff --git a/dev/package-lock.json b/dev/package-lock.json new file mode 100644 index 0000000000000..a57f45bcf7184 --- /dev/null +++ b/dev/package-lock.json @@ -0,0 +1,979 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "@babel/code-frame": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", + "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "dev": true, + "requires": { + "@babel/highlight": "^7.10.4" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.0.tgz", + "integrity": "sha512-V3ts7zMSu5lfiwWDVWzRDGIN+lnCEUdaXgtVHJgLb1rGaA6jMrtB9EmE7L18foXJIE8Un/A/h6NJfGQp/e1J4A==", + "dev": true + }, + "@babel/highlight": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.14.0.tgz", + "integrity": "sha512-YSCOwxvTYEIMSGaBQb5kDDsCopDdiUGsqpatp3fOlI4+2HQSkTmEVWnVuySdAC5EWCqSWWTv0ib63RjR7dTBdg==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.14.0", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "dependencies": { + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + } + } + }, + "@eslint/eslintrc": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.0.tgz", + "integrity": "sha512-2ZPCc+uNbjV5ERJr+aKSPRwZgKd2z11x0EgLvb1PURmUrn9QNRXFqje0Ldq454PfAVyaJYyrDvvIKSFP4NnBog==", + "dev": true, + "requires": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + } + } + }, + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, + "acorn-jsx": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", + "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", + "dev": true + }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true + }, + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "astral-regex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", + "dev": true + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, + "chalk": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", + "integrity": "sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "enquirer": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", + "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "dev": true, + "requires": { + "ansi-colors": "^4.1.1" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "eslint": { + "version": "7.25.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.25.0.tgz", + "integrity": "sha512-TVpSovpvCNpLURIScDRB6g5CYu/ZFq9GfX2hLNIV4dSBKxIWojeDODvYl3t0k0VtMxYeR8OXPCFE5+oHMlGfhw==", + "dev": true, + "requires": { + "@babel/code-frame": "7.12.11", + "@eslint/eslintrc": "^0.4.0", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "enquirer": "^2.3.5", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^2.0.0", + "espree": "^7.3.1", + "esquery": "^1.4.0", + "esutils": "^2.0.2", + "file-entry-cache": "^6.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^13.6.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash": "^4.17.21", + "minimatch": "^3.0.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "progress": "^2.0.0", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^6.0.4", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + } + }, + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } + } + }, + "eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "dev": true + }, + "espree": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", + "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", + "dev": true, + "requires": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.3.1", + "eslint-visitor-keys": "^1.3.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esquery": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", + "dev": true, + "requires": { + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, + "file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "requires": { + "flat-cache": "^3.0.4" + } + }, + "flat-cache": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", + "integrity": "sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg==", + "dev": true, + "requires": { + "flatted": "^3.1.0", + "rimraf": "^3.0.2" + } + }, + "flatted": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz", + "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==", + "dev": true + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globals": { + "version": "13.8.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.8.0.tgz", + "integrity": "sha512-rHtdA6+PDBIjeEvA91rpqzEvk/k3/i7EeNQiryiWuJH0Hw9cpyJMAt2jtbAwUaRdhD+573X4vWw6IcjKPasi9Q==", + "dev": true, + "requires": { + "type-fest": "^0.20.2" + }, + "dependencies": { + "type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true + } + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, + "levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "requires": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", + "dev": true + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=", + "dev": true + }, + "lodash.truncate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", + "integrity": "sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM=", + "dev": true + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "optionator": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "dev": true, + "requires": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" + } + }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true + }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, + "regexpp": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", + "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", + "dev": true + }, + "require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "slice-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", + "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + } + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "string-width": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.2.tgz", + "integrity": "sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "table": { + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/table/-/table-6.6.0.tgz", + "integrity": "sha512-iZMtp5tUvcnAdtHpZTWLPF0M7AgiQsURR2DwmxnJwSy8I3+cY+ozzVvYha3BOLG2TB+L0CqjIz+91htuj6yCXg==", + "dev": true, + "requires": { + "ajv": "^8.0.1", + "lodash.clonedeep": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ajv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.2.0.tgz", + "integrity": "sha512-WSNGFuyWd//XO8n/m/EaOlNLtO0yL8EXT/74LqT4khdhpZjP7lkj/kT5uwRmGitKEVp/Oj7ZUHeGfPtgHhQ5CA==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true + } + } + }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, + "type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "requires": { + "prelude-ls": "^1.2.1" + } + }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + }, + "uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "requires": { + "punycode": "^2.1.0" + } + }, + "v8-compile-cache": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", + "integrity": "sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } +} diff --git a/dev/package.json b/dev/package.json new file mode 100644 index 0000000000000..0391a3983f78f --- /dev/null +++ b/dev/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "eslint": "^7.25.0" + } +} From 6f0ef93f9a72ee704785cfe2421e3fe3587b4df6 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Sat, 8 May 2021 00:03:02 +0900 Subject: [PATCH 4/7] [SPARK-35297][CORE][DOC][MINOR] Modify the comment about the executor ### What changes were proposed in this pull request? Now Spark Executor already can be used in Kubernetes scheduler. So we should modify the annotation in the Executor.scala. ### Why are the changes needed? only comment ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? no Closes #32426 from jerqi/master. Authored-by: RoryQi <1242949407@qq.com> Signed-off-by: Takeshi Yamamuro --- core/src/main/scala/org/apache/spark/executor/Executor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index 8fc1c80cb7f69..acb83f33b2e3c 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -55,7 +55,7 @@ import org.apache.spark.util.io.ChunkedByteBuffer /** * Spark executor, backed by a threadpool to run tasks. * - * This can be used with Mesos, YARN, and the standalone scheduler. + * This can be used with Mesos, YARN, kubernetes and the standalone scheduler. * An internal RPC interface is used for communication with the driver, * except in the case of Mesos fine-grained mode. */ From 33fbf5647b4a5587c78ac51339c0cbc9d70547a4 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 7 May 2021 09:07:57 -0700 Subject: [PATCH 5/7] [SPARK-35288][SQL] StaticInvoke should find the method without exact argument classes match ### What changes were proposed in this pull request? This patch proposes to make StaticInvoke able to find method with given method name even the parameter types do not exactly match to argument classes. ### Why are the changes needed? Unlike `Invoke`, `StaticInvoke` only tries to get the method with exact argument classes. If the calling method's parameter types are not exactly matched with the argument classes, `StaticInvoke` cannot find the method. `StaticInvoke` should be able to find the method under the cases too. ### Does this PR introduce _any_ user-facing change? Yes. `StaticInvoke` can find a method even the argument classes are not exactly matched. ### How was this patch tested? Unit test. Closes #32413 from viirya/static-invoke. Authored-by: Liang-Chi Hsieh Signed-off-by: Liang-Chi Hsieh --- .../expressions/objects/objects.scala | 56 ++++++++++--------- .../expressions/ObjectExpressionsSuite.scala | 34 +++++++++-- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index 40378a3a5f72d..5d79774a303d2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -145,6 +145,34 @@ trait InvokeLike extends Expression with NonSQLExpression { } } } + + final def findMethod(cls: Class[_], functionName: String, argClasses: Seq[Class[_]]): Method = { + // Looking with function name + argument classes first. + try { + cls.getMethod(functionName, argClasses: _*) + } catch { + case _: NoSuchMethodException => + // For some cases, e.g. arg class is Object, `getMethod` cannot find the method. + // We look at function name + argument length + val m = cls.getMethods.filter { m => + m.getName == functionName && m.getParameterCount == arguments.length + } + if (m.isEmpty) { + sys.error(s"Couldn't find $functionName on $cls") + } else if (m.length > 1) { + // More than one matched method signature. Exclude synthetic one, e.g. generic one. + val realMethods = m.filter(!_.isSynthetic) + if (realMethods.length > 1) { + // Ambiguous case, we don't know which method to choose, just fail it. + sys.error(s"Found ${realMethods.length} $functionName on $cls") + } else { + realMethods.head + } + } else { + m.head + } + } + } } /** @@ -236,7 +264,7 @@ case class StaticInvoke( override def children: Seq[Expression] = arguments lazy val argClasses = ScalaReflection.expressionJavaClasses(arguments) - @transient lazy val method = cls.getDeclaredMethod(functionName, argClasses : _*) + @transient lazy val method = findMethod(cls, functionName, argClasses) override def eval(input: InternalRow): Any = { invoke(null, method, arguments, input, dataType) @@ -326,31 +354,7 @@ case class Invoke( @transient lazy val method = targetObject.dataType match { case ObjectType(cls) => - // Looking with function name + argument classes first. - try { - Some(cls.getMethod(encodedFunctionName, argClasses: _*)) - } catch { - case _: NoSuchMethodException => - // For some cases, e.g. arg class is Object, `getMethod` cannot find the method. - // We look at function name + argument length - val m = cls.getMethods.filter { m => - m.getName == encodedFunctionName && m.getParameterCount == arguments.length - } - if (m.isEmpty) { - sys.error(s"Couldn't find $encodedFunctionName on $cls") - } else if (m.length > 1) { - // More than one matched method signature. Exclude synthetic one, e.g. generic one. - val realMethods = m.filter(!_.isSynthetic) - if (realMethods.length > 1) { - // Ambiguous case, we don't know which method to choose, just fail it. - sys.error(s"Found ${realMethods.length} $encodedFunctionName on $cls") - } else { - Some(realMethods.head) - } - } else { - Some(m.head) - } - } + Some(findMethod(cls, encodedFunctionName, argClasses)) case _ => None } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala index ea38f190a0067..5e4c5bcd0ecc9 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala @@ -638,8 +638,22 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val clsType = ObjectType(classOf[ConcreteClass]) val obj = new ConcreteClass + val input = (1, 2) checkObjectExprEvaluation( - Invoke(Literal(obj, clsType), "testFunc", IntegerType, Seq(Literal(1))), 0) + Invoke(Literal(obj, clsType), "testFunc", IntegerType, + Seq(Literal(input, ObjectType(input.getClass)))), 2) + } + + test("SPARK-35288: static invoke should find method without exact param type match") { + val input = (1, 2) + + checkObjectExprEvaluation( + StaticInvoke(TestStaticInvoke.getClass, IntegerType, "func", + Seq(Literal(input, ObjectType(input.getClass)))), 3) + + checkObjectExprEvaluation( + StaticInvoke(TestStaticInvoke.getClass, IntegerType, "func", + Seq(Literal(1, IntegerType))), -1) } test("SPARK-35281: StaticInvoke shouldn't box primitive when result is nullable") { @@ -659,12 +673,24 @@ class TestBean extends Serializable { assert(i != null, "this setter should not be called with null.") } +object TestStaticInvoke { + def func(param: Any): Int = param match { + case pair: Tuple2[_, _] => + pair.asInstanceOf[Tuple2[Int, Int]]._1 + pair.asInstanceOf[Tuple2[Int, Int]]._2 + case _ => -1 + } +} + abstract class BaseClass[T] { - def testFunc(param: T): T + def testFunc(param: T): Int } -class ConcreteClass extends BaseClass[Int] with Serializable { - override def testFunc(param: Int): Int = param - 1 +class ConcreteClass extends BaseClass[Product] with Serializable { + override def testFunc(param: Product): Int = param match { + case _: Tuple2[_, _] => 2 + case _: Tuple3[_, _, _] => 3 + case _ => 4 + } } case object TestFun { From b4ec9e230484db88c6220c27e43e3db11f3bdeef Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Fri, 7 May 2021 15:06:04 -0700 Subject: [PATCH 6/7] [SPARK-35321][SQL] Don't register Hive permanent functions when creating Hive client ### What changes were proposed in this pull request? Instantiate a new Hive client through `Hive.getWithFastCheck(conf, false)` instead of `Hive.get(conf)`. ### Why are the changes needed? [HIVE-10319](https://issues.apache.org/jira/browse/HIVE-10319) introduced a new API `get_all_functions` which is only supported in Hive 1.3.0/2.0.0 and up. As result, when Spark 3.x talks to a HMS service of version 1.2 or lower, the following error will occur: ``` Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.thrift.TApplicationException: Invalid method name: 'get_all_functions' at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3897) at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:248) at org.apache.hadoop.hive.ql.metadata.Hive.registerAllFunctionsOnce(Hive.java:231) ... 96 more Caused by: org.apache.thrift.TApplicationException: Invalid method name: 'get_all_functions' at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:79) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_functions(ThriftHiveMetastore.java:3845) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_functions(ThriftHiveMetastore.java:3833) ``` The `get_all_functions` is called only when `doRegisterAllFns` is set to true: ```java private Hive(HiveConf c, boolean doRegisterAllFns) throws HiveException { conf = c; if (doRegisterAllFns) { registerAllFunctionsOnce(); } } ``` what this does is to register all Hive permanent functions defined in HMS in Hive's `FunctionRegistry` class, via iterating through results from `get_all_functions`. To Spark, this seems unnecessary as it loads Hive permanent (not built-in) UDF via directly calling the HMS API, i.e., `get_function`. The `FunctionRegistry` is only used in loading Hive's built-in function that is not supported by Spark. At this time, it only applies to `histogram_numeric`. ### Does this PR introduce _any_ user-facing change? Yes with this fix Spark now should be able to talk to HMS server with Hive 1.2.x and lower (with HIVE-24608 too) ### How was this patch tested? Manually started a HMS server of Hive version 1.2.2, with patched Hive 2.3.8 using HIVE-24608. Without the PR it failed with the above exception. With the PR the error disappeared and I can successfully perform common operations such as create table, create database, list tables, etc. Closes #32446 from sunchao/SPARK-35321. Authored-by: Chao Sun Signed-off-by: Dongjoon Hyun --- .../apache/spark/sql/hive/client/HiveClientImpl.scala | 4 ++-- .../org/apache/spark/sql/hive/client/HiveShim.scala | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index e9728b8c0461d..9bb3f45b96c11 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -273,7 +273,7 @@ private[hive] class HiveClientImpl( if (clientLoader.cachedHive != null) { clientLoader.cachedHive.asInstanceOf[Hive] } else { - val c = Hive.get(conf) + val c = shim.getHive(conf) clientLoader.cachedHive = c c } @@ -303,7 +303,7 @@ private[hive] class HiveClientImpl( // with the side-effect of Hive.get(conf) to avoid using out-of-date HiveConf. // See discussion in https://github.com/apache/spark/pull/16826/files#r104606859 // for more details. - Hive.get(conf) + shim.getHive(conf) // setCurrentSessionState will use the classLoader associated // with the HiveConf in `state` to override the context class loader of the current // thread. diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala index 2f7fe96013dea..b0a877d732be7 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala @@ -177,6 +177,8 @@ private[client] sealed abstract class Shim { def getMSC(hive: Hive): IMetaStoreClient + def getHive(hiveConf: HiveConf): Hive + protected def findMethod(klass: Class[_], name: String, args: Class[_]*): Method = { klass.getMethod(name, args: _*) } @@ -199,6 +201,8 @@ private[client] class Shim_v0_12 extends Shim with Logging { getMSCMethod.invoke(hive).asInstanceOf[IMetaStoreClient] } + override def getHive(hiveConf: HiveConf): Hive = Hive.get(hiveConf) + private lazy val startMethod = findStaticMethod( classOf[SessionState], @@ -1316,6 +1320,13 @@ private[client] class Shim_v2_1 extends Shim_v2_0 { override def alterPartitions(hive: Hive, tableName: String, newParts: JList[Partition]): Unit = { alterPartitionsMethod.invoke(hive, tableName, newParts, environmentContextInAlterTable) } + + // HIVE-10319 introduced a new HMS thrift API `get_all_functions` which is used by + // `Hive.get` since version 2.1.0, when it loads all Hive permanent functions during + // initialization. This breaks compatibility with HMS server of lower versions. + // To mitigate here we use `Hive.getWithFastCheck` instead which skips loading the permanent + // functions and therefore avoids calling `get_all_functions`. + override def getHive(hiveConf: HiveConf): Hive = Hive.getWithFastCheck(hiveConf, false) } private[client] class Shim_v2_2 extends Shim_v2_1 From f47e0f83794fc9beee3c07dca4c0bb7e0eab81e4 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Fri, 7 May 2021 20:34:51 -0700 Subject: [PATCH 7/7] [SPARK-35261][SQL] Support static magic method for stateless Java ScalarFunction ### What changes were proposed in this pull request? This allows `ScalarFunction` implemented in Java to optionally specify the magic method `invoke` to be static, which can be used if the UDF is stateless. Comparing to the non-static method, it can potentially give better performance due to elimination of dynamic dispatch, etc. Also added a benchmark to measure performance of: the default `produceResult`, non-static magic method and static magic method. ### Why are the changes needed? For UDFs that are stateless (e.g., no need to maintain intermediate state between each function call), it's better to allow users to implement the UDF function as static method which could potentially give better performance. ### Does this PR introduce _any_ user-facing change? Yes. Spark users can now have the choice to define static magic method for `ScalarFunction` when it is written in Java and when the UDF is stateless. ### How was this patch tested? Added new UT. Closes #32407 from sunchao/SPARK-35261. Authored-by: Chao Sun Signed-off-by: Dongjoon Hyun --- .../catalog/functions/ScalarFunction.java | 60 +++++-- .../sql/catalyst/analysis/Analyzer.scala | 5 +- .../V2FunctionBenchmark-jdk11-results.txt | 44 ++++++ .../V2FunctionBenchmark-results.txt | 44 ++++++ .../catalog/functions/JavaLongAdd.java | 130 ++++++++++++++++ .../catalog/functions/JavaStrLen.java | 48 +++--- .../connector/DataSourceV2FunctionSuite.scala | 22 +++ .../functions/V2FunctionBenchmark.scala | 147 ++++++++++++++++++ 8 files changed, 461 insertions(+), 39 deletions(-) create mode 100644 sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt create mode 100644 sql/core/benchmarks/V2FunctionBenchmark-results.txt create mode 100644 sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java index ef755aae3fb07..858ab923490fc 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/functions/ScalarFunction.java @@ -29,33 +29,62 @@ *

* The JVM type of result values produced by this function must be the type used by Spark's * InternalRow API for the {@link DataType SQL data type} returned by {@link #resultType()}. + * The mapping between {@link DataType} and the corresponding JVM type is defined below. *

* IMPORTANT: the default implementation of {@link #produceResult} throws - * {@link UnsupportedOperationException}. Users can choose to override this method, or implement - * a "magic method" with name {@link #MAGIC_METHOD_NAME} which takes individual parameters - * instead of a {@link InternalRow}. The magic method will be loaded by Spark through Java - * reflection and will also provide better performance in general, due to optimizations such as - * codegen, removal of Java boxing, etc. - * + * {@link UnsupportedOperationException}. Users must choose to either override this method, or + * implement a magic method with name {@link #MAGIC_METHOD_NAME}, which takes individual parameters + * instead of a {@link InternalRow}. The magic method approach is generally recommended because it + * provides better performance over the default {@link #produceResult}, due to optimizations such + * as whole-stage codegen, elimination of Java boxing, etc. + *

+ * In addition, for stateless Java functions, users can optionally define the + * {@link #MAGIC_METHOD_NAME} as a static method, which further avoids certain runtime costs such + * as Java dynamic dispatch. + *

* For example, a scalar UDF for adding two integers can be defined as follow with the magic * method approach: * * 

  *   public class IntegerAdd implements{@code ScalarFunction} {
+ *     public DataType[] inputTypes() {
+ *       return new DataType[] { DataTypes.IntegerType, DataTypes.IntegerType };
+ *     }
  *     public int invoke(int left, int right) {
  *       return left + right;
  *     }
  *   }
  *
- * In this case, since {@link #MAGIC_METHOD_NAME} is defined, Spark will use it over - * {@link #produceResult} to evalaute the inputs. In general Spark looks up the magic method by - * first converting the actual input SQL data types to their corresponding Java types following - * the mapping defined below, and then checking if there is a matching method from all the - * declared methods in the UDF class, using method name (i.e., {@link #MAGIC_METHOD_NAME}) and - * the Java types. If no magic method is found, Spark will falls back to use {@link #produceResult}. + * In the above, since {@link #MAGIC_METHOD_NAME} is defined, and also that it has + * matching parameter types and return type, Spark will use it to evaluate inputs. + *

+ * As another example, in the following: + * 

+ *   public class IntegerAdd implements{@code ScalarFunction} {
+ *     public DataType[] inputTypes() {
+ *       return new DataType[] { DataTypes.IntegerType, DataTypes.IntegerType };
+ *     }
+ *     public static int invoke(int left, int right) {
+ *       return left + right;
+ *     }
+ *     public Integer produceResult(InternalRow input) {
+ *       return input.getInt(0) + input.getInt(1);
+ *     }
+ *   }
+ *
+ * + * the class defines both the magic method and the {@link #produceResult}, and Spark will use + * {@link #MAGIC_METHOD_NAME} over the {@link #produceResult(InternalRow)} as it takes higher + * precedence. Also note that the magic method is annotated as a static method in this case. + *

+ * Resolution on magic method is done during query analysis, where Spark looks up the magic + * method by first converting the actual input SQL data types to their corresponding Java types + * following the mapping defined below, and then checking if there is a matching method from all the + * declared methods in the UDF class, using method name and the Java types. *

- * The following are the mapping from {@link DataType SQL data type} to Java type through - * the magic method approach: + * The following are the mapping from {@link DataType SQL data type} to Java type which is used + * by Spark to infer parameter types for the magic methods as well as return value type for + * {@link #produceResult}: *

    *
  • {@link org.apache.spark.sql.types.BooleanType}: {@code boolean}
    • *
  • {@link org.apache.spark.sql.types.ByteType}: {@code byte}
    • @@ -80,7 +109,8 @@ * {@link org.apache.spark.sql.catalyst.util.MapData} *
* - * @param the JVM type of result values + * @param the JVM type of result values, MUST be consistent with the {@link DataType} + * returned via {@link #resultType()}, according to the mapping above. */ public interface ScalarFunction extends BoundFunction { String MAGIC_METHOD_NAME = "invoke"; diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index b3310635cdcb6..757778b66c345 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.analysis -import java.lang.reflect.Method +import java.lang.reflect.{Method, Modifier} import java.util import java.util.Locale import java.util.concurrent.atomic.AtomicBoolean @@ -2181,6 +2181,9 @@ class Analyzer(override val catalogManager: CatalogManager) // match the input type through `BoundFunction.inputTypes`. val argClasses = inputType.fields.map(_.dataType) findMethod(scalarFunc, MAGIC_METHOD_NAME, argClasses) match { + case Some(m) if Modifier.isStatic(m.getModifiers) => + StaticInvoke(scalarFunc.getClass, scalarFunc.resultType(), + MAGIC_METHOD_NAME, arguments, returnNullable = scalarFunc.isResultNullable) case Some(_) => val caller = Literal.create(scalarFunc, ObjectType(scalarFunc.getClass)) Invoke(caller, MAGIC_METHOD_NAME, scalarFunc.resultType(), diff --git a/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt b/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt new file mode 100644 index 0000000000000..564742f653e57 --- /dev/null +++ b/sql/core/benchmarks/V2FunctionBenchmark-jdk11-results.txt @@ -0,0 +1,44 @@ +OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +scalar function (long + long) -> long, result_nullable = true codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------------------------------------ +native_long_add 17789 18405 580 28.1 35.6 1.0X +java_long_add_default 85058 87073 NaN 5.9 170.1 0.2X +java_long_add_magic 20262 20641 352 24.7 40.5 0.9X +java_long_add_static_magic 19458 19524 105 25.7 38.9 0.9X +scala_long_add_default 85892 86496 560 5.8 171.8 0.2X +scala_long_add_magic 20164 20330 212 24.8 40.3 0.9X + +OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +scalar function (long + long) -> long, result_nullable = false codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------------------------------------- +native_long_add 18290 18467 157 27.3 36.6 1.0X +java_long_add_default 82415 82687 270 6.1 164.8 0.2X +java_long_add_magic 19941 20032 85 25.1 39.9 0.9X +java_long_add_static_magic 17861 17940 92 28.0 35.7 1.0X +scala_long_add_default 83800 85639 NaN 6.0 167.6 0.2X +scala_long_add_magic 20103 20123 18 24.9 40.2 0.9X + +OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +scalar function (long + long) -> long, result_nullable = true codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------------------------------------- +native_long_add 46039 46199 162 10.9 92.1 1.0X +java_long_add_default 113199 113773 720 4.4 226.4 0.4X +java_long_add_magic 158252 159419 1075 3.2 316.5 0.3X +java_long_add_static_magic 157162 157676 516 3.2 314.3 0.3X +scala_long_add_default 112363 113264 1503 4.4 224.7 0.4X +scala_long_add_magic 158122 159010 835 3.2 316.2 0.3X + +OpenJDK 64-Bit Server VM 11.0.11+9-LTS on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz +scalar function (long + long) -> long, result_nullable = false codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------------------------------------- +native_long_add 42685 42743 54 11.7 85.4 1.0X +java_long_add_default 92041 92236 202 5.4 184.1 0.5X +java_long_add_magic 148299 148722 397 3.4 296.6 0.3X +java_long_add_static_magic 140599 141064 442 3.6 281.2 0.3X +scala_long_add_default 91896 92980 959 5.4 183.8 0.5X +scala_long_add_magic 148031 148802 759 3.4 296.1 0.3X + diff --git a/sql/core/benchmarks/V2FunctionBenchmark-results.txt b/sql/core/benchmarks/V2FunctionBenchmark-results.txt new file mode 100644 index 0000000000000..2035aa3633b80 --- /dev/null +++ b/sql/core/benchmarks/V2FunctionBenchmark-results.txt @@ -0,0 +1,44 @@ +OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz +scalar function (long + long) -> long, result_nullable = true codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------------------------------------ +native_long_add 10559 11585 903 47.4 21.1 1.0X +java_long_add_default 78979 80089 987 6.3 158.0 0.1X +java_long_add_magic 14061 14326 305 35.6 28.1 0.8X +java_long_add_static_magic 11971 12150 242 41.8 23.9 0.9X +scala_long_add_default 77254 78565 1254 6.5 154.5 0.1X +scala_long_add_magic 13174 13232 51 38.0 26.3 0.8X + +OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz +scalar function (long + long) -> long, result_nullable = false codegen = true: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------------------------------------- +native_long_add 10489 10665 162 47.7 21.0 1.0X +java_long_add_default 66636 68422 NaN 7.5 133.3 0.2X +java_long_add_magic 13504 14213 883 37.0 27.0 0.8X +java_long_add_static_magic 11726 11984 240 42.6 23.5 0.9X +scala_long_add_default 75906 76130 196 6.6 151.8 0.1X +scala_long_add_magic 14480 14770 261 34.5 29.0 0.7X + +OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz +scalar function (long + long) -> long, result_nullable = true codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------------------------------------------- +native_long_add 39178 39548 323 12.8 78.4 1.0X +java_long_add_default 84756 85509 1092 5.9 169.5 0.5X +java_long_add_magic 199140 200801 1823 2.5 398.3 0.2X +java_long_add_static_magic 203500 207050 NaN 2.5 407.0 0.2X +scala_long_add_default 101180 101421 387 4.9 202.4 0.4X +scala_long_add_magic 193277 197006 1138 2.6 386.6 0.2X + +OpenJDK 64-Bit Server VM 1.8.0_292-b10 on Linux 5.4.0-1046-azure +Intel(R) Xeon(R) CPU E5-2673 v3 @ 2.40GHz +scalar function (long + long) -> long, result_nullable = false codegen = false: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +-------------------------------------------------------------------------------------------------------------------------------------------------------------- +native_long_add 37064 37333 235 13.5 74.1 1.0X +java_long_add_default 104439 107802 NaN 4.8 208.9 0.4X +java_long_add_magic 212496 214321 NaN 2.4 425.0 0.2X +java_long_add_static_magic 239551 240619 1652 2.1 479.1 0.2X +scala_long_add_default 122413 123171 788 4.1 244.8 0.3X +scala_long_add_magic 215912 222715 NaN 2.3 431.8 0.2X + diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java new file mode 100644 index 0000000000000..e2e7136d6f44c --- /dev/null +++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaLongAdd.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test.org.apache.spark.sql.connector.catalog.functions; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.functions.BoundFunction; +import org.apache.spark.sql.connector.catalog.functions.ScalarFunction; +import org.apache.spark.sql.connector.catalog.functions.UnboundFunction; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.LongType; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +public class JavaLongAdd implements UnboundFunction { + private final ScalarFunction impl; + + public JavaLongAdd(ScalarFunction impl) { + this.impl = impl; + } + + @Override + public String name() { + return "long_add"; + } + + @Override + public BoundFunction bind(StructType inputType) { + if (inputType.fields().length != 2) { + throw new UnsupportedOperationException("Expect two arguments"); + } + StructField[] fields = inputType.fields(); + if (!(fields[0].dataType() instanceof LongType)) { + throw new UnsupportedOperationException("Expect first argument to be LongType"); + } + if (!(fields[1].dataType() instanceof LongType)) { + throw new UnsupportedOperationException("Expect second argument to be LongType"); + } + return impl; + } + + @Override + public String description() { + return "long_add"; + } + + private abstract static class JavaLongAddBase implements ScalarFunction { + private final boolean isResultNullable; + + JavaLongAddBase(boolean isResultNullable) { + this.isResultNullable = isResultNullable; + } + + @Override + public DataType[] inputTypes() { + return new DataType[] { DataTypes.LongType, DataTypes.LongType }; + } + + @Override + public DataType resultType() { + return DataTypes.LongType; + } + + @Override + public boolean isResultNullable() { + return isResultNullable; + } + } + + public static class JavaLongAddDefault extends JavaLongAddBase { + public JavaLongAddDefault(boolean isResultNullable) { + super(isResultNullable); + } + + @Override + public String name() { + return "long_add_default"; + } + + @Override + public Long produceResult(InternalRow input) { + return input.getLong(0) + input.getLong(1); + } + } + + public static class JavaLongAddMagic extends JavaLongAddBase { + public JavaLongAddMagic(boolean isResultNullable) { + super(isResultNullable); + } + + @Override + public String name() { + return "long_add_magic"; + } + + public long invoke(long left, long right) { + return left + right; + } + } + + public static class JavaLongAddStaticMagic extends JavaLongAddBase { + public JavaLongAddStaticMagic(boolean isResultNullable) { + super(isResultNullable); + } + + @Override + public String name() { + return "long_add_static_magic"; + } + + public static long invoke(long left, long right) { + return left + right; + } + } +} diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java index 8b2d883a3703f..7cd010b9365bb 100644 --- a/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java +++ b/sql/core/src/test/java/test/org/apache/spark/sql/connector/catalog/functions/JavaStrLen.java @@ -58,7 +58,7 @@ public String description() { " strlen(string) -> int"; } - public static class JavaStrLenDefault implements ScalarFunction { + private abstract static class JavaStrLenBase implements ScalarFunction { @Override public DataType[] inputTypes() { return new DataType[] { DataTypes.StringType }; @@ -73,7 +73,9 @@ public DataType resultType() { public String name() { return "strlen"; } + } + public static class JavaStrLenDefault extends JavaStrLenBase { @Override public Integer produceResult(InternalRow input) { String str = input.getString(0); @@ -81,42 +83,42 @@ public Integer produceResult(InternalRow input) { } } - public static class JavaStrLenMagic implements ScalarFunction { - @Override - public DataType[] inputTypes() { - return new DataType[] { DataTypes.StringType }; + public static class JavaStrLenMagic extends JavaStrLenBase { + public int invoke(UTF8String str) { + return str.toString().length(); } + } - @Override - public DataType resultType() { - return DataTypes.IntegerType; + public static class JavaStrLenStaticMagic extends JavaStrLenBase { + public static int invoke(UTF8String str) { + return str.toString().length(); } + } + public static class JavaStrLenBoth extends JavaStrLenBase { @Override - public String name() { - return "strlen"; + public Integer produceResult(InternalRow input) { + String str = input.getString(0); + return str.length(); } - public int invoke(UTF8String str) { - return str.toString().length(); + return str.toString().length() + 100; } } - public static class JavaStrLenNoImpl implements ScalarFunction { - @Override - public DataType[] inputTypes() { - return new DataType[] { DataTypes.StringType }; + // even though the static magic method is present, it has incorrect parameter type and so Spark + // should fallback to the non-static magic method + public static class JavaStrLenBadStaticMagic extends JavaStrLenBase { + public static int invoke(String str) { + return str.length(); } - @Override - public DataType resultType() { - return DataTypes.IntegerType; + public int invoke(UTF8String str) { + return str.toString().length() + 100; } + } - @Override - public String name() { - return "strlen"; - } + public static class JavaStrLenNoImpl extends JavaStrLenBase { } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala index fe856ffecb84a..b269da39daf38 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala @@ -183,6 +183,28 @@ class DataSourceV2FunctionSuite extends DatasourceV2SQLBase { checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil) } + test("scalar function: static magic method in Java") { + catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps) + addFunction(Identifier.of(Array("ns"), "strlen"), + new JavaStrLen(new JavaStrLenStaticMagic)) + checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(3) :: Nil) + } + + test("scalar function: magic method should take higher precedence in Java") { + catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps) + addFunction(Identifier.of(Array("ns"), "strlen"), + new JavaStrLen(new JavaStrLenBoth)) + // to differentiate, the static method returns string length + 100 + checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(103) :: Nil) + } + + test("scalar function: bad static magic method should fallback to non-static") { + catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps) + addFunction(Identifier.of(Array("ns"), "strlen"), + new JavaStrLen(new JavaStrLenBadStaticMagic)) + checkAnswer(sql("SELECT testcat.ns.strlen('abc')"), Row(103) :: Nil) + } + test("scalar function: no implementation found in Java") { catalog("testcat").asInstanceOf[SupportsNamespaces].createNamespace(Array("ns"), emptyProps) addFunction(Identifier.of(Array("ns"), "strlen"), diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala new file mode 100644 index 0000000000000..9328a9a8e93e3 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/functions/V2FunctionBenchmark.scala @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.connector.functions + +import test.org.apache.spark.sql.connector.catalog.functions.JavaLongAdd +import test.org.apache.spark.sql.connector.catalog.functions.JavaLongAdd.{JavaLongAddDefault, JavaLongAddMagic, JavaLongAddStaticMagic} + +import org.apache.spark.benchmark.Benchmark +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.catalyst.expressions.{BinaryArithmetic, Expression} +import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._ +import org.apache.spark.sql.catalyst.util.TypeUtils +import org.apache.spark.sql.connector.catalog.{Identifier, InMemoryCatalog} +import org.apache.spark.sql.connector.catalog.functions.{BoundFunction, ScalarFunction, UnboundFunction} +import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{AbstractDataType, DataType, LongType, NumericType, StructType} + +/** + * Benchmark to measure DataSourceV2 UDF performance + * {{{ + * To run this benchmark: + * 1. without sbt: + * bin/spark-submit --class + * --jars , + * 2. build/sbt "sql/test:runMain " + * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain " + * Results will be written to "benchmarks/V2FunctionBenchmark-results.txt". + * }}} + * '''NOTE''': to update the result of this benchmark, please use Github benchmark action: + * https://spark.apache.org/developer-tools.html#github-workflow-benchmarks + */ +object V2FunctionBenchmark extends SqlBasedBenchmark { + val catalogName: String = "benchmark_catalog" + + override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { + val N = 500L * 1000 * 1000 + Seq(true, false).foreach { codegenEnabled => + Seq(true, false).foreach { resultNullable => + scalarFunctionBenchmark(N, codegenEnabled = codegenEnabled, + resultNullable = resultNullable) + } + } + } + + private def scalarFunctionBenchmark( + N: Long, + codegenEnabled: Boolean, + resultNullable: Boolean): Unit = { + withSQLConf(s"spark.sql.catalog.$catalogName" -> classOf[InMemoryCatalog].getName) { + createFunction("java_long_add_default", + new JavaLongAdd(new JavaLongAddDefault(resultNullable))) + createFunction("java_long_add_magic", new JavaLongAdd(new JavaLongAddMagic(resultNullable))) + createFunction("java_long_add_static_magic", + new JavaLongAdd(new JavaLongAddStaticMagic(resultNullable))) + createFunction("scala_long_add_default", + LongAddUnbound(new LongAddWithProduceResult(resultNullable))) + createFunction("scala_long_add_magic", LongAddUnbound(new LongAddWithMagic(resultNullable))) + + val codeGenFactoryMode = if (codegenEnabled) FALLBACK else NO_CODEGEN + withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> codegenEnabled.toString, + SQLConf.CODEGEN_FACTORY_MODE.key -> codeGenFactoryMode.toString) { + val name = s"scalar function (long + long) -> long, result_nullable = $resultNullable " + + s"codegen = $codegenEnabled" + val benchmark = new Benchmark(name, N, output = output) + benchmark.addCase(s"native_long_add", numIters = 3) { _ => + spark.range(N).select(Column(NativeAdd($"id".expr, $"id".expr, resultNullable))).noop() + } + Seq("java_long_add_default", "java_long_add_magic", "java_long_add_static_magic", + "scala_long_add_default", "scala_long_add_magic").foreach { functionName => + benchmark.addCase(s"$functionName", numIters = 3) { _ => + spark.range(N).selectExpr(s"$catalogName.$functionName(id, id)").noop() + } + } + benchmark.run() + } + } + } + + private def createFunction(name: String, fn: UnboundFunction): Unit = { + val catalog = spark.sessionState.catalogManager.catalog(catalogName) + val ident = Identifier.of(Array.empty, name) + catalog.asInstanceOf[InMemoryCatalog].createFunction(ident, fn) + } + + case class NativeAdd( + left: Expression, + right: Expression, + override val nullable: Boolean) extends BinaryArithmetic { + override protected val failOnError: Boolean = true + override def inputType: AbstractDataType = NumericType + override def symbol: String = "+" + override def exactMathMethod: Option[String] = Some("addExact") + + private lazy val numeric = TypeUtils.getNumeric(dataType, failOnError) + protected override def nullSafeEval(input1: Any, input2: Any): Any = + numeric.plus(input1, input2) + + override protected def withNewChildrenInternal( + newLeft: Expression, + newRight: Expression): NativeAdd = copy(left = newLeft, right = newRight) + } + + case class LongAddUnbound(impl: ScalarFunction[Long]) extends UnboundFunction { + override def bind(inputType: StructType): BoundFunction = impl + override def description(): String = name() + override def name(): String = "long_add_unbound" + } + + abstract class LongAddBase(resultNullable: Boolean) extends ScalarFunction[Long] { + override def inputTypes(): Array[DataType] = Array(LongType, LongType) + override def resultType(): DataType = LongType + override def isResultNullable: Boolean = resultNullable + } + + class LongAddWithProduceResult(resultNullable: Boolean) extends LongAddBase(resultNullable) { + override def produceResult(input: InternalRow): Long = { + input.getLong(0) + input.getLong(1) + } + override def name(): String = "long_add_default" + } + + class LongAddWithMagic(resultNullable: Boolean) extends LongAddBase(resultNullable) { + def invoke(left: Long, right: Long): Long = { + left + right + } + override def name(): String = "long_add_magic" + } +} +