-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Upon job end, the driver may call a method to both clean the tracking hashmap and deregister the source from the MetricsSystem. The new method will remove all metrics belonging to a given job UUID.
- Loading branch information
Showing
4 changed files
with
103 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
package org.apache.spark.datasource | ||
|
||
import org.scalatest.{FlatSpec, Matchers} | ||
import org.apache.spark.SparkEnv | ||
import org.apache.spark.sql.SparkSession | ||
|
||
class MetricsSourceTest extends FlatSpec with Matchers { | ||
val spark: SparkSession = SparkSession | ||
.builder() | ||
.master("local[*]") | ||
.config("spark.ui.enabled", "false") // comment this out to use Spark UI during tests, on https://localhost:4040 by default | ||
// https://medium.com/@mrpowers/how-to-cut-the-run-time-of-a-spark-sbt-test-suite-by-40-52d71219773f | ||
.config("spark.sql.shuffle.partitions", "1") | ||
.config("spark.sql.storeAssignmentPolicy", "legacy") | ||
.config("spark.app.id", this.getClass.getName + math.floor(math.random() * 1000).toLong.toString) | ||
.getOrCreate() | ||
|
||
"A MetricsSource" should "register a new metric only once" in { | ||
val metric = MetricsSource.getOrCreateCounter("prefix", "name") | ||
val sameMetric = MetricsSource.getOrCreateCounter("prefix", "name") | ||
|
||
metric.getCount() should equal(0) | ||
sameMetric.getCount() should equal(0) | ||
|
||
metric.inc() | ||
|
||
metric.getCount() should equal(1) | ||
sameMetric.getCount() should equal(1) | ||
} | ||
|
||
it should "deregister a metric only once" in { | ||
MetricsSource.getOrCreateCounter("removePrefix.jobId", "name") | ||
MetricsSource.getOrCreateCounter("removePrefix.otherJobId", "name") | ||
|
||
Option(MetricsSource.metricsMap.get("removePrefix.jobId.name")) shouldBe 'defined | ||
Option(MetricsSource.metricsMap.get("removePrefix.otherJobId.name")) shouldBe 'defined | ||
SparkEnv.get.metricsSystem.getSourcesByName("removePrefix.jobId").size should equal(1) | ||
SparkEnv.get.metricsSystem.getSourcesByName("removePrefix.otherJobId").size should equal(1) | ||
|
||
MetricsSource.removeJobMetrics("removePrefix", "jobId") | ||
|
||
Option(MetricsSource.metricsMap.get("removePrefix.jobId.name")) shouldBe 'empty | ||
Option(MetricsSource.metricsMap.get("removePrefix.otherJobId.name")) shouldBe 'defined | ||
SparkEnv.get.metricsSystem.getSourcesByName("removePrefix.jobId") shouldBe 'empty | ||
SparkEnv.get.metricsSystem.getSourcesByName("removePrefix.otherJobId").size should equal(1) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters